{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install iterative-stratification --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T21:49:44.240206Z","iopub.execute_input":"2025-12-06T21:49:44.240446Z","iopub.status.idle":"2025-12-06T21:49:48.687118Z","shell.execute_reply.started":"2025-12-06T21:49:44.240421Z","shell.execute_reply":"2025-12-06T21:49:48.686148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport cv2 \nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nfrom torch import optim\nimport timm  \nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:06.851534Z","iopub.execute_input":"2025-12-06T19:51:06.852072Z","iopub.status.idle":"2025-12-06T19:51:17.884156Z","shell.execute_reply.started":"2025-12-06T19:51:06.852040Z","shell.execute_reply":"2025-12-06T19:51:17.883559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    # --- (Paths) ---\n    BASE_PATH = '/kaggle/input/csiro-biomass'\n    TRAIN_CSV = os.path.join(BASE_PATH, 'train.csv')\n    IMAGE_DIR = os.path.join(BASE_PATH, 'train')\n    \n    MODEL_NAME = 'convnext_small.fb_in22k_ft_in1k'\n    MODEL_NAME = 'convnext_tiny.fb_in22k_ft_in1k'\n    # MODEL_NAME = 'convnext_base.fb_in22k_ft_in1k'\n    \n    IMG_SIZE = 768\n    fold_num = 4\n    \n    # ---  ---\n    BATCH_SIZE = 2\n    EPOCHS = 30\n    LEARNING_RATE = 1e-4    \n    NUM_WORKERS = 2  \n    \n    # --- Loss ---\n    TARGET_COLS = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g']\n\n    ALL_TARGET_COLS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    R2_WEIGHTS = [0.1, 0.1, 0.1, 0.2, 0.5]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:17.885374Z","iopub.execute_input":"2025-12-06T19:51:17.885755Z","iopub.status.idle":"2025-12-06T19:51:17.890604Z","shell.execute_reply.started":"2025-12-06T19:51:17.885737Z","shell.execute_reply":"2025-12-06T19:51:17.889869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from albumentations import (\n    Compose, \n    Resize, \n    Normalize,\n    HorizontalFlip, \n    VerticalFlip,\n    RandomRotate90, \n    ColorJitter\n)\n\n# --- Augmentations ---\ntrain_aug = Compose([\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        RandomRotate90(p=0.5),\n\n        ColorJitter(\n            brightness=0.2, \n            contrast=0.2, \n            saturation=0.2, \n            hue=0.1, \n            p=0.75\n        ),\n        Resize(Config.IMG_SIZE, Config.IMG_SIZE),\n\n        Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n        ),\n\n        ToTensorV2()\n    ])\n\nvalid_aug = Compose([\n        Resize(Config.IMG_SIZE, Config.IMG_SIZE),\n        \n        Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n        ),\n        ToTensorV2()\n    ])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:17.891449Z","iopub.execute_input":"2025-12-06T19:51:17.891701Z","iopub.status.idle":"2025-12-06T19:51:17.909915Z","shell.execute_reply.started":"2025-12-06T19:51:17.891676Z","shell.execute_reply":"2025-12-06T19:51:17.909142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, \n                 transform, \n                 image_dir, \n                 train_target_cols, \n                 all_target_cols,\n                 fit_scaler: True\n                ):\n        self.df = df\n        self.transform = transform\n        self.image_dir = image_dir\n        \n        self.image_paths = df['image_path'].values\n        self.train_targets = df[train_target_cols].values\n        self.all_targets = df[all_target_cols].values\n\n        self.scaler = None\n        if fit_scaler:\n            self.scaler = StandardScaler()\n            # log1p handles zeros perfectly and reduces skew\n            self.train_targets = self.scaler.fit_transform(self.train_targets)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_path_suffix = self.image_paths[idx]\n        train_target_vals = self.train_targets[idx]\n        all_target_vals = self.all_targets[idx]\n        \n        # 2. (2000x1000)\n        filename = os.path.basename(img_path_suffix)\n        full_path = os.path.join(self.image_dir, filename)\n        image = cv2.imread(full_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        # 3. \n        height, width, _ = image.shape\n        mid_point = width // 2\n        img_left = image[:, :mid_point]\n        img_right = image[:, mid_point:]\n        \n        # 4.\n        img_left_tensor = self.transform(image=img_left)['image']\n        img_right_tensor = self.transform (image=img_right)['image']\n        \n        # 5. Targets\n        train_target_tensor = torch.tensor(train_target_vals, dtype=torch.float32)\n        all_targets_tensor = torch.tensor(all_target_vals, dtype=torch.float32)\n        \n        return img_left_tensor, img_right_tensor, train_target_tensor, all_targets_tensor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:17.911096Z","iopub.execute_input":"2025-12-06T19:51:17.911267Z","iopub.status.idle":"2025-12-06T19:51:18.433541Z","shell.execute_reply.started":"2025-12-06T19:51:17.911254Z","shell.execute_reply":"2025-12-06T19:51:18.432757Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassRegressionHead(nn.Module):\n    def __init__(self, target_name, n_features = 768, dropout = 0.3, neurons = 384):\n        super(BiomassRegressionHead, self).__init__()\n        self.target_name = target_name\n        self.head = nn.Sequential(\n            nn.Linear(n_features, neurons),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(neurons, 1)\n        )\n\n    def forward(self, features):\n        return self.head(features)\n\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, pretrained = True, n_targets=3, neurons = 384, dropout=0.3):\n        super(BiomassModel, self).__init__()\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg',\n            # drop_path_rate=0.2 # optional\n        )\n        self.n_features = self.backbone.num_features\n        self.n_combined_features = self.n_features * 2\n        \n        self.head_dry_clover = BiomassRegressionHead(target_name = 'Dry_Clover_g', n_features = self.n_combined_features,neurons = neurons, dropout = dropout)\n        self.head_dry_dead = BiomassRegressionHead(target_name = 'Dry_Dead_g', n_features = self.n_combined_features,neurons = neurons, dropout = dropout)\n        self.head_dry_green = BiomassRegressionHead(target_name = 'Dry_Green_g', n_features = self.n_combined_features,neurons = neurons, dropout = dropout)\n        \n\n    def forward(self, img_left, img_right):\n        features_left = self.backbone(img_left) # Shape: [batch, n_features]\n        features_right = self.backbone(img_right) # Shape: [batch, n_features]\n    \n        combined = torch.cat([features_left, features_right], dim=1) # Shape: [batch, 2 * n_features]\n        \n        dry_clover = self.head_dry_clover(combined)\n        dry_dead = self.head_dry_dead(combined)\n        dry_green = self.head_dry_green(combined)\n        \n        return dry_clover, dry_dead, dry_green","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:24.911160Z","iopub.execute_input":"2025-12-06T19:51:24.911868Z","iopub.status.idle":"2025-12-06T19:51:24.920236Z","shell.execute_reply.started":"2025-12-06T19:51:24.911840Z","shell.execute_reply":"2025-12-06T19:51:24.919371Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## KFold Splitting","metadata":{}},{"cell_type":"code","source":"df_long = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ndf_wide = df_long.pivot(\n    index='image_path',\n    columns='target_name',\n    values='target'\n)\ndf_wide = df_wide.reset_index()\ndf_wide.columns.name = None\n\nmeta_cols = ['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']\nmeta_df = df_long[meta_cols].drop_duplicates()\ndf_wide = df_wide.merge(meta_df, on='image_path', how='left')\n\n\n# Manual mapping for known inconsistencies\nspecies_corrections = {\n    'Barleygrass': 'BarleyGrass',\n}\n\ndef normalize_species(species_str):\n    tokens = species_str.split('_')\n    normalized = [species_corrections.get(s, s) for s in tokens]\n    return '_'.join(sorted(normalized))\n\ndf_wide['Species'] = df_wide['Species'].apply(normalize_species)\ndf_wide","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:27.571775Z","iopub.execute_input":"2025-12-06T19:51:27.572304Z","iopub.status.idle":"2025-12-06T19:51:27.637620Z","shell.execute_reply.started":"2025-12-06T19:51:27.572284Z","shell.execute_reply":"2025-12-06T19:51:27.636980Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\ndf = df_wide.copy()\n\n# 1. Total biomass bin (must be balanced)\ndf['total_bin'] = pd.qcut(df['Dry_Total_g'], q=10, labels=False, duplicates='drop')\n\n# 2. Season (Australian seasons)\ndf['Sampling_Date'] = pd.to_datetime(df['Sampling_Date'])\ndf['month'] = df['Sampling_Date'].dt.month\ndf['season'] = ((df['month'] + 9) % 12) // 3 + 1   # 1=Spring, 2=Summer, 3=Autumn, 4=Winter\n\n# 3. Binned continuous\ndf['ndvi_bin'] = pd.qcut(df['Pre_GSHH_NDVI'], q=5, labels=False, duplicates='drop')\ndf['height_bin'] = pd.qcut(df['Height_Ave_cm'], q=5, labels=False, duplicates='drop')\n\n# 4. State as categorical\ndf['state_enc'] = df['State'].astype('category').cat.codes\n\n\n\n# ==================== MULTI-HOT SPECIES ====================\n\nall_species = ['Ryegrass', 'Clover', 'Phalaris', 'Lucerne', 'Fescue', 'BarleyGrass',\n               'Capeweed', 'SilverGrass', 'SpearGrass', 'Bromegrass', 'CrumbWeed',\n               'SubcloverDalkeith', 'SubcloverLosa', 'WhiteClover']\n\n# Manual mapping for known variants\nspecies_map = {\n    'SubcloverDalkeith': 'Clover',\n    'SubcloverLosa': 'Clover',\n    'WhiteClover': 'Clover',\n}\n\ndef get_species_set(species_str):\n    if pd.isna(species_str):\n        return set()\n    tokens = species_str.split('_')\n    normalized = [species_map.get(t, t) for t in tokens]\n    # Special case: \"Mixed\" = Ryegrass + Clover\n    if 'Mixed' in tokens:\n        return {'Ryegrass', 'Clover'}\n    return set(normalized)\n\n# Create multi-hot columns\nfor sp in all_species:\n    df[f'sp_{sp}'] = df['Species'].apply(lambda x: 1 if sp in get_species_set(x) else 0)\n\n\n# =====================================================================\n\n# Final multilabel array — now includes individual species presence!\nlabel_columns = ['total_bin', 'state_enc', 'season', 'ndvi_bin', 'height_bin'] + [f'sp_{sp}' for sp in all_species]\n\nlabels = df[label_columns].values.astype(int)\n\n# Multilabel Stratified K-Fold\nmskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ndf['fold'] = -1\nfor fold, (_, val_idx) in enumerate(mskf.split(df, labels)):\n    df.loc[val_idx, 'fold'] = fold\n\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:29.293980Z","iopub.execute_input":"2025-12-06T19:51:29.294618Z","iopub.status.idle":"2025-12-06T19:51:29.431447Z","shell.execute_reply.started":"2025-12-06T19:51:29.294594Z","shell.execute_reply":"2025-12-06T19:51:29.430861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = ['image_path',\t'Dry_Clover_g',\t'Dry_Dead_g',\t'Dry_Green_g',\t'Dry_Total_g',\t'GDM_g']\ndf_val = df[df.fold == Config.fold_num].copy()\ndf_train = df[df.fold != Config.fold_num].copy()\n\ndf_val = df_val[cols]\ndf_train = df_train[cols]\ndf_val","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:32.703105Z","iopub.execute_input":"2025-12-06T19:51:32.703833Z","iopub.status.idle":"2025-12-06T19:51:32.721278Z","shell.execute_reply.started":"2025-12-06T19:51:32.703809Z","shell.execute_reply":"2025-12-06T19:51:32.720590Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = BiomassDataset(\n    df = df_train,\n    transform = train_aug,\n    image_dir = Config.IMAGE_DIR,\n    train_target_cols = Config.TARGET_COLS,\n    all_target_cols = Config.ALL_TARGET_COLS,\n    fit_scaler = False\n)\n\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=Config.BATCH_SIZE,\n    shuffle=True,\n    num_workers=Config.NUM_WORKERS\n)\n\nval_dataset = BiomassDataset(\n    df = df_val,\n    transform = valid_aug,\n    image_dir = Config.IMAGE_DIR,\n    train_target_cols = Config.TARGET_COLS,\n    all_target_cols = Config.ALL_TARGET_COLS,\n    fit_scaler = False\n)\n\nval_loader = DataLoader(\n    val_dataset,\n    batch_size = Config.BATCH_SIZE,\n    shuffle = False,\n    num_workers = Config.NUM_WORKERS\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:35.668821Z","iopub.execute_input":"2025-12-06T19:51:35.669312Z","iopub.status.idle":"2025-12-06T19:51:35.677263Z","shell.execute_reply.started":"2025-12-06T19:51:35.669287Z","shell.execute_reply":"2025-12-06T19:51:35.676547Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"def custom_loss(preds, y_true):\n    dry_green  = torch.clamp(preds['dry_green'],  min=0.0)\n    dry_dead   = torch.clamp(preds['dry_dead'],   min=0.0)\n    dry_clover = torch.clamp(preds['dry_clover'], min=0.0)\n\n    # Compute derived targets\n    gdm = dry_green + dry_clover\n    dry_total = gdm + dry_dead\n\n    # Stack predictions and ground truths\n    y_preds = torch.stack([dry_green, dry_dead, dry_clover, gdm, dry_total], dim=1)\n    y_true  = y_true.to(y_preds.device)\n\n    # Define weights per target type\n    weights = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5], dtype=y_preds.dtype, device=y_preds.device)  # shape [5]\n\n    # Flatten all values and assign weights\n    y_true_flat  = y_true.flatten()\n    y_preds_flat = y_preds.flatten()\n    weights_flat = weights.repeat(y_true.shape[0])  # expand weights across samples\n\n    # Compute weighted mean\n    y_mean = torch.sum(weights_flat * y_true_flat) / torch.sum(weights_flat)\n\n    # Compute weighted residual and total sum of squares\n    ss_res = torch.sum(weights_flat * (y_true_flat - y_preds_flat) ** 2)\n    ss_tot = torch.sum(weights_flat * (y_true_flat - y_mean) ** 2)\n\n    # Final weighted R²\n    # r2_weighted = 1 - ss_res / ss_tot\n    return ss_res / ss_tot \n\ndef train_one_epoch(model, loader, criterion, optimizer, device):\n    model.train() \n    epoch_loss = 0.0\n    step = 0    \n\n    debug_step = len(loader) // 10\n\n    for (img_left, img_right, train_targets, all_targets) in tqdm(loader, desc=\"Training\", leave=False):\n        step += 1\n            \n        # 1. device (GPU/CPU)\n        img_left = img_left.to(device)\n        img_right = img_right.to(device)\n        targets = train_targets.to(device) # Chỉ 3 mục tiêu cho loss\n        \n        optimizer.zero_grad()\n        \n        # 3. Forward pass \n        dry_clover, dry_dead, dry_green = model(img_left, img_right)\n        \n        preds = {\n            \"dry_clover\": dry_clover,\n            \"dry_dead\": dry_dead,\n            \"dry_green\": dry_green\n        }\n\n        \n        # # 4. Loss \n        # dry_clover_loss = criterion(dry_clover, targets[:,0].unsqueeze(-1))\n        # dry_dead_loss = criterion(dry_dead, targets[:,1].unsqueeze(-1))\n        # dry_green_loss = criterion(dry_green, targets[:,2].unsqueeze(-1))\n\n        # loss = (dry_clover_loss + dry_dead_loss + dry_green_loss) / 3\n        loss = custom_loss(preds, all_targets)\n\n    \n        loss.backward()\n        optimizer.step()\n        \n        epoch_loss += loss.item()\n\n        if step % debug_step == 0 or step == len(loader):\n            print(\n                f\"\\t[+] step:     {step}/{len(loader)}\\n\"\n                f\"\\t[+] avg_loss: {loss.item():.5f}\"\n            )\n            # print(\n            #     f\"\\t[+] step: {step}/{len(loader)}\\n\"\n            #     f\"\\t[+] dry_clover_loss: {dry_clover_loss:.5f}\\n\"\n            #     f\"\\t[+] dry_dead_loss: {dry_dead_loss:.5f}\\n\"\n            #     f\"\\t[+] dry_green_loss: {dry_green_loss:.5f}\\n\"\n            #     f\"\\t[+] avg_loss: {loss.item():.5f}\\n\"\n            # )\n\n    return epoch_loss / len(loader)\n\n\ndef biomass_r2_score(preds, y_true):\n    # Extract predictions\n    dry_green = np.maximum(preds['dry_green'], 0)\n    dry_dead = np.maximum(preds['dry_dead'], 0)\n    dry_clover = np.maximum(preds['dry_clover'], 0)\n\n    # Compute derived targets\n    gdm = dry_green + dry_clover\n    dry_total = gdm + dry_dead\n\n    # Stack predictions and ground truths\n    y_preds = np.stack([dry_green, dry_dead, dry_clover, gdm, dry_total], axis=1)\n    y_true = np.asarray(y_true)  # shape [N, 5]\n\n    # Define weights per target type\n    weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])  # shape [5]\n\n    # Flatten all values and assign weights\n    y_true_flat = y_true.flatten()\n    y_preds_flat = y_preds.flatten()\n    weights_flat = np.repeat(weights, y_true.shape[0])\n\n    # Compute weighted mean\n    y_mean = np.sum(weights_flat * y_true_flat) / np.sum(weights_flat)\n\n    # Compute weighted residual and total sum of squares\n    ss_res = np.sum(weights_flat * (y_true_flat - y_preds_flat) ** 2)\n    ss_tot = np.sum(weights_flat * (y_true_flat - y_mean) ** 2)\n\n    # Final weighted R²\n    r2_weighted = 1 - ss_res / ss_tot\n    return r2_weighted\n\ndef validate_one_epoch(model, loader, scaler, device):\n    model.eval()\n    preds = {\n        \"dry_clover\": [],\n        \"dry_dead\": [],\n        \"dry_green\": []\n    }\n    all_targets = []\n    for (img_left, img_right, _, true_5_targets) in tqdm(loader, desc=\"Validation\", leave=False):\n        img_left = img_left.to(device)\n        img_right = img_right.to(device)\n        all_targets.append(true_5_targets.numpy())\n        with torch.no_grad():        \n            dry_clover, dry_dead, dry_green = model(img_left, img_right)\n\n        if scaler is not None:\n            scaled_pred = torch.cat([dry_clover, dry_dead, dry_green], dim=1)   # shape: [batch, 3]\n            scaled_pred_np = scaled_pred.cpu().numpy()\n            \n            # Step 3: Inverse scaling → back to log1p space → back to real grams\n            unscaled_real = scaler.inverse_transform(scaled_pred_np)   # inverse StandardScaler\n\n            # Clip negative values\n            unscaled_real = np.clip(unscaled_real, a_min=0, a_max=None)\n\n            preds['dry_clover'].extend(unscaled_real[:, 0].tolist())\n            preds['dry_dead'].extend(unscaled_real[:, 1].tolist())\n            preds['dry_green'].extend(unscaled_real[:, 2].tolist())\n\n        else:\n            preds['dry_clover'].extend(dry_clover.squeeze().cpu().tolist())\n            preds['dry_dead'].extend(dry_dead.squeeze().cpu().tolist())\n            preds['dry_green'].extend(dry_green.squeeze().cpu().tolist())\n\n    all_targets = np.concatenate(all_targets, axis=0)\n    preds['dry_clover'] = np.array(preds['dry_clover'])\n    preds['dry_dead'] = np.array(preds['dry_dead'])\n    preds['dry_green'] = np.array(preds['dry_green'])\n    \n    r2_score = biomass_r2_score(preds, all_targets)\n    del preds, all_targets\n    return r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:40.870767Z","iopub.execute_input":"2025-12-06T19:51:40.871045Z","iopub.status.idle":"2025-12-06T19:51:40.886937Z","shell.execute_reply.started":"2025-12-06T19:51:40.871027Z","shell.execute_reply":"2025-12-06T19:51:40.886131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.optim.lr_scheduler import ReduceLROnPlateau\n\nmodel_name = Config.MODEL_NAME # = 'convnext_small.fb_in22k_ft_in1k'\nmodel = BiomassModel(model_name = model_name, dropout=0.3).to(device)\nprint(f\"[+] Total N° of parameters: {sum(p.numel() for p in model.parameters())}\")\n\n# criterion = nn.SmoothL1Loss()\ncriterion = nn.MSELoss()\noptimizer = optim.Adam(\n        model.parameters(),\n        lr=Config.LEARNING_RATE\n    )\n\n\nscheduler = ReduceLROnPlateau(optimizer,\n    mode='max',              \n    factor=0.4,              # reduce LR by a factor of\n    patience=2,              # wait 3 epochs with no improvement\n    min_lr=1e-6              # don't go below this LR\n)\n\nscaler = train_dataset.scaler","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:46.172228Z","iopub.execute_input":"2025-12-06T19:51:46.172825Z","iopub.status.idle":"2025-12-06T19:51:50.082821Z","shell.execute_reply.started":"2025-12-06T19:51:46.172800Z","shell.execute_reply":"2025-12-06T19:51:50.082051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_score = -float('inf')\nval_scores = []\ntrain_losses = []\nfor epoch in tqdm(range(Config.EPOCHS)):\n    print(f\"[+] Epoch : {epoch}\")\n    epoch_loss = train_one_epoch(model, train_loader, criterion, optimizer, device)\n    val_score = validate_one_epoch(model, val_loader, scaler, device)\n    \n    val_scores.append(val_score)\n    train_losses.append(epoch_loss)\n    scheduler.step(val_score)\n    print(f\"[+] Epoch Loss: {epoch_loss:.5f}\")\n    \n    if val_score > best_score:\n        \n        best_score = val_score\n        torch.save({\n            'model_state_dict': model.state_dict(),\n            'scaler': scaler,\n            \"model_name\": model_name, \n            'val_score': best_score,\n        }, f'best_model_fold_{Config.fold_num}_model_{model_name}.pth')\n\n        print(f\"✨ New Best: {val_score:.5f}\")\n    else:\n        print(f\"r2_score: {val_score:.5f} / {best_score:.5f}\")\n    \n\n    # Debug\n    torch.cuda.empty_cache()\n    print(f\"=========================\\n\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T19:51:55.824131Z","iopub.execute_input":"2025-12-06T19:51:55.824437Z","iopub.status.idle":"2025-12-06T21:15:02.308659Z","shell.execute_reply.started":"2025-12-06T19:51:55.824414Z","shell.execute_reply":"2025-12-06T21:15:02.307823Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(10, 4))  # two subplots side by side\n\n# --- Train Loss subplot ---\naxes[0].plot(train_losses, label='train_loss', color='blue')\naxes[0].set_title(\"Train Loss\")\naxes[0].grid(True)\n\n# Add horizontal line at min loss\nmin_loss = min(train_losses)\naxes[0].axhline(y=min_loss, color='red', linestyle='--', label='min loss')\naxes[0].text(3, min_loss, f\"min={min_loss:.4f}\", \n             va='bottom', ha='center', color='red')\n\naxes[0].legend()\n\n# --- Validation Score subplot ---\naxes[1].plot(val_scores, label='val_score', color='orange')\naxes[1].set_title(\"Validation Score\")\naxes[1].grid(True)\n\n# Add horizontal line at max val_score\nmax_val = max(val_scores)\naxes[1].axhline(y=max_val, color='blue', linestyle='--', label='max val')\naxes[1].text(3, max_val, f\"max={max_val:.4f}\", \n             va='bottom', ha='center', color='blue')\n\naxes[1].legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T21:15:07.916928Z","iopub.execute_input":"2025-12-06T21:15:07.917757Z","iopub.status.idle":"2025-12-06T21:15:08.326378Z","shell.execute_reply.started":"2025-12-06T21:15:07.917723Z","shell.execute_reply":"2025-12-06T21:15:08.325685Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"checkpoint = torch.load(f'best_model_fold_{Config.fold_num}_model_{model_name}.pth', map_location=device, weights_only=False)\nmodel_name = checkpoint['model_name']\nmodel = BiomassModel(model_name=model_name, pretrained=False).to(device)\nmodel.load_state_dict(checkpoint['model_state_dict'])\nmodel.eval()\n\n# Retrieve the best validation score\nbest_score = checkpoint['val_score']\nscaler = checkpoint['scaler']\n\nprint(f\"Model Name: {model_name}\")\nprint(f\"✅ Loaded model with best val_score: {best_score:.5f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:46:20.209722Z","iopub.execute_input":"2025-12-06T18:46:20.210009Z","iopub.status.idle":"2025-12-06T18:46:21.978249Z","shell.execute_reply.started":"2025-12-06T18:46:20.209987Z","shell.execute_reply":"2025-12-06T18:46:21.977603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"row = df_val.iloc[np.random.randint(len(val_dataset))]\nimage_path = row.image_path\nimage_path = os.path.join('/kaggle/input/csiro-biomass', image_path)\nimage = cv2.imread(image_path)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n# 3. Split Image\nheight, width, _ = image.shape\nmid_point = width // 2\niml_np = image[:, :mid_point]\nimr_np = image[:, mid_point:]\niml = valid_aug(image=iml_np)['image'].unsqueeze(0).to(device)\nimr = valid_aug(image=imr_np)['image'].unsqueeze(0).to(device)\n\nwith torch.no_grad():\n    dry_clover, dry_dead, dry_green = model(iml, imr)\n\ndry_clover = dry_clover.cpu().item()\ndry_dead = dry_dead.cpu().item()\ndry_green = dry_green.cpu().item()\n\ndry_clover, dry_dead, dry_green = scaler.inverse_transform([[dry_clover, dry_dead, dry_green]])[0]   # inverse StandardScaler\n\nif dry_green < 0:\n    dry_green = 0\n\nif dry_dead < 0:\n    dry_dead = 0\n\nif dry_clover < 0:\n    dry_clover = 0\n    \ngdm = dry_clover + dry_green\ndry_total = gdm + dry_dead\n\nresult = pd.DataFrame([{\n    'image_path': row.image_path,\n    'Dry_Clover_g': dry_clover,\n    'Dry_Dead_g': dry_dead,\n    'dry_green': dry_green,\n    'Dry_Total_g': dry_total,\n    'GDM_g': gdm\n}])\ndisplay(pd.DataFrame([row]))\nprint(f\"{'-'*8}\")\ndisplay(result)\n# print(f\"{'-'*8}\")\nImage.open(image_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T18:46:25.763619Z","iopub.execute_input":"2025-12-06T18:46:25.764394Z","iopub.status.idle":"2025-12-06T18:46:26.529768Z","shell.execute_reply.started":"2025-12-06T18:46:25.764365Z","shell.execute_reply":"2025-12-06T18:46:26.528327Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"## Hyperparameter Tuning","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\n\nimport cv2 \nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nfrom torch import optim\nimport timm  \nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    BASE_PATH = '/kaggle/input/csiro-biomass'\n    TRAIN_CSV = os.path.join(BASE_PATH, 'train.csv')\n    IMAGE_DIR = os.path.join(BASE_PATH, 'train')\n    \n    MODEL_NAME = 'convnext_small.fb_in22k_ft_in1k'\n    fold_num = 4\n    \n    # ---  ---\n    BATCH_SIZE = 2\n    NUM_WORKERS = 2  \n    \n    # --- Loss ---\n    TARGET_COLS = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g']\n\n    ALL_TARGET_COLS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    R2_WEIGHTS = [0.1, 0.1, 0.1, 0.2, 0.5]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\n# from optuna.integration import TFKerasPruningCallback\nfrom optuna.storages import RDBStorage\nimport sqlite3\nimport json\nimport os","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, loader, criterion, optimizer, device):\n    model.train() \n    epoch_loss = 0.0\n    step = 0    \n\n    for (img_left, img_right, train_targets, all_targets) in loader:\n        step += 1\n        \n        # 1. device (GPU/CPU)\n        img_left = img_left.to(device)\n        img_right = img_right.to(device)\n        targets = train_targets.to(device) # Chỉ 3 mục tiêu cho loss\n        \n        optimizer.zero_grad()\n        \n        # 3. Forward pass \n        dry_clover, dry_dead, dry_green = model(img_left, img_right)\n        \n        # 4. Loss \n        dry_clover_loss = criterion(dry_clover, targets[:,0].unsqueeze(-1))\n        dry_dead_loss = criterion(dry_dead, targets[:,1].unsqueeze(-1))\n        dry_green_loss = criterion(dry_green, targets[:,2].unsqueeze(-1))\n\n        loss = dry_clover_loss + dry_dead_loss + dry_green_loss\n        loss = loss / 3\n        loss.backward()\n        optimizer.step()\n        \n        epoch_loss += loss.item()\n    return epoch_loss / len(loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate_one_epoch(model, loader, scaler, device):\n    model.eval()\n    preds = {\n        \"dry_clover\": [],\n        \"dry_dead\": [],\n        \"dry_green\": []\n    }\n    all_targets = []\n    for (img_left, img_right, _, true_5_targets) in loader:\n        img_left = img_left.to(device)\n        img_right = img_right.to(device)\n        all_targets.append(true_5_targets.numpy())\n        with torch.no_grad():        \n            dry_clover, dry_dead, dry_green = model(img_left, img_right)\n\n        if scaler is not None:\n            scaled_pred = torch.cat([dry_clover, dry_dead, dry_green], dim=1)   # shape: [batch, 3]\n            scaled_pred_np = scaled_pred.cpu().numpy()\n            \n            # Step 3: Inverse scaling → back to log1p space → back to real grams\n            unscaled_real = scaler.inverse_transform(scaled_pred_np)   # inverse StandardScaler\n\n            # Clip negative values\n            unscaled_real = np.clip(unscaled_real, a_min=0, a_max=None)\n\n            preds['dry_clover'].extend(unscaled_real[:, 0].tolist())\n            preds['dry_dead'].extend(unscaled_real[:, 1].tolist())\n            preds['dry_green'].extend(unscaled_real[:, 2].tolist())\n\n        else:\n            preds['dry_clover'].extend(dry_clover.squeeze().cpu().tolist())\n            preds['dry_dead'].extend(dry_dead.squeeze().cpu().tolist())\n            preds['dry_green'].extend(dry_green.squeeze().cpu().tolist())\n\n    all_targets = np.concatenate(all_targets, axis=0)\n    preds['dry_clover'] = np.array(preds['dry_clover'])\n    preds['dry_dead'] = np.array(preds['dry_dead'])\n    preds['dry_green'] = np.array(preds['dry_green'])\n    \n    r2_score = biomass_r2_score(preds, all_targets)\n    del preds, all_targets\n    return r2_score","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassRegressionHead(nn.Module):\n    def __init__(self, target_name, n_features = 768, dropout = 0.3, neurons = 384):\n        super(BiomassRegressionHead, self).__init__()\n        self.target_name = target_name\n        self.head = nn.Sequential(\n            nn.Linear(n_features, neurons),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(neurons, 1)\n        )\n\n    def forward(self, features):\n        return self.head(features)\n\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, pretrained = True, n_targets=3, neurons = 384, dropout=0.3):\n        super(BiomassModel, self).__init__()\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg'\n        )\n        self.n_features = self.backbone.num_features\n        self.n_combined_features = self.n_features * 2\n        \n        self.head_dry_clover = BiomassRegressionHead(target_name = 'Dry_Clover_g', n_features = self.n_combined_features,neurons = neurons, dropout = dropout)\n        self.head_dry_dead = BiomassRegressionHead(target_name = 'Dry_Dead_g', n_features = self.n_combined_features,neurons = neurons, dropout = dropout)\n        self.head_dry_green = BiomassRegressionHead(target_name = 'Dry_Green_g', n_features = self.n_combined_features,neurons = neurons, dropout = dropout)\n        \n\n    def forward(self, img_left, img_right):\n        features_left = self.backbone(img_left) # Shape: [batch, n_features]\n        features_right = self.backbone(img_right) # Shape: [batch, n_features]\n    \n        combined = torch.cat([features_left, features_right], dim=1) # Shape: [batch, 2 * n_features]\n        \n        dry_clover = self.head_dry_clover(combined)\n        dry_dead = self.head_dry_dead(combined)\n        dry_green = self.head_dry_green(combined)\n        \n        return dry_clover, dry_dead, dry_green","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_dataloaders(train_aug, valid_aug):\n    train_dataset = BiomassDataset(\n        df = df_train,\n        transform = train_aug,\n        image_dir = Config.IMAGE_DIR,\n        train_target_cols = Config.TARGET_COLS,\n        all_target_cols = Config.ALL_TARGET_COLS,\n        fit_scaler = True\n    )\n\n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=Config.BATCH_SIZE,\n        shuffle=True,\n        num_workers=Config.NUM_WORKERS\n    )\n    \n    val_dataset = BiomassDataset(\n        df = df_val,\n        transform = valid_aug,\n        image_dir = Config.IMAGE_DIR,\n        train_target_cols = Config.TARGET_COLS,\n        all_target_cols = Config.ALL_TARGET_COLS,\n        fit_scaler = False\n    )\n    \n    val_loader = DataLoader(\n        val_dataset,\n        batch_size = Config.BATCH_SIZE,\n        shuffle = False,\n        num_workers = Config.NUM_WORKERS\n    )\n\n    return train_loader, val_loader","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial):\n    global best_trial_score\n    # ==============================\n    # HYPERPARAMETERS TO TUNE\n    # ==============================\n    lr          = trial.suggest_float(\"lr\", 5e-5, 5e-3, log=True)\n    # wd          = trial.suggest_float(\"weight_decay\", 1e-6, 1e-1, log=True)\n    dropout     = trial.suggest_float(\"dropout\", 0.1, 0.5)\n    img_size    = trial.suggest_categorical(\"img_size\", [512, 768, 896])\n    neurons     = trial.suggest_int(\"neurons\", low = 256, high=512, step = 16)\n    lr_scheduling_factor = trial.suggest_float(\"lr_scheduling_factor\", 0.1, 0.9)\n    lr_scheduling_patience  = trial.suggest_int(\"lr_scheduling_patience\", low=1, high=5, step=1)\n    \n    # Augmentations strength\n    bright = trial.suggest_float(\"bright\", 0.1, 0.4)\n    contrast = trial.suggest_float(\"contrast\", 0.1, 0.4)\n    hue = trial.suggest_float(\"hue\", 0.05, 0.2)\n\n    # ==============================\n    # MODEL WITH SUGGESTED PARAMS\n    # ==============================\n    model = BiomassModel(model_name = Config.MODEL_NAME,\n                         neurons = neurons, \n                         dropout=dropout\n                        ).to(device)\n\n    optimizer = optim.Adam(\n        model.parameters(),\n        lr=lr\n    )\n    scheduler = ReduceLROnPlateau(optimizer,\n        mode='max',              \n        factor=lr_scheduling_factor,              # reduce LR by a factor of\n        patience=lr_scheduling_patience,              # wait 3 epochs with no improvement\n        min_lr=1e-6              # don't go below this LR\n    )\n    \n    # Dynamic aug based on trial\n    train_aug = A.Compose([\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.ColorJitter(brightness=bright, \n                      contrast=contrast, \n                      hue=hue, \n                      p=0.6),\n        # A.CoarseDropout(max_holes=8, \n        #                 max_height=32, \n        #                 max_width=32, \n        #                 p=0.5),\n        # A.GaussNoise(\n        #     var_limit=(10.0, 50.0),\n        #     mean=0,\n        #     per_channel=True,\n        #     p=0.3\n        # )\n        A.Resize(img_size, img_size),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n        ),\n        ToTensorV2()\n    ])\n\n    valid_aug = A.Compose([\n        A.Resize(img_size, img_size),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n        ),\n        ToTensorV2()\n        ])\n\n\n    train_loader, val_loader = create_dataloaders(train_aug, valid_aug)\n    scaler = train_loader.dataset.scaler  # from fit_scaler=True\n    criterion = nn.MSELoss()\n    best_score = -float('inf')\n\n    for epoch in tqdm(range(20), leave=False, desc=f\"Trial {trial.number}\"):\n        epoch_loss = train_one_epoch(model, train_loader, criterion, optimizer, device)\n        val_score = validate_one_epoch(model, val_loader, scaler, device)\n        scheduler.step(val_score)\n\n        \n        if val_score > best_score:\n            best_score = val_score\n            if best_score > 0.8:\n                print(f\"✨ Trial Best: {val_score:.5f}\")\n            \n        if val_score > best_trial_score:\n            best_trial_score = val_score\n            print(f\"🌟 New Global Best: {val_score:.5f}\")\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'scaler': scaler,\n                \"model_name\": Config.MODEL_NAME, \n                'val_score': best_trial_score,\n            }, f'best_model_optuna.pth')\n        \n    \n        # Debug\n        torch.cuda.empty_cache()\n\n    return best_score","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_trial_score = 0.87\nif os.path.exists('best_model_optuna.pth'):\n    checkpoint = torch.load(BEST_MODEL_PATH)\n    print(f\"Loaded global best model: {checkpoint['val_score']:.6f}\")\n    best_trial_score = max(checkpoint['val_score'], best_trial_score) # save only if > 0.85 whic baseline without tuning hyperparameters\n    \ndb_path = \"/kaggle/working/csiro_optuna.db\"\nstorage = RDBStorage(url=f\"sqlite:///{db_path}\")\n\nstudy = optuna.create_study(\n    direction=\"maximize\",\n    storage=storage,\n    study_name=\"csiro_fold4_final_push\",\n    load_if_exists=True,\n    # pruner=optuna.pruners.MedianPruner(n_startup_trials=20, n_warmup_steps=7)\n)\nprint(f\"Study has {len(study.trials)} completed trials. Resuming from trial {len(study.trials)}...\")\nstudy.optimize(objective, n_trials=200)  \n\nprint(\"BEST TRIAL:\")\nprint(study.best_trial.params)\nprint(f\"BEST VALID R²: {study.best_value:.6f}\")","metadata":{"trusted":true,"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}