{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":696121,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":527976,"modelId":542033}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom PIL import Image\n\nfrom torch.utils.data import Dataset\nfrom PIL import Image\nimport torch\n\nimport timm\nfrom transformers import get_cosine_schedule_with_warmup\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.model_selection import GroupKFold,StratifiedGroupKFold\n\nSEED = 42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:23:59.574449Z","iopub.execute_input":"2025-12-23T10:23:59.574756Z","iopub.status.idle":"2025-12-23T10:24:12.647782Z","shell.execute_reply.started":"2025-12-23T10:23:59.574727Z","shell.execute_reply":"2025-12-23T10:24:12.647146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntest = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\nimg_train_dir = r'/kaggle/input/csiro-biomass'\nimg_test_dir = r'/kaggle/input/csiro-biomass'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:12.648922Z","iopub.execute_input":"2025-12-23T10:24:12.649327Z","iopub.status.idle":"2025-12-23T10:24:12.670107Z","shell.execute_reply.started":"2025-12-23T10:24:12.649306Z","shell.execute_reply":"2025-12-23T10:24:12.669546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:12.670890Z","iopub.execute_input":"2025-12-23T10:24:12.671161Z","iopub.status.idle":"2025-12-23T10:24:12.693444Z","shell.execute_reply.started":"2025-12-23T10:24:12.671136Z","shell.execute_reply":"2025-12-23T10:24:12.692924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:12.694512Z","iopub.execute_input":"2025-12-23T10:24:12.694728Z","iopub.status.idle":"2025-12-23T10:24:12.701182Z","shell.execute_reply.started":"2025-12-23T10:24:12.694703Z","shell.execute_reply":"2025-12-23T10:24:12.700622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['group_id'] = train['Sampling_Date'].astype(str) + \"-\" + train['State'].astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:12.701805Z","iopub.execute_input":"2025-12-23T10:24:12.702111Z","iopub.status.idle":"2025-12-23T10:24:12.718763Z","shell.execute_reply.started":"2025-12-23T10:24:12.702093Z","shell.execute_reply":"2025-12-23T10:24:12.718258Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Exploration for Modelling techniques\n","metadata":{}},{"cell_type":"markdown","source":"## Data Cleaning and Format setting\n- converting this data from long format to wide format, with image getting all it's values in row fashion","metadata":{}},{"cell_type":"code","source":"class config:\n    weights = {\n            \"Dry_Green_g\": 0.1,\n            \"Dry_Dead_g\": 0.1,\n            \"Dry_Clover_g\": 0.1,\n            \"GDM_g\": 0.2,\n            \"Dry_Total_g\": 0.5\n    }\n    model_name1 = 'vit_base_patch14_reg4_dinov2.lvd142m'\n    base_lr = 1e-5\n    reg_lr = 3e-4\n    epochs= 25\n    warmup_ratio = 0.1\n\ndef long_to_wide(data,\n                 index_cols=['image_path','group_id'],\n                drop_col = 'index'):\n    df = data.pivot(index=index_cols, columns='target_name', values='target')\n    df = df.reset_index()\n    df.columns.name = None\n    cols = list(config.weights.keys())\n    df[cols] = df[cols].fillna(0.0)\n\n    ##applying transforms lop1p()\n\n    for c in cols:\n        df[f'log_{c}'] = np.log1p(df[c])\n    return df\n\ndf = long_to_wide(train)\ndf['image_path'] = df['image_path'].apply(lambda path: os.path.join(img_train_dir,path))\ndisplay(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:13.921637Z","iopub.execute_input":"2025-12-23T10:24:13.922350Z","iopub.status.idle":"2025-12-23T10:24:13.965298Z","shell.execute_reply.started":"2025-12-23T10:24:13.922321Z","shell.execute_reply":"2025-12-23T10:24:13.964581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = [f'log_{c}' for c in config.weights.keys()]\nfig,ax = plt.subplots(3,2)\nax = ax.flatten()\nfor idx,c in enumerate(cols):\n    sns.histplot(\n        data=df,\n        x= c\n        ,ax=ax[idx]\n    )\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:15.831862Z","iopub.execute_input":"2025-12-23T10:24:15.832416Z","iopub.status.idle":"2025-12-23T10:24:16.610988Z","shell.execute_reply.started":"2025-12-23T10:24:15.832390Z","shell.execute_reply":"2025-12-23T10:24:16.610416Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model and Data loader preparation","metadata":{}},{"cell_type":"code","source":"##using timm\nclass WeightDataset(Dataset):\n    def __init__(self, df, is_train, cols_dict=config.weights, transforms=None):\n        self.df = df.reset_index(drop=True)\n        self.targets = [f'log_{c}' for c in cols_dict.keys()]\n        self.is_train = is_train\n        self.transforms = transforms  # timm transforms\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n\n        img = Image.open(row[\"image_path\"]).convert(\"RGB\")\n\n        if self.transforms is not None:\n            img = self.transforms(img)  # -> Tensor [3, H, W]\n\n        if self.is_train:\n            targets = torch.from_numpy(\n                        row[self.targets]\n                        .astype(\"float32\")\n                        .values\n                    )\n            return img, targets\n\n        return img\n\n\nsplitter = GroupKFold(n_splits=5)\n# splitter= StratifiedGroupKFold(n_splits=5)\n\ndef get_loaders(data,splitter,train_transforms,val_transforms,batches=[16,32]):\n    loaders = []\n    for idx,(train_idx,val_idx) in enumerate(splitter.split(X=data.drop(columns=['group_id']),groups=data.group_id)):\n        print(f'Making loader: {idx+1}')\n        train = data.iloc[train_idx]\n        val = data.iloc[val_idx]\n\n        train_d = WeightDataset(train,is_train=True,transforms=train_transforms) ##changed for timm\n        val_d = WeightDataset(val,is_train=True,transforms=val_transforms)\n\n        loaders.append(\n            (\n                DataLoader(train_d,batch_size=batches[0],num_workers=2,pin_memory=True,shuffle=True),\n                DataLoader(val_d,batch_size=batches[1],num_workers=2,shuffle=False)\n            )\n        )\n    return loaders","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:19.468857Z","iopub.execute_input":"2025-12-23T10:24:19.469474Z","iopub.status.idle":"2025-12-23T10:24:19.477633Z","shell.execute_reply.started":"2025-12-23T10:24:19.469443Z","shell.execute_reply":"2025-12-23T10:24:19.477064Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Defining Models","metadata":{}},{"cell_type":"code","source":"class BioEstimatorD(nn.Module):\n    def __init__(self, name):\n        super().__init__()\n        self.backbone = timm.create_model(\n            name,\n            pretrained=True,\n            num_classes=0,\n            global_pool=\"\"\n        )\n\n        self.embed_dim = self.backbone.num_features\n\n        self.head = nn.Sequential(\n            nn.Linear(self.embed_dim, 512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.1),\n            nn.Linear(512, 256),\n            nn.ReLU(inplace=True),\n            nn.Linear(256, 5)\n        )\n\n    def forward(self, x):\n        feats = self.backbone.forward_features(x)  # [B, N, C]\n        cls = feats[:, 0]                           # CLS token\n        return self.head(cls)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:20.884136Z","iopub.execute_input":"2025-12-23T10:24:20.884418Z","iopub.status.idle":"2025-12-23T10:24:20.889831Z","shell.execute_reply.started":"2025-12-23T10:24:20.884397Z","shell.execute_reply":"2025-12-23T10:24:20.889062Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Defining Transforms","metadata":{}},{"cell_type":"code","source":"import timm\nfrom torchvision import transforms\n\nmodel = timm.create_model(config.model_name1, pretrained=True)\ndata_config = timm.data.resolve_model_data_config(model)\nbase_val_tfms = timm.data.create_transform(\n    **data_config,\n    is_training=False\n)\n\n\ntrain_transforms = transforms.Compose([\n    # --- your geometry aug ---\n    transforms.RandomHorizontalFlip(p=0.55),\n    transforms.RandomVerticalFlip(p=0.55),\n    transforms.RandomRotation( degrees=30,\n    interpolation= transforms.InterpolationMode.BILINEAR,\n    expand=False),\n\n    # --- timm-approved preprocessing ---\n    *base_val_tfms.transforms\n])\n\nval_transforms = base_val_tfms\ndel model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:21.944169Z","iopub.execute_input":"2025-12-23T10:24:21.944917Z","iopub.status.idle":"2025-12-23T10:24:25.595129Z","shell.execute_reply.started":"2025-12-23T10:24:21.944887Z","shell.execute_reply":"2025-12-23T10:24:25.594345Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Defining Loss Function","metadata":{}},{"cell_type":"code","source":"def get_weights(weights=config.weights):\n    w_list= []\n    for things in weights.items():\n        w_list.append(things[1])\n    return torch.tensor(w_list)\n\nw_tensor = get_weights()\n\nclass weightedMSE(torch.nn.Module):\n    def __init__(self,weights):\n        super().__init__()\n        self.weights = weights\n        \n    def forward(self,inputs,labels):\n        loss = (inputs-labels)**2\n        loss = loss*self.weights\n        return loss.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:25.596359Z","iopub.execute_input":"2025-12-23T10:24:25.596665Z","iopub.status.idle":"2025-12-23T10:24:25.601738Z","shell.execute_reply.started":"2025-12-23T10:24:25.596647Z","shell.execute_reply":"2025-12-23T10:24:25.600989Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Setting Up HyperParams","metadata":{}},{"cell_type":"code","source":"NUM_EPOCHS = config.epochs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:25.602579Z","iopub.execute_input":"2025-12-23T10:24:25.602946Z","iopub.status.idle":"2025-12-23T10:24:25.645366Z","shell.execute_reply.started":"2025-12-23T10:24:25.602917Z","shell.execute_reply":"2025-12-23T10:24:25.644603Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# processor = AutoImageProcessor.from_pretrained(config.model_id)\nall_loaders = get_loaders(df,splitter,train_transforms,val_transforms,batches=[16,32])\n# all_loaders2 = get_loaders(df,splitter,train_transforms2,val_transforms2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:25.646718Z","iopub.execute_input":"2025-12-23T10:24:25.646964Z","iopub.status.idle":"2025-12-23T10:24:25.670289Z","shell.execute_reply.started":"2025-12-23T10:24:25.646948Z","shell.execute_reply":"2025-12-23T10:24:25.669737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_batch(model,batch,loss_fn,optimizer,scheduler,scaler,DEVICE):\n    images = batch[0].to(DEVICE)\n    targets = batch[1].to(DEVICE)\n    optimizer.zero_grad()\n\n    with autocast(device_type = DEVICE):\n        outputs = model(images)\n        loss = loss_fn(outputs,targets)\n\n    scaler.scale(loss).backward()\n    scaler.step(optimizer)\n    scaler.update()\n    scheduler.step()\n    return loss.item()\n            \ndef val_one_batch(model,batch,loss_fn,scaler,DEVICE):\n    with torch.no_grad():\n        images = batch[0].to(DEVICE)\n        targets = batch[1].to(DEVICE)\n        with autocast(device_type= DEVICE):\n            outputs = model(images)\n            loss = loss_fn(outputs,targets)\n    return loss.item(),outputs.float().cpu(),targets.float().cpu()\n\nfrom sklearn.metrics import r2_score\n\ndef get_competition_metric(preds, targets):\n    weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n    w_flat = np.tile(weights, (preds.shape[0], 1)).flatten()\n    y_true = targets.flatten()\n    y_pred = preds.flatten()\n    y_bar_w = np.sum(w_flat * y_true) / np.sum(w_flat)\n    ss_res = np.sum(w_flat * (y_true - y_pred)**2)\n    ss_tot = np.sum(w_flat * (y_true - y_bar_w)**2)\n    return 1 - (ss_res / ss_tot)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:27.922756Z","iopub.execute_input":"2025-12-23T10:24:27.923464Z","iopub.status.idle":"2025-12-23T10:24:27.930702Z","shell.execute_reply.started":"2025-12-23T10:24:27.923439Z","shell.execute_reply":"2025-12-23T10:24:27.929946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def freeze_backbone(model):\n#     for name, param in model.named_parameters():\n#         if \"backbone\" in name:\n#             param.requires_grad = False\n\n# def unfreeze_backbone(model):\n#     for name, param in model.named_parameters():\n#         if \"backbone\" in name:\n#             param.requires_grad = True\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:28.143980Z","iopub.execute_input":"2025-12-23T10:24:28.144674Z","iopub.status.idle":"2025-12-23T10:24:28.147935Z","shell.execute_reply.started":"2025-12-23T10:24:28.144650Z","shell.execute_reply":"2025-12-23T10:24:28.147117Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def set_seed(seed):\n    import random\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-23T10:24:47.750455Z","iopub.execute_input":"2025-12-23T10:24:47.750762Z","iopub.status.idle":"2025-12-23T10:24:47.755147Z","shell.execute_reply.started":"2025-12-23T10:24:47.750737Z","shell.execute_reply":"2025-12-23T10:24:47.754374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.amp import autocast,GradScaler \nfrom tqdm.notebook import trange,tqdm \nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' \nscaler = GradScaler(DEVICE) \nexperiment = 1 \n\nfor seed in [108]:\n    set_seed(seed=seed)\n    print(f'Training SEED: {seed}')\n    model_file = f'/kaggle/working/seed_{seed}_ver_{experiment}_best' \n    os.makedirs(model_file,exist_ok=True)\n    for idx, loaders in enumerate(all_loaders):\n        if (seed == 42 and idx == 4 )or (seed == 108):\n            print(f'Training fold {idx+1}')\n        \n            model = BioEstimatorD(\n                name=config.model_name1,\n            ).to(DEVICE)\n    \n            for p in model.parameters():\n                p.requires_grad = True\n            \n            backbone_params = model.backbone.parameters()\n            head_params = model.head.parameters()\n            \n            optimizer = torch.optim.AdamW(\n                [\n                    {\n                        \"params\": backbone_params, \n                        \"lr\": 1e-5  # Low LR for the pre-trained backbone\n                    },\n                    {\n                        \"params\": head_params, \n                        \"lr\": 3e-4  # High LR for the fresh regression head\n                    },\n                ],\n                weight_decay=0.01\n            )\n                \n            scheduler = get_cosine_schedule_with_warmup(\n                optimizer,\n                num_warmup_steps=int(0.1 * NUM_EPOCHS * len(loaders[0])),\n                num_training_steps=NUM_EPOCHS * len(loaders[0])\n            )\n        \n            criterion = weightedMSE(w_tensor.to(DEVICE))\n            best_loss = float(\"inf\")\n            best_r2 = -float(\"inf\")\n            trigger = 0\n            patience = 5\n        \n            for epoch in trange(NUM_EPOCHS, desc=f'Fold {idx+1}'):\n                train_losses = []\n        \n                for batch in tqdm(loaders[0], desc=\"TRAIN\", leave=False):\n                    loss = train_one_batch(\n                        model, batch, criterion,\n                        optimizer, scheduler,\n                        scaler, DEVICE\n                    )\n                    train_losses.append(loss)\n        \n                avg_train_loss = np.mean(train_losses)\n        \n    \n                val_losses = []\n                all_outputs, all_targets = [], []\n        \n                for batch in tqdm(loaders[1], desc=\"VAL\", leave=False):\n                    loss, outputs, targets = val_one_batch(\n                        model, batch, criterion,\n                        scaler, DEVICE\n                    )\n                    val_losses.append(loss)\n                    all_outputs.append(outputs)\n                    all_targets.append(targets)\n        \n                avg_val_loss = np.mean(val_losses)\n                preds = np.vstack(all_outputs)\n                targets = np.vstack(all_targets)\n        \n                epoch_r2 = get_competition_metric(preds, targets)\n        \n                print(\n                    f\"Epoch {epoch+1} | \"\n                    f\"Train {avg_train_loss:.4f} | \"\n                    f\"Val {avg_val_loss:.4f} | \"\n                    f\"R2 {epoch_r2:.4f}\"\n                )\n        \n                # ================= CHECKPOINT =================\n                if avg_val_loss < best_loss:\n                    best_loss = avg_val_loss\n                    best_r2 = epoch_r2\n                    trigger = 0\n                    torch.save(\n                        model.state_dict(),\n                        f\"{model_file}/DINO_fold{idx+1}.pth\"\n                    )\n                    print(f\"--> Saved (R2={best_r2:.4f})\")\n                else:\n                    trigger += 1\n                    if trigger >= patience:\n                        print(\"EARLY STOPPING\")\n                        break\n        \n            del model, optimizer, scheduler\n            torch.cuda.empty_cache()","metadata":{"trusted":true,"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-12-23T10:25:08.612779Z","iopub.execute_input":"2025-12-23T10:25:08.613095Z","iopub.status.idle":"2025-12-23T11:57:07.943968Z","shell.execute_reply.started":"2025-12-23T10:25:08.613067Z","shell.execute_reply":"2025-12-23T11:57:07.943208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}