{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom PIL import Image\n\nfrom transformers import AutoModel,AutoImageProcessor,get_cosine_schedule_with_warmup\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom sklearn.model_selection import GroupKFold\n\nSEED = 42","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:37:22.763309Z","iopub.execute_input":"2025-12-13T09:37:22.763822Z","iopub.status.idle":"2025-12-13T09:37:33.443541Z","shell.execute_reply.started":"2025-12-13T09:37:22.763794Z","shell.execute_reply":"2025-12-13T09:37:33.442949Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Introduction \n---\n- I am to provide a comprehensive EDA and some modelling techniques I have thought about this problem\n- using pytorch_lightning makes it easy for quick experimentation","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntest = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\nimg_train_dir = r'/kaggle/input/csiro-biomass'\nimg_test_dir = r'/kaggle/input/csiro-biomass'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:37:33.444807Z","iopub.execute_input":"2025-12-13T09:37:33.445451Z","iopub.status.idle":"2025-12-13T09:37:33.459968Z","shell.execute_reply.started":"2025-12-13T09:37:33.445430Z","shell.execute_reply":"2025-12-13T09:37:33.459173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:37:33.460813Z","iopub.execute_input":"2025-12-13T09:37:33.461624Z","iopub.status.idle":"2025-12-13T09:37:33.474236Z","shell.execute_reply.started":"2025-12-13T09:37:33.461600Z","shell.execute_reply":"2025-12-13T09:37:33.473706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T11:29:06.153889Z","iopub.execute_input":"2025-12-13T11:29:06.154734Z","iopub.status.idle":"2025-12-13T11:29:06.163987Z","shell.execute_reply.started":"2025-12-13T11:29:06.154704Z","shell.execute_reply":"2025-12-13T11:29:06.163421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['group_id'] = train['Sampling_Date'].astype(str) + '_' + train['State'].astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:37:33.484994Z","iopub.execute_input":"2025-12-13T09:37:33.485290Z","iopub.status.idle":"2025-12-13T09:37:33.498194Z","shell.execute_reply.started":"2025-12-13T09:37:33.485273Z","shell.execute_reply":"2025-12-13T09:37:33.497655Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Exploration for Modelling techniques\n","metadata":{}},{"cell_type":"markdown","source":"## Data Cleaning and Format setting\n- converting this data from long format to wide format, with image getting all it's values in row fashion","metadata":{}},{"cell_type":"code","source":"class config:\n    weights = {\n            \"Dry_Green_g\": 0.1,\n            \"Dry_Dead_g\": 0.1,\n            \"Dry_Clover_g\": 0.1,\n            \"GDM_g\": 0.2,\n            \"Dry_Total_g\": 0.5\n    }\n    model_id = r'facebook/dinov2-base'\n    base_lr = 1e-5\n    reg_lr = 3e-4\n    epochs=20\n\ndef long_to_wide(data,\n                 index_cols=['image_path','group_id'],\n                drop_col = 'index'):\n    df = data.pivot(index=index_cols, columns='target_name', values='target')\n    df = df.reset_index()\n    df.columns.name = None\n    cols = list(config.weights.keys())\n    df[cols] = df[cols].fillna(0.0)\n\n    ##applying transforms lop1p()\n\n    for c in cols:\n        df[f'log_{c}'] = np.log1p(df[c])\n    return df\n\ndf = long_to_wide(train)\ndf['image_path'] = df['image_path'].apply(lambda path: os.path.join(img_train_dir,path))\ndisplay(df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:50:00.946192Z","iopub.execute_input":"2025-12-13T10:50:00.946998Z","iopub.status.idle":"2025-12-13T10:50:00.971303Z","shell.execute_reply.started":"2025-12-13T10:50:00.946971Z","shell.execute_reply":"2025-12-13T10:50:00.970728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = [f'log_{c}' for c in config.weights.keys()]\nfig,ax = plt.subplots(3,2)\nax = ax.flatten()\nfor idx,c in enumerate(cols):\n    sns.histplot(\n        data=df,\n        x= c\n        ,ax=ax[idx]\n    )\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:50:03.103857Z","iopub.execute_input":"2025-12-13T10:50:03.104387Z","iopub.status.idle":"2025-12-13T10:50:03.889504Z","shell.execute_reply.started":"2025-12-13T10:50:03.104364Z","shell.execute_reply":"2025-12-13T10:50:03.888841Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model and Data loader preparation","metadata":{}},{"cell_type":"code","source":"class weight(Dataset):\n    def __init__(self,df,processor,is_train,cols_dict=config.weights,transforms=None):\n        self.df = df\n        self.targets = [f'log_{c}' for c in list(cols_dict.keys())]\n        self.processor = processor\n        self.is_train = is_train\n        self.transforms = transforms\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self,idx):\n        row = self.df.iloc[idx]\n        img_path = row['image_path']\n        img = Image.open(img_path).convert(\"RGB\")\n\n        if self.transforms:\n            img = self.transforms(img)\n\n        outputs = self.processor(images=img,return_tensors='pt')\n        img = outputs['pixel_values'].squeeze(0)\n        if self.is_train:\n            targets = torch.tensor(row[self.targets].values.astype('float32'))\n            return img,targets\n\n        return img \n\nsplitter = GroupKFold(n_splits=5)\n\ndef get_loaders(data,processor,splitter,train_transforms,val_transforms,batches=[16,32]):\n    loaders = []\n    for idx,(train_idx,val_idx) in enumerate(splitter.split(data.drop(columns=['group_id']),groups=data.group_id)):\n        print(f'Making loader: {idx+1}')\n        train = data.iloc[train_idx]\n        val = data.iloc[val_idx]\n\n        train_d = weight(train,processor,is_train=True,transforms=train_transforms)\n        val_d = weight(val,processor,is_train=True,transforms=val_transforms)\n\n        loaders.append(\n            (\n                DataLoader(train_d,batch_size=batches[0],num_workers=2,pin_memory=True,shuffle=True),\n                DataLoader(val_d,batch_size=batches[1],num_workers=2,shuffle=False)\n            )\n        )\n    return loaders","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:50:08.304435Z","iopub.execute_input":"2025-12-13T10:50:08.304730Z","iopub.status.idle":"2025-12-13T10:50:08.313490Z","shell.execute_reply.started":"2025-12-13T10:50:08.304709Z","shell.execute_reply":"2025-12-13T10:50:08.312724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class bioEstimator(torch.nn.Module):\n    def __init__(self,model_id):\n        super().__init__()\n        self.base_model = AutoModel.from_pretrained(model_id)\n        self.regressor = torch.nn.Sequential(\n            nn.Linear(768,512),\n            nn.ReLU(),\n            nn.Linear(512,256),\n            nn.ReLU(),\n            nn.Linear(256,5)\n        )\n    def forward(self,x):\n        outputs = self.base_model(x)\n\n        features = outputs.last_hidden_state[:, 0, :] #grab the cls token\n\n        return self.regressor(features)\n        \nclass bioEstimatorD(torch.nn.Module):\n    def __init__(self,model_id):\n        super().__init__()\n        self.base_model = AutoModel.from_pretrained(model_id)\n        self.regressor = torch.nn.Sequential(\n            nn.Linear(768,512),\n            nn.ReLU(),\n            nn.Linear(512,256),\n            nn.Dropout(0.1),\n            nn.ReLU(),\n            nn.Linear(256,5)\n        )\n    def forward(self,x):\n        outputs = self.base_model(x)\n\n        features = outputs.last_hidden_state[:, 0, :] #grab the cls token\n\n        return self.regressor(features)\n\nclass bioEstimatorAdv(torch.nn.Module):\n    def __init__(self,model_id):\n        super().__init__()\n        self.base_model = AutoModel.from_pretrained(model_id)\n        self.regressor = nn.Sequential(\n                        nn.Linear(768, 512),\n                        nn.LayerNorm(512),      # Stabilizes gradients\n                        nn.GELU(),              # Smoother than ReLU\n                        nn.Dropout(0.3),        # The dropout you found useful\n                        nn.Linear(512, 256),    # Added a second hidden layer for complexity\n                        nn.LayerNorm(256),\n                        nn.GELU(),\n                        nn.Linear(256, 5)\n                )\n    def forward(self,x):\n        outputs = self.base_model(x)\n\n        features = outputs.last_hidden_state[:, 0, :] #grab the cls token\n\n        return self.regressor(features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:45:26.442131Z","iopub.execute_input":"2025-12-13T10:45:26.442460Z","iopub.status.idle":"2025-12-13T10:45:26.452052Z","shell.execute_reply.started":"2025-12-13T10:45:26.442432Z","shell.execute_reply":"2025-12-13T10:45:26.451473Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Defining Transforms","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n    #can't use color jitters or random crops \n            transforms.RandomHorizontalFlip(p=0.5),\n            transforms.RandomVerticalFlip(p=0.5),\n            transforms.RandomRotation(degrees=25)\n])\n\nval_transforms = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:25:32.600855Z","iopub.execute_input":"2025-12-13T10:25:32.601123Z","iopub.status.idle":"2025-12-13T10:25:32.605416Z","shell.execute_reply.started":"2025-12-13T10:25:32.601105Z","shell.execute_reply":"2025-12-13T10:25:32.604820Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Defining Loss Function","metadata":{}},{"cell_type":"code","source":"def get_weights(weights=config.weights):\n    w_list= []\n    for things in weights.items():\n        w_list.append(things[1])\n    return torch.tensor(w_list)\n\nw_tensor = get_weights()\n\nclass weightedMSE(torch.nn.Module):\n    def __init__(self,weights):\n        super().__init__()\n        self.weights = weights\n        \n    def forward(self,inputs,labels):\n        loss = (inputs-labels)**2\n        loss = loss*self.weights\n        return loss.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:59:05.859392Z","iopub.execute_input":"2025-12-13T10:59:05.860207Z","iopub.status.idle":"2025-12-13T10:59:05.865932Z","shell.execute_reply.started":"2025-12-13T10:59:05.860177Z","shell.execute_reply":"2025-12-13T10:59:05.865218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Setting Up HyperParams","metadata":{}},{"cell_type":"code","source":"NUM_EPOCHS = config.epochs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:59:11.795134Z","iopub.execute_input":"2025-12-13T10:59:11.795846Z","iopub.status.idle":"2025-12-13T10:59:11.799111Z","shell.execute_reply.started":"2025-12-13T10:59:11.795823Z","shell.execute_reply":"2025-12-13T10:59:11.798470Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"processor = AutoImageProcessor.from_pretrained(config.model_id)\nall_loaders = get_loaders(df,processor,splitter,train_transforms,val_transforms)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:25:36.427783Z","iopub.execute_input":"2025-12-13T10:25:36.428471Z","iopub.status.idle":"2025-12-13T10:25:36.620862Z","shell.execute_reply.started":"2025-12-13T10:25:36.428451Z","shell.execute_reply":"2025-12-13T10:25:36.620110Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_batch(model,batch,loss_fn,optimizer,scheduler,scaler,DEVICE):\n    images = batch[0].to(DEVICE)\n    targets = batch[1].to(DEVICE)\n    optimizer.zero_grad()\n\n    with autocast(DEVICE):\n        outputs = model(images)\n        loss = loss_fn(outputs,targets)\n\n    scaler.scale(loss).backward()\n    scaler.step(optimizer)\n    scaler.update()\n    scheduler.step()\n    return loss.item()\n            \ndef val_one_batch(model,batch,loss_fn,scaler,DEVICE):\n    with torch.no_grad():\n        images = batch[0].to(DEVICE)\n        targets = batch[1].to(DEVICE)\n        with autocast(DEVICE):\n            outputs = model(images)\n            loss = loss_fn(outputs,targets)\n    return loss.item(),outputs.float().cpu(),targets.float().cpu()\n\nfrom sklearn.metrics import r2_score\n\ndef get_competition_metric(preds, targets):\n    # preds and targets must be RAW GRAMS (not logs)\n    # Shape: [N_samples, 5]\n    \n    # 1. Define Weights [Green, Dead, Clover, GDM, Total]\n    weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n    \n    # 2. Flatten everything to 1D arrays (Global calculation)\n    # We repeat the weights to match the number of rows\n    w_flat = np.tile(weights, (preds.shape[0], 1)).flatten()\n    y_true = targets.flatten()\n    y_pred = preds.flatten()\n    \n    y_bar_w = np.sum(w_flat * y_true) / np.sum(w_flat)\n    \n    ss_res = np.sum(w_flat * (y_true - y_pred)**2)\n    \n    ss_tot = np.sum(w_flat * (y_true - y_bar_w)**2)\n    \n    # 6. Final R2\n    return 1 - (ss_res / ss_tot)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T10:59:20.990130Z","iopub.execute_input":"2025-12-13T10:59:20.990415Z","iopub.status.idle":"2025-12-13T10:59:20.998829Z","shell.execute_reply.started":"2025-12-13T10:59:20.990395Z","shell.execute_reply":"2025-12-13T10:59:20.997965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.amp import autocast,GradScaler\nfrom tqdm.notebook import trange,tqdm\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nscaler = GradScaler(DEVICE)\nexperiment = 3\nmodel_file = f'/kaggle/working/ver_{experiment}'\nos.makedirs(model_file,exist_ok=True)\n\nfor idx,loaders in enumerate(all_loaders):\n    print(f'Training {idx+1} model')\n    \n    model = bioEstimatorAdv(config.model_id).to(DEVICE)\n    optimizer = torch.optim.AdamW([\n        {'params':model.base_model.parameters(),'lr':config.base_lr*5},\n        {'params':model.regressor.parameters(),'lr':config.reg_lr*10/3}\n])\n    w_tensor = w_tensor.to(DEVICE)\n    criterion = weightedMSE(w_tensor)\n    \n    scheduler = get_cosine_schedule_with_warmup(optimizer,num_warmup_steps=int(0.1*(NUM_EPOCHS*len(loaders[0]))),num_training_steps=NUM_EPOCHS*len(loaders[0]))\n    best_loss = float('inf') # We save based on Loss (safer)\n    best_r2 = -float('inf')  # But we track R2\n    patience = 3\n    trigger = 0\n    \n    for epochs in trange(NUM_EPOCHS,desc=f'Fold: {idx+1}'):\n    \n        ## Training \n        model.train()\n        train_losses = []\n        for batch in tqdm(loaders[0],desc='TRAINING',leave=False):\n            loss = train_one_batch(model,batch,criterion,optimizer,scheduler,scaler,DEVICE)\n            train_losses.append(loss)\n        avg_training_loss = np.mean(train_losses)\n\n        ##Validation\n        model.eval()\n        val_losses = []\n        all_outputs = []\n        all_targets = []\n        \n        for batch in tqdm(loaders[1],desc=\"VALIDATION\",leave=False):\n            loss,outputs,targets = val_one_batch(model,batch,criterion,scaler,DEVICE)\n            \n            val_losses.append(loss)\n            all_targets.append(targets)\n            all_outputs.append(outputs)\n\n        avg_validation_loss = np.mean(val_losses)\n        final_preds = np.vstack(all_outputs)   # Shape: [N_val, 5]\n        final_targets = np.vstack(all_targets)\n        \n        epoch_r2 = get_competition_metric(final_preds, final_targets)\n        \n        print(f'Epoch {epochs+1}: Train Loss={avg_training_loss:.4f}, Val Loss={avg_validation_loss:.4f}, Val R2={epoch_r2:.4f}')\n        if avg_validation_loss < best_loss:\n            best_loss = avg_validation_loss\n            best_r2 = epoch_r2\n            trigger = 0\n            save_path = f'{model_file}/DINO_fold{idx+1}.pth'\n            torch.save(model.state_dict(), save_path)\n            print(f'--> Saved Best Model! (R2: {best_r2:.4f})')\n        else:\n            trigger+=1\n            if trigger >= patience:\n                print(\"EARLY STOPPING\")\n                del model, optimizer, scheduler\n                torch.cuda.empty_cache()\n                break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T11:11:42.890455Z","iopub.execute_input":"2025-12-13T11:11:42.891173Z","iopub.status.idle":"2025-12-13T11:26:27.854208Z","shell.execute_reply.started":"2025-12-13T11:11:42.891146Z","shell.execute_reply":"2025-12-13T11:26:27.853224Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}