{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13846405,"sourceType":"datasetVersion","datasetId":8679417,"isSourceIdPinned":true},{"sourceId":4537,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3329,"modelId":986}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🌿🌱 CSIRO Biomass Regression using Dense Features of DINOv2\n\n- Use the dense patch-based features extracted by the DINOv2 model\n- For each patch feature vector, use a common MLP (with weight sharing) to make predictions for each patch\n- Average the MLP predictions for each patch to obtain final predictions\n- Uses sharpness-aware minimization (https://github.com/davda54/sam) for training (training code not shown here)\n- Computes loss only on image-level labels (TODO: incorporate regularizations to make the problem more well-conditioned, or use other methods to compute loss on patch-level labels)","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"#!pip install transformers==4.57.1\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torchvision\nimport os\nfrom torchvision.datasets import ImageFolder\nfrom torch.utils.data import DataLoader, Subset, Dataset\nimport torchvision.transforms as transforms\nfrom PIL import Image\n!cp -r \"/kaggle/input/rsna-models/facebookresearch_dinov2_main (1)/root/.cache/torch/hub/facebookresearch_dinov2_main\" /kaggle/working/dinov2\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-29T02:26:20.336667Z","iopub.execute_input":"2025-11-29T02:26:20.336908Z","iopub.status.idle":"2025-11-29T02:26:30.727973Z","shell.execute_reply.started":"2025-11-29T02:26:20.336881Z","shell.execute_reply":"2025-11-29T02:26:30.727132Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load DINOv2 backbone","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working/dinov2\nimport torch.nn as nn\nfrom transformers import Dinov2Model\nmodel = Dinov2Model.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\nmodel.head = nn.Identity()\nmodel = model.cuda()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T02:26:30.730124Z","iopub.execute_input":"2025-11-29T02:26:30.730727Z","iopub.status.idle":"2025-11-29T02:27:18.526950Z","shell.execute_reply.started":"2025-11-29T02:26:30.730703Z","shell.execute_reply":"2025-11-29T02:27:18.525952Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MLP Architecture","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nclass BiomassMLP(nn.Module):\n    # and also weight initialization\n    def __init__(self, input_size, hidden_size=512, dropout_rate=0.3):\n        super(BiomassMLP, self).__init__()\n        self.network = nn.Sequential(\n            nn.Linear(input_size, hidden_size // 4),\n            #nn.BatchNorm1d(hidden_size),\n            nn.LeakyReLU(),\n            nn.Linear(hidden_size // 4, 1)\n        )\n        #self.network = nn.Linear(input_size, 1) # use a simple MLP with a single activation function, or a polynomial function\n        #self.network = SimplePolynomialLayer(input_size)\n    def forward(self, x):\n        #return self.network(x)\n        return torch.mean(torch.relu(self.network(x)), dim=(1, 2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T02:27:18.549700Z","iopub.execute_input":"2025-11-29T02:27:18.549962Z","iopub.status.idle":"2025-11-29T02:27:18.555221Z","shell.execute_reply.started":"2025-11-29T02:27:18.549944Z","shell.execute_reply":"2025-11-29T02:27:18.554392Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inference\n\n- TTA (test-time augmentation) is incorporated into the inference\n- The augmentations include horizontal / vertical flips, rotations, and Gaussian blurs","metadata":{}},{"cell_type":"code","source":"mapping = {\"Dry_Clover_g\": 0, \"Dry_Dead_g\": 1, \"Dry_Green_g\": 2, \"Dry_Total_g\": 3, \"GDM_g\": 4}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T02:27:18.555840Z","iopub.execute_input":"2025-11-29T02:27:18.556068Z","iopub.status.idle":"2025-11-29T02:27:18.570117Z","shell.execute_reply.started":"2025-11-29T02:27:18.556053Z","shell.execute_reply":"2025-11-29T02:27:18.569403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_embeds = {}\ncounter = 0\nimport torchvision.transforms as transforms\nimport pandas as pd\nfrom PIL import Image\ntransform = transforms.Compose([transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(mean, std)])\ntest_df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")\naugmentation_transforms = [\n    transforms.Compose([transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(mean, std), torchvision.transforms.GaussianBlur(5)]),  # Original\n    transforms.Compose([transforms.RandomHorizontalFlip(p=1.0), transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(mean, std), torchvision.transforms.GaussianBlur(5)]),  # Horizontal flip\n    transforms.Compose([transforms.RandomVerticalFlip(p=1.0), transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(mean, std), torchvision.transforms.GaussianBlur(5)]),  # Vertical flip\n    transforms.Compose([transforms.RandomHorizontalFlip(p=1.0), transforms.RandomVerticalFlip(p=1.0), transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(mean, std), torchvision.transforms.GaussianBlur(5)]),  # Both flips\n    transforms.Compose([transforms.RandomRotation(degrees=90), transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(mean, std), torchvision.transforms.GaussianBlur(5)]),  # 90 degree rotation\n    transforms.Compose([transforms.RandomRotation(degrees=270), transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(mean, std), torchvision.transforms.GaussianBlur(5)]),  # 270 degree rotation\n]\nroot = \"/kaggle/input/csiro-biomass/\"\nsample_ids = []\nfrom tqdm import tqdm\nfor i in tqdm(range(len(test_df))):\n    entry = test_df.iloc[i]\n    file_path = root + entry['image_path']\n    sample_id = entry['sample_id']\n    #y = torch.tensor([[entry['target']]])\n    if sample_id.split(\"_\")[0] not in sample_ids:\n        image_embeddings = []\n        for aug in augmentation_transforms:\n            img = Image.open(file_path)\n            x = aug(img).unsqueeze(0)\n            with torch.no_grad():\n                x = x.cuda()\n                image_embeddings.append(torch.cat([model(x).last_hidden_state[:,1:,:].cpu()[0]]).unsqueeze(0))\n                counter += 1\n        #print(image_embeddings[0].shape, sample_id)\n        test_embeds[sample_id.split(\"_\")[0]] = torch.stack(image_embeddings, dim=0)\n        #print(torch.stack(image_embeddings, dim=0).shape)\n        sample_ids.append(sample_id.split(\"_\")[0])        \n    if counter % 100 == 0:\n        print(f\"{counter} batches processed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T02:27:18.570757Z","iopub.execute_input":"2025-11-29T02:27:18.570975Z","iopub.status.idle":"2025-11-29T02:27:20.642131Z","shell.execute_reply.started":"2025-11-29T02:27:18.570959Z","shell.execute_reply":"2025-11-29T02:27:20.641341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"regressors = [[None for i in range(5)] for j in range(5)]\nfor i in range(5):\n    for j in range(5):\n        regressors[i][j] = torch.load(f\"/kaggle/input/csiro-mlps-run1/target{i}/fold{j}.pt\", weights_only=False)\nimport joblib\n(selected_features, scalers) = joblib.load(\"/kaggle/input/csiro-mlps-run1/sfs_scalers_run1.joblib\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T02:27:20.643992Z","iopub.execute_input":"2025-11-29T02:27:20.644295Z","iopub.status.idle":"2025-11-29T02:27:21.124460Z","shell.execute_reply.started":"2025-11-29T02:27:20.644276Z","shell.execute_reply":"2025-11-29T02:27:21.123836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions = []\nsample_ids = []\ntest_df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")\nfor i in range(len(test_df)):\n    entry = test_df.iloc[i]\n    X = test_embeds[entry['sample_id'].split(\"__\")[0]]\n    sample_ids.append(entry['sample_id'])\n    models = regressors[mapping[entry['sample_id'].split(\"__\")[1]]]\n    sfs = selected_features[mapping[entry['sample_id'].split(\"__\")[1]]]\n    scaler_list = scalers[mapping[entry['sample_id'].split(\"__\")[1]]]\n    prediction = 0.0\n    for i in range(len(models)):\n        item = models[i]\n        scaler = scaler_list[i]\n        sf = sfs[i]\n        item.eval()\n        #print(data)\n        #print(item)\n        #print(item(data))\n        #print(data.shape)\n        single_pred = torch.mean(torch.relu(item(X.squeeze(1).cuda())))\n        #single_pred = item(torch.tensor(scaler.transform(X[:,sf])))\n        if single_pred < 0.0:\n            single_pred = 0.0\n        prediction += single_pred.cpu()\n    prediction = prediction / 5\n    predictions.append(float(prediction))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T02:27:21.125237Z","iopub.execute_input":"2025-11-29T02:27:21.125497Z","iopub.status.idle":"2025-11-29T02:27:21.282374Z","shell.execute_reply.started":"2025-11-29T02:27:21.125467Z","shell.execute_reply":"2025-11-29T02:27:21.281777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%cd /kaggle/working\nsubmission = pd.DataFrame({\n    'sample_id': sample_ids,\n    'target': predictions\n})\n\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T02:27:21.283069Z","iopub.execute_input":"2025-11-29T02:27:21.283317Z","iopub.status.idle":"2025-11-29T02:27:21.306453Z","shell.execute_reply.started":"2025-11-29T02:27:21.283301Z","shell.execute_reply":"2025-11-29T02:27:21.305884Z"}},"outputs":[],"execution_count":null}]}