{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":292209427,"sourceType":"kernelVersion"},{"sourceId":293448493,"sourceType":"kernelVersion"}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In each version a different ViT Model is used.","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/transformers-wheel torch tqdm torchvision protobuf==3.20.3","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:10:38.310865Z","iopub.execute_input":"2026-01-23T08:10:38.311128Z","iopub.status.idle":"2026-01-23T08:10:44.134842Z","shell.execute_reply.started":"2026-01-23T08:10:38.311101Z","shell.execute_reply":"2026-01-23T08:10:44.134040Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport optuna\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoImageProcessor, AutoModel\nfrom PIL import Image\nimport urllib.request\nimport torchvision.transforms as T\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom tqdm import tqdm\nimport cv2\nimport gc\ndef report_gpu():\n    print(torch.cuda.list_gpu_processes())\n    gc.collect()\n    torch.cuda.empty_cache()\ndino_h = 518\ndino_w = 1036\nb_conv = 1024\ndino_small = 384\ndino_base = 768\ndino_large = 1024\ndino_giant = 1536\nh_14 = 518\nh_14_dim= 1280\nl_16 = 512\nl_16_dim = 1024\nb_16 = 518\nb_16_dim = 768\nLEARNING_RATE = 1e-3\nWEIGHT_DECAY = 1e-4\nEPOCHS = 50\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nMAIN_PATH = \"/kaggle/input/csiro-biomass\"\nHF_MODEL_PATH = \"/kaggle/input/dino-v2-models/dinov3_model_l16\"\nWEIGHTS_PATH = \"/kaggle/input/vit-models/vit_l_16_state_dict.pth\"\nreport_gpu()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:10:44.136920Z","iopub.execute_input":"2026-01-23T08:10:44.137258Z","iopub.status.idle":"2026-01-23T08:11:24.292286Z","shell.execute_reply.started":"2026-01-23T08:10:44.137204Z","shell.execute_reply":"2026-01-23T08:11:24.291529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:24.293094Z","iopub.execute_input":"2026-01-23T08:11:24.293716Z","iopub.status.idle":"2026-01-23T08:11:24.327348Z","shell.execute_reply.started":"2026-01-23T08:11:24.293689Z","shell.execute_reply":"2026-01-23T08:11:24.326546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, _ in train_data.iterrows():\n    train_data.loc[i, \"sample_id\"] = train_data.loc[i, \"sample_id\"].split(\"_\")[0]\ncols = [c for c in train_data.columns if c not in [\"target_name\", \"target\"]]\ndf_static = (\n    train_data[cols]\n      .drop_duplicates(subset=\"sample_id\")\n)\n\ndf_pivot = (\n    train_data.pivot(index=\"sample_id\", columns=\"target_name\", values=\"target\")\n      .rename(columns=lambda x: f\"{x}_target\")\n      .reset_index()\n)\n\ndf_final = df_static.merge(df_pivot, on=\"sample_id\", how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:24.328529Z","iopub.execute_input":"2026-01-23T08:11:24.328941Z","iopub.status.idle":"2026-01-23T08:11:24.873959Z","shell.execute_reply.started":"2026-01-23T08:11:24.328903Z","shell.execute_reply":"2026-01-23T08:11:24.873429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_final","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:24.875684Z","iopub.execute_input":"2026-01-23T08:11:24.875920Z","iopub.status.idle":"2026-01-23T08:11:24.903443Z","shell.execute_reply.started":"2026-01-23T08:11:24.875898Z","shell.execute_reply":"2026-01-23T08:11:24.902683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"processor = AutoImageProcessor.from_pretrained(HF_MODEL_PATH)\nmodel = AutoModel.from_pretrained(HF_MODEL_PATH)\nmodel.eval()\nmodel.to(device)\n\nprint(f\"Model Ready !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:24.904358Z","iopub.execute_input":"2026-01-23T08:11:24.904649Z","iopub.status.idle":"2026-01-23T08:11:33.735280Z","shell.execute_reply.started":"2026-01-23T08:11:24.904625Z","shell.execute_reply":"2026-01-23T08:11:33.734505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model_path = \"/kaggle/input/dinov2/pytorch/small/1\"\n# processor = AutoImageProcessor.from_pretrained(model_path)\n# model = AutoModel.from_pretrained(model_path).to(device)\n# for param in model.parameters():\n#     param.requires_grad = False\n# model.eval()\n# model.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.736256Z","iopub.execute_input":"2026-01-23T08:11:33.736507Z","iopub.status.idle":"2026-01-23T08:11:33.739822Z","shell.execute_reply.started":"2026-01-23T08:11:33.736485Z","shell.execute_reply":"2026-01-23T08:11:33.739151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocess = T.Compose([\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.740875Z","iopub.execute_input":"2026-01-23T08:11:33.741605Z","iopub.status.idle":"2026-01-23T08:11:33.766487Z","shell.execute_reply.started":"2026-01-23T08:11:33.741571Z","shell.execute_reply":"2026-01-23T08:11:33.765923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model = vit_l_16(image_size = 512, weights=None)\n# state_dict = torch.load(WEIGHTS_PATH, map_location=\"cpu\")\n# model.load_state_dict(state_dict)\n\n# model.heads = nn.Identity()\n# model.eval()\n# model.to(device)\n\n# print(f\"ViT Large 16 Ready !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.767307Z","iopub.execute_input":"2026-01-23T08:11:33.767576Z","iopub.status.idle":"2026-01-23T08:11:33.780765Z","shell.execute_reply.started":"2026-01-23T08:11:33.767547Z","shell.execute_reply":"2026-01-23T08:11:33.780228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Step3ConsistentHead(nn.Module):\n    def __init__(self, feat_dim=768, hidden_dim=1024):\n        super().__init__()\n        self.norm = nn.LayerNorm(feat_dim)\n        self.attn = nn.MultiheadAttention(embed_dim=feat_dim, num_heads=8, batch_first=True)\n        \n        self.mlp_shared = nn.Sequential(\n            nn.Linear(feat_dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(0.3),\n            nn.Linear(hidden_dim, hidden_dim // 2),\n            nn.GELU(),\n            nn.Dropout(0.3),\n            nn.Linear(hidden_dim // 2 , hidden_dim // 4),\n            nn.GELU()\n        )\n        # 4 outputs: Clover, Dead, Green, Total\n        self.regressor = nn.Linear(hidden_dim // 4, 4) \n        self.softplus = nn.Softplus()\n\n    def forward(self, x):\n        res = x\n        x = self.norm(x)\n        x, _ = self.attn(x, x, x)\n        x = (x + res).mean(dim=1) \n        shared = self.mlp_shared(x)\n        return self.softplus(self.regressor(shared))\n\nclass Step3MultiScaleModel(nn.Module):\n    def __init__(self, vit_backbone, head):\n        super().__init__()\n        self.vit = vit_backbone\n        self.head = head\n\n    def forward(self, x):\n        b = x.shape[0]\n        feats_list = []\n\n        # 1. Process Global Views First\n        # Resize to 512x1024 and split\n        global_img = F.interpolate(x, size=(512, 1024), mode='bilinear', align_corners=False)\n        g_views = [global_img[:, :, :, :512], global_img[:, :, :, 512:]]\n        \n        # Immediate Extraction\n        with torch.no_grad(): # Backbone is frozen per ABC.md\n            g_batch = torch.cat(g_views, dim=0)\n            g_feats = self.vit(g_batch).last_hidden_state.detach()\n            feats_list.append(g_feats)\n        \n        # CLEANUP: Remove global images from VRAM\n        del global_img, g_views, g_batch\n        torch.cuda.empty_cache()\n\n        # 2. Process Local Tiles Sequentially (2 at a time)\n        h_step, w_step = x.shape[2] // 2, x.shape[3] // 4 \n        tile_coords = [(i, j) for i in range(2) for j in range(4)]\n        \n        for idx in range(0, 8, 2):\n            pair_tiles = []\n            for i, j in tile_coords[idx : idx + 2]:\n                # Dynamic crop to save memory\n                tile = x[:, :, i*h_step:(i+1)*h_step, j*w_step:(j+1)*w_step]\n                pair_tiles.append(F.interpolate(tile, size=(512, 512), mode='bilinear'))\n            \n            # Extract features for this pair\n            with torch.no_grad():\n                t_batch = torch.cat(pair_tiles, dim=0)\n                t_feats = self.vit(t_batch).last_hidden_state.detach()\n                feats_list.append(t_feats)\n            \n            # CLEANUP: Remove tile tensors immediately\n            del pair_tiles, t_batch\n            torch.cuda.empty_cache()\n\n        # 3. Concatenate all 10 views (2 global + 8 local)\n        feats = torch.cat(feats_list, dim=0) # [10*B, Patches, 768]\n        \n        # 4. Re-organize for Head\n        num_patches = feats.shape[1]\n        feats = feats.view(10, b, num_patches, -1).permute(1, 0, 2, 3).flatten(1, 2)\n        \n        return self.head(feats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.781733Z","iopub.execute_input":"2026-01-23T08:11:33.782026Z","iopub.status.idle":"2026-01-23T08:11:33.797294Z","shell.execute_reply.started":"2026-01-23T08:11:33.781995Z","shell.execute_reply":"2026-01-23T08:11:33.796753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"head_step3 = Step3ConsistentHead(feat_dim=1024).to(device)\nmodel_step3 = Step3MultiScaleModel(model, head_step3).to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.798126Z","iopub.execute_input":"2026-01-23T08:11:33.798486Z","iopub.status.idle":"2026-01-23T08:11:33.877213Z","shell.execute_reply.started":"2026-01-23T08:11:33.798452Z","shell.execute_reply":"2026-01-23T08:11:33.876608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_step3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.878103Z","iopub.execute_input":"2026-01-23T08:11:33.878336Z","iopub.status.idle":"2026-01-23T08:11:33.883926Z","shell.execute_reply.started":"2026-01-23T08:11:33.878314Z","shell.execute_reply":"2026-01-23T08:11:33.883392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, transform):\n        self.df = df\n        self.transform = transform\n        self.image_paths = df['image_path'].values\n        self.labels = torch.tensor(df[[\"Dry_Clover_g_target\", \"Dry_Dead_g_target\", \"Dry_Green_g_target\"]].values, dtype=torch.float32)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        path = os.path.join(MAIN_PATH, self.image_paths[idx])\n        image = Image.open(path).convert(\"RGB\")\n        return self.transform(image), self.labels[idx]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.884723Z","iopub.execute_input":"2026-01-23T08:11:33.885250Z","iopub.status.idle":"2026-01-23T08:11:33.900035Z","shell.execute_reply.started":"2026-01-23T08:11:33.885212Z","shell.execute_reply":"2026-01-23T08:11:33.899296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ds = BiomassDataset(df_final, preprocess)\ntrain_loader = DataLoader(train_ds, batch_size=1, shuffle=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.902740Z","iopub.execute_input":"2026-01-23T08:11:33.903337Z","iopub.status.idle":"2026-01-23T08:11:33.923175Z","shell.execute_reply.started":"2026-01-23T08:11:33.903313Z","shell.execute_reply":"2026-01-23T08:11:33.922586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for param in model_step3.vit.parameters():\n    param.requires_grad = False\n\n# Use AdamW for Attention blocks\noptimizer = optim.AdamW(model_step3.head.parameters(), lr=1e-4, weight_decay=1e-2)\ncriterion = nn.MSELoss()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.924100Z","iopub.execute_input":"2026-01-23T08:11:33.924417Z","iopub.status.idle":"2026-01-23T08:11:33.938926Z","shell.execute_reply.started":"2026-01-23T08:11:33.924379Z","shell.execute_reply":"2026-01-23T08:11:33.938403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for epoch in range(50):\n    model_step3.train()\n    alpha = 0.5\n\n    for images, labels in tqdm(train_loader):\n        \n        images, labels = images.to(device), labels.to(device)\n        target_total = labels.sum(dim=1, keepdim=True)\n        targets_with_total = torch.cat([labels, target_total], dim=1)\n    \n        optimizer.zero_grad()\n        outputs = model_step3(images)\n        base_loss = criterion(outputs, targets_with_total)\n    \n        predicted_sum = outputs[:, :3].sum(dim=1, keepdim=True)\n        explicit_total = outputs[:, 3:]\n        sum_consistency_loss = criterion(predicted_sum, explicit_total)\n    \n        total_loss = base_loss + (alpha * sum_consistency_loss)\n    \n        total_loss.backward()\n        optimizer.step()\n    print(f\"Epoch {epoch + 1} completed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-23T08:11:33.939824Z","iopub.execute_input":"2026-01-23T08:11:33.940100Z","execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.save(model_step3.state_dict(), 'model_weights.pth')","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for param in model_step1.vit.parameters():\n#     param.requires_grad = True\n\n# optimizer = optim.AdamW([\n#     {'params': model_step1.vit.parameters(), 'lr': 1e-6},\n#     {'params': model_step1.head.parameters(), 'lr': 1e-5}\n# ], weight_decay=1e-2)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for epoch in range(5):\n#     model_step1.train()\n#     total_loss = 0\n#     for images, labels in tqdm(train_loader):\n#         images, labels = images.to(device), labels.to(device)\n        \n#         optimizer.zero_grad()\n#         outputs = model_step1(images)\n#         loss = criterion(outputs, labels)\n#         loss.backward()\n#         optimizer.step()\n#         total_loss += loss.item()\n#     print(f\"Epoch {epoch+1} | Loss: {total_loss/len(train_loader):.4f}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(\"Saving weights...\")\n# if isinstance(model_step1, nn.DataParallel):\n#     torch.save(model_step1.module.state_dict(), \"vit_h_14_finetuned_biomass_2.pth\")\n# else:\n#     torch.save(model_step1.state_dict(), \"vit_h_14_finetuned_biomass_2.pth\")\n\n# print(\"Saved clean weights (no 'module.' prefix).\")","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_data_2 = test_data.copy()\n# for i, _ in test_data.iterrows():\n#     test_data_2.loc[i, \"sample_id\"] = test_data_2.loc[i, \"sample_id\"].split(\"_\")[0]\n# cols = [c for c in test_data_2.columns if c not in [\"target_name\", \"target\"]]\n# final_test_data = (\n#     test_data_2[cols]\n#       .drop_duplicates(subset=\"sample_id\")\n# )","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# final_test_data","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class TestDataset(Dataset):\n#     def __init__(self, df, preprocess):\n#         self.df = df\n#         self.preprocess = preprocess\n#         self.image_paths = df['image_path'].values\n        \n#     def __len__(self):\n#         return len(self.df)\n\n#     def __getitem__(self, idx):\n#         path = os.path.join(MAIN_PATH, self.image_paths[idx])\n#         image = Image.open(path).convert(\"RGB\")\n\n#         return self.preprocess(image), torch.tensor(0.0)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_ds = TestDataset(final_test_data, preprocess)\n# test_loader = DataLoader(test_ds, batch_size=2, shuffle=False)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# all_preds = []\n\n# with torch.no_grad():\n#     model_step3.eval()\n#     for images, _ in tqdm(test_loader):\n#         images = images.to(device)\n#         outputs = model_step3(images)    \n#         all_preds.append(outputs.detach().cpu().numpy())\n\n# predictions = np.vstack(all_preds)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# predictions","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# clover = predictions[:, 0]\n# dead = predictions[:, 1]\n# green = predictions[:, 2]\n# total = clover + dead + green\n# gdm = clover + green\n# print(f\"Targets Predicted !\")","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# c, d, g, t, gd = 0, 0, 0, 0, 0\n# submission = []\n# for _, row in test_data.iterrows():\n#     if \"Clover\" in row[\"sample_id\"]:\n#         value = clover[c].item()\n#         c += 1\n#     elif \"Dead\" in row[\"sample_id\"]:\n#         value = dead[d].item()\n#         d += 1\n#     elif \"Green\" in row[\"sample_id\"]:\n#         value = green[g].item()\n#         g += 1\n#     elif \"Total\" in row[\"sample_id\"]:\n#         value = total[t].item()\n#         t += 1\n#     else:\n#         value = gdm[gd].item()\n#         gd += 1\n#     submission_entry = {\n#         \"sample_id\" : row[\"sample_id\"],\n#         \"target\" : value\n#     }\n#     submission.append(submission_entry)\n# submission = pd.DataFrame(submission)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# submission.to_csv(\"/kaggle/working/submission.csv\", index = False)","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"The entire notebook runs successfully, if error happens, I don't just know why.\")","metadata":{"trusted":true,"execution":{"execution_failed":"2026-01-23T09:03:31.744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}