{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false},{"sourceType":"kernelVersion","sourceId":292209427,"isSourceIdPinned":false},{"sourceType":"kernelVersion","sourceId":293448493,"isSourceIdPinned":false},{"sourceType":"kernelVersion","sourceId":293711421,"isSourceIdPinned":false}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is the updated submission notebook of the CSIRO Competition, which uses Test Time Augmentation for predictions. For the original model training script go to the following link :-\n\nhttps://www.kaggle.com/code/kushubhai/fork-of-the-vit-approach","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/transformers-wheel torch tqdm torchvision protobuf==3.20.3","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-21T11:26:02.344663Z","iopub.execute_input":"2026-02-21T11:26:02.344982Z","iopub.status.idle":"2026-02-21T11:26:08.023841Z","shell.execute_reply.started":"2026-02-21T11:26:02.344945Z","shell.execute_reply":"2026-02-21T11:26:08.022749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport optuna\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoImageProcessor, AutoModel\nfrom PIL import Image\nimport urllib.request\nimport torchvision.transforms as T\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom tqdm import tqdm\nimport cv2\nimport gc\n\ndef report_gpu():\n    print(torch.cuda.list_gpu_processes())\n    gc.collect()\n    torch.cuda.empty_cache()\ndino_h = 518\ndino_w = 1036\nb_conv = 1024\ndino_small = 384\ndino_base = 768\ndino_large = 1024\ndino_giant = 1536\nh_14 = 518\nh_14_dim= 1280\nl_16 = 512\nl_16_dim = 1024\nb_16 = 518\nb_16_dim = 768\nLEARNING_RATE = 1e-3\nWEIGHT_DECAY = 1e-4\nEPOCHS = 50\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nMAIN_PATH = \"/kaggle/input/csiro-biomass\"\nHF_MODEL_PATH = \"/kaggle/input/dino-v2-models/dinov3_model_l16\"\nreport_gpu()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:40:30.654680Z","iopub.execute_input":"2026-02-21T10:40:30.654945Z","iopub.status.idle":"2026-02-21T10:41:08.171694Z","shell.execute_reply.started":"2026-02-21T10:40:30.654909Z","shell.execute_reply":"2026-02-21T10:41:08.170943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:08.172695Z","iopub.execute_input":"2026-02-21T10:41:08.173416Z","iopub.status.idle":"2026-02-21T10:41:08.209849Z","shell.execute_reply.started":"2026-02-21T10:41:08.173390Z","shell.execute_reply":"2026-02-21T10:41:08.209147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"processor = AutoImageProcessor.from_pretrained(HF_MODEL_PATH)\nmodel = AutoModel.from_pretrained(HF_MODEL_PATH)\nmodel.eval()\nmodel.to(device)\n\nprint(f\"Model Ready !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:08.210802Z","iopub.execute_input":"2026-02-21T10:41:08.211089Z","iopub.status.idle":"2026-02-21T10:41:17.462323Z","shell.execute_reply.started":"2026-02-21T10:41:08.211057Z","shell.execute_reply":"2026-02-21T10:41:17.461686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocess = T.Compose([\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:17.463147Z","iopub.execute_input":"2026-02-21T10:41:17.463425Z","iopub.status.idle":"2026-02-21T10:41:17.467426Z","shell.execute_reply.started":"2026-02-21T10:41:17.463400Z","shell.execute_reply":"2026-02-21T10:41:17.466753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Step3ConsistentHead(nn.Module):\n    def __init__(self, feat_dim=768, hidden_dim=1024):\n        super().__init__()\n        self.norm = nn.LayerNorm(feat_dim)\n        self.attn = nn.MultiheadAttention(embed_dim=feat_dim, num_heads=8, batch_first=True)\n        \n        self.mlp_shared = nn.Sequential(\n            nn.Linear(feat_dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(0.1),\n            nn.Linear(hidden_dim, hidden_dim // 2),\n            nn.GELU(),\n            nn.Dropout(0.1),\n            nn.Linear(hidden_dim // 2, hidden_dim // 4),\n            nn.GELU()\n        )\n        self.regressor = nn.Linear(hidden_dim // 4, 4) \n        self.softplus = nn.Softplus()\n\n    def forward(self, x):\n        res = x\n        x = self.norm(x)\n        x, _ = self.attn(x, x, x)\n        x = (x + res).mean(dim=1) \n        shared = self.mlp_shared(x)\n        return self.softplus(self.regressor(shared))\n\nclass Step3MultiScaleModel(nn.Module):\n    def __init__(self, vit_backbone, head):\n        super().__init__()\n        self.vit = vit_backbone\n        self.head = head\n\n    def forward(self, x):\n        b = x.shape[0]\n        feats_list = []\n\n        global_img = F.interpolate(x, size=(512, 1024), mode='bilinear', align_corners=False)\n        g_views = [global_img[:, :, :, :512], global_img[:, :, :, 512:]]\n\n        with torch.no_grad():\n            g_batch = torch.cat(g_views, dim=0)\n            g_feats = self.vit(g_batch).last_hidden_state.detach()\n            feats_list.append(g_feats)\n\n        del global_img, g_views, g_batch\n        torch.cuda.empty_cache()\n\n        h_step, w_step = x.shape[2] // 2, x.shape[3] // 4 \n        tile_coords = [(i, j) for i in range(2) for j in range(4)]\n        \n        for idx in range(0, 8, 2):\n            pair_tiles = []\n            for i, j in tile_coords[idx : idx + 2]:\n\n                tile = x[:, :, i*h_step:(i+1)*h_step, j*w_step:(j+1)*w_step]\n                pair_tiles.append(F.interpolate(tile, size=(512, 512), mode='bilinear'))\n\n            with torch.no_grad():\n                t_batch = torch.cat(pair_tiles, dim=0)\n                t_feats = self.vit(t_batch).last_hidden_state.detach()\n                feats_list.append(t_feats)\n\n            del pair_tiles, t_batch\n            torch.cuda.empty_cache()\n\n        feats = torch.cat(feats_list, dim=0)\n\n        num_patches = feats.shape[1]\n        feats = feats.view(10, b, num_patches, -1).permute(1, 0, 2, 3).flatten(1, 2)\n        \n        return self.head(feats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:17.469183Z","iopub.execute_input":"2026-02-21T10:41:17.469528Z","iopub.status.idle":"2026-02-21T10:41:17.515958Z","shell.execute_reply.started":"2026-02-21T10:41:17.469468Z","shell.execute_reply":"2026-02-21T10:41:17.515381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"head_step3 = Step3ConsistentHead(feat_dim=1024).to(device)\nmodel_step3 = Step3MultiScaleModel(model, head_step3).to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:17.518279Z","iopub.execute_input":"2026-02-21T10:41:17.518780Z","iopub.status.idle":"2026-02-21T10:41:17.589335Z","shell.execute_reply.started":"2026-02-21T10:41:17.518755Z","shell.execute_reply":"2026-02-21T10:41:17.588539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_step3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:17.590218Z","iopub.execute_input":"2026-02-21T10:41:17.590513Z","iopub.status.idle":"2026-02-21T10:41:17.597170Z","shell.execute_reply.started":"2026-02-21T10:41:17.590459Z","shell.execute_reply":"2026-02-21T10:41:17.596436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_step3.load_state_dict(torch.load('/kaggle/input/notebooks/kushubhai/fork-of-the-vit-approach/model_weights.pth'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:17.598104Z","iopub.execute_input":"2026-02-21T10:41:17.598467Z","iopub.status.idle":"2026-02-21T10:41:23.043898Z","shell.execute_reply.started":"2026-02-21T10:41:17.598435Z","shell.execute_reply":"2026-02-21T10:41:23.043215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data_2 = test_data.copy()\nfor i, _ in test_data.iterrows():\n    test_data_2.loc[i, \"sample_id\"] = test_data_2.loc[i, \"sample_id\"].split(\"_\")[0]\ncols = [c for c in test_data_2.columns if c not in [\"target_name\", \"target\"]]\nfinal_test_data = (\n    test_data_2[cols]\n      .drop_duplicates(subset=\"sample_id\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:23.044828Z","iopub.execute_input":"2026-02-21T10:41:23.045092Z","iopub.status.idle":"2026-02-21T10:41:23.067909Z","shell.execute_reply.started":"2026-02-21T10:41:23.045070Z","shell.execute_reply":"2026-02-21T10:41:23.067354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_test_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:23.068809Z","iopub.execute_input":"2026-02-21T10:41:23.069091Z","iopub.status.idle":"2026-02-21T10:41:23.094069Z","shell.execute_reply.started":"2026-02-21T10:41:23.069059Z","shell.execute_reply":"2026-02-21T10:41:23.093541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, preprocess):\n        self.df = df\n        self.preprocess = preprocess\n        self.image_paths = df['image_path'].values\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        path = os.path.join(MAIN_PATH, self.image_paths[idx])\n        image = Image.open(path).convert(\"RGB\")\n\n        return self.preprocess(image), torch.tensor(0.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:23.095005Z","iopub.execute_input":"2026-02-21T10:41:23.095310Z","iopub.status.idle":"2026-02-21T10:41:23.100192Z","shell.execute_reply.started":"2026-02-21T10:41:23.095277Z","shell.execute_reply":"2026-02-21T10:41:23.099440Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ds = TestDataset(final_test_data, preprocess)\ntest_loader = DataLoader(test_ds, batch_size=2, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:23.101026Z","iopub.execute_input":"2026-02-21T10:41:23.101312Z","iopub.status.idle":"2026-02-21T10:41:23.112063Z","shell.execute_reply.started":"2026-02-21T10:41:23.101281Z","shell.execute_reply":"2026-02-21T10:41:23.111534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_preds = []\n\nwith torch.no_grad():\n    model_step3.eval()\n    for images, _ in tqdm(test_loader):\n        images = images.to(device)\n        out1 = model_step3(images)\n        \n        images_hf = torch.flip(images, dims=[3])\n        out2 = model_step3(images_hf)\n        \n        images_vf = torch.flip(images, dims=[2])\n        out3 = model_step3(images_vf)\n\n        avg_output = (out1 + out2 + out3) / 3.0\n        \n        all_preds.append(avg_output.detach().cpu().numpy())\n\npredictions = np.vstack(all_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:23.112843Z","iopub.execute_input":"2026-02-21T10:41:23.113038Z","iopub.status.idle":"2026-02-21T10:41:29.031131Z","shell.execute_reply.started":"2026-02-21T10:41:23.113019Z","shell.execute_reply":"2026-02-21T10:41:29.030241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:29.032280Z","iopub.execute_input":"2026-02-21T10:41:29.032586Z","iopub.status.idle":"2026-02-21T10:41:29.038020Z","shell.execute_reply.started":"2026-02-21T10:41:29.032562Z","shell.execute_reply":"2026-02-21T10:41:29.037354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"clover = predictions[:, 0]\ndead = predictions[:, 1]\ngreen = predictions[:, 2]\ntotal = clover + dead + green\ngdm = clover + green\nprint(f\"Targets Predicted !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:29.038996Z","iopub.execute_input":"2026-02-21T10:41:29.039440Z","iopub.status.idle":"2026-02-21T10:41:29.111356Z","shell.execute_reply.started":"2026-02-21T10:41:29.039418Z","shell.execute_reply":"2026-02-21T10:41:29.110710Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"c, d, g, t, gd = 0, 0, 0, 0, 0\nsubmission = []\nfor _, row in test_data.iterrows():\n    if \"Clover\" in row[\"sample_id\"]:\n        value = clover[c].item()\n        c += 1\n    elif \"Dead\" in row[\"sample_id\"]:\n        value = dead[d].item()\n        d += 1\n    elif \"Green\" in row[\"sample_id\"]:\n        value = green[g].item()\n        g += 1\n    elif \"Total\" in row[\"sample_id\"]:\n        value = total[t].item()\n        t += 1\n    else:\n        value = gdm[gd].item()\n        gd += 1\n    submission_entry = {\n        \"sample_id\" : row[\"sample_id\"],\n        \"target\" : value\n    }\n    submission.append(submission_entry)\nsubmission = pd.DataFrame(submission)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:29.112189Z","iopub.execute_input":"2026-02-21T10:41:29.112386Z","iopub.status.idle":"2026-02-21T10:41:29.124561Z","shell.execute_reply.started":"2026-02-21T10:41:29.112353Z","shell.execute_reply":"2026-02-21T10:41:29.124027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:29.125313Z","iopub.execute_input":"2026-02-21T10:41:29.125569Z","iopub.status.idle":"2026-02-21T10:41:29.139241Z","shell.execute_reply.started":"2026-02-21T10:41:29.125547Z","shell.execute_reply":"2026-02-21T10:41:29.138535Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"/kaggle/working/submission.csv\", index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:29.140133Z","iopub.execute_input":"2026-02-21T10:41:29.140345Z","iopub.status.idle":"2026-02-21T10:41:29.156418Z","shell.execute_reply.started":"2026-02-21T10:41:29.140325Z","shell.execute_reply":"2026-02-21T10:41:29.155844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"The entire notebook runs successfully, if error happens, I don't just know why.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-21T10:41:29.157324Z","iopub.execute_input":"2026-02-21T10:41:29.157554Z","iopub.status.idle":"2026-02-21T10:41:29.160883Z","shell.execute_reply.started":"2026-02-21T10:41:29.157533Z","shell.execute_reply":"2026-02-21T10:41:29.160361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}