{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":286802727,"sourceType":"kernelVersion"},{"sourceId":286813572,"sourceType":"kernelVersion"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In each version a different ViT Model is used.","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/transformers-wheel torch tqdm torchvision optuna-integration[xgboost] keras keras-cv transformers==4.39.0 protobuf==3.20.3","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:07.560205Z","iopub.execute_input":"2025-12-17T19:32:07.560577Z","iopub.status.idle":"2025-12-17T19:32:10.339639Z","shell.execute_reply.started":"2025-12-17T19:32:07.560545Z","shell.execute_reply":"2025-12-17T19:32:10.338785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport optuna\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.models import vit_h_14, vit_l_16, vit_b_16\nfrom PIL import Image\nimport urllib.request\nimport torchvision.transforms as T\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom tqdm import tqdm\nimport cv2\n\nh_14 = 518\nl_16 = 512\nb_16 = 384\nLEARNING_RATE = 1e-3\nWEIGHT_DECAY = 1e-4\nEPOCHS = 50\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nMAIN_PATH = \"/kaggle/input/csiro-biomass\"\nWEIGHTS_PATH = \"/kaggle/input/vit-models/vit_l_16_state_dict.pth\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:10.341877Z","iopub.execute_input":"2025-12-17T19:32:10.342502Z","iopub.status.idle":"2025-12-17T19:32:21.181578Z","shell.execute_reply.started":"2025-12-17T19:32:10.342459Z","shell.execute_reply":"2025-12-17T19:32:21.180851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:21.182473Z","iopub.execute_input":"2025-12-17T19:32:21.182860Z","iopub.status.idle":"2025-12-17T19:32:21.216180Z","shell.execute_reply.started":"2025-12-17T19:32:21.182827Z","shell.execute_reply":"2025-12-17T19:32:21.215405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, _ in train_data.iterrows():\n    train_data.loc[i, \"sample_id\"] = train_data.loc[i, \"sample_id\"].split(\"_\")[0]\ncols = [c for c in train_data.columns if c not in [\"target_name\", \"target\"]]\ndf_static = (\n    train_data[cols]\n      .drop_duplicates(subset=\"sample_id\")\n)\n\ndf_pivot = (\n    train_data.pivot(index=\"sample_id\", columns=\"target_name\", values=\"target\")\n      .rename(columns=lambda x: f\"{x}_target\")\n      .reset_index()\n)\n\ndf_final = df_static.merge(df_pivot, on=\"sample_id\", how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:21.217137Z","iopub.execute_input":"2025-12-17T19:32:21.217376Z","iopub.status.idle":"2025-12-17T19:32:21.727505Z","shell.execute_reply.started":"2025-12-17T19:32:21.217353Z","shell.execute_reply":"2025-12-17T19:32:21.726876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_final","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:21.728374Z","iopub.execute_input":"2025-12-17T19:32:21.728658Z","iopub.status.idle":"2025-12-17T19:32:21.755799Z","shell.execute_reply.started":"2025-12-17T19:32:21.728634Z","shell.execute_reply":"2025-12-17T19:32:21.755227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = vit_l_16(image_size = l_16, weights=None)\nstate_dict = torch.load(WEIGHTS_PATH, map_location=\"cpu\")\nmodel.load_state_dict(state_dict)\n\npreprocess = T.Compose([\n    T.Resize(l_16, interpolation=T.InterpolationMode.BILINEAR),\n    T.CenterCrop(l_16),\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nmodel.heads = nn.Identity()\nmodel.eval()\nmodel.to(device)\n\nprint(f\"ViT Large 16 Ready !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:21.756682Z","iopub.execute_input":"2025-12-17T19:32:21.756960Z","iopub.status.idle":"2025-12-17T19:32:34.203082Z","shell.execute_reply.started":"2025-12-17T19:32:21.756939Z","shell.execute_reply":"2025-12-17T19:32:34.202280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Head(nn.Module):\n    def __init__(self, in_features = 1536, hidden_features = 1024, out_features = 3):\n        super().__init__()\n        self.layers = nn.Sequential(\n            nn.Linear(in_features, hidden_features),\n            nn.BatchNorm1d(hidden_features),\n            nn.ReLU(),\n            nn.Linear(hidden_features, hidden_features // 2),\n            nn.BatchNorm1d(hidden_features // 2),\n            nn.ReLU(),\n            nn.Linear(hidden_features // 2, hidden_features // 4),\n            nn.BatchNorm1d(hidden_features // 4),\n            nn.ReLU(),\n            nn.Linear(hidden_features // 4, hidden_features // 8),\n            nn.BatchNorm1d(hidden_features // 8),\n            nn.ReLU(),\n            nn.Linear(hidden_features // 8, out_features),\n            nn.ReLU()\n        )\n    def forward(self, x):\n        return self.layers(x)\nHead_model = Head(in_features = 1024).to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:34.205743Z","iopub.execute_input":"2025-12-17T19:32:34.206065Z","iopub.status.idle":"2025-12-17T19:32:34.229112Z","shell.execute_reply.started":"2025-12-17T19:32:34.206040Z","shell.execute_reply":"2025-12-17T19:32:34.228522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessed_images = []\nfor _, row in tqdm(df_final.iterrows(), total=len(df_final)):\n    image_path = os.path.join(MAIN_PATH, row[\"image_path\"])\n    try:\n        img = Image.open(image_path).convert(\"RGB\")\n    except Exception as e:\n        print(f\"Skipping bad image {image_path}: {e}\")\n        continue\n    img_tensor = preprocess(img).unsqueeze(0).to(device)\n    with torch.no_grad():\n        feature = model(img_tensor)\n\n    preprocessed_images.append(feature.cpu())\n    \npreprocessed_images = torch.cat(preprocessed_images, dim=0)\n\nprint(f\"Final Tensor Shape: {preprocessed_images.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:32:34.229834Z","iopub.execute_input":"2025-12-17T19:32:34.230062Z","iopub.status.idle":"2025-12-17T19:34:25.239941Z","shell.execute_reply.started":"2025-12-17T19:32:34.230029Z","shell.execute_reply":"2025-12-17T19:34:25.238991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = preprocessed_images.detach().cpu().numpy().squeeze()\ny = df_final.loc[:, [\"Dry_Clover_g_target\", \"Dry_Dead_g_target\", \"Dry_Green_g_target\"]]\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:25.241310Z","iopub.execute_input":"2025-12-17T19:34:25.241608Z","iopub.status.idle":"2025-12-17T19:34:25.249999Z","shell.execute_reply.started":"2025-12-17T19:34:25.241581Z","shell.execute_reply":"2025-12-17T19:34:25.249318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class custom_dataset(Dataset):\n    def __init__(self, features, labels):\n        self.features = features\n        self.labels = labels\n\n    def __len__(self):\n        return self.features.shape[0]\n\n    def __getitem__(self, idx):\n        return self.features[idx], torch.tensor(self.labels.iloc[idx, :].values, dtype = torch.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:25.251087Z","iopub.execute_input":"2025-12-17T19:34:25.251460Z","iopub.status.idle":"2025-12-17T19:34:25.266559Z","shell.execute_reply.started":"2025-12-17T19:34:25.251425Z","shell.execute_reply":"2025-12-17T19:34:25.265604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = custom_dataset(X, y)\ntrain_loader = DataLoader(train_dataset, batch_size = 16, shuffle = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:25.267846Z","iopub.execute_input":"2025-12-17T19:34:25.268263Z","iopub.status.idle":"2025-12-17T19:34:25.281558Z","shell.execute_reply.started":"2025-12-17T19:34:25.268224Z","shell.execute_reply":"2025-12-17T19:34:25.280393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = optim.SGD(Head_model.parameters(), lr = LEARNING_RATE, weight_decay = WEIGHT_DECAY)\ncriterion = nn.MSELoss()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:25.282787Z","iopub.execute_input":"2025-12-17T19:34:25.283231Z","iopub.status.idle":"2025-12-17T19:34:25.298159Z","shell.execute_reply.started":"2025-12-17T19:34:25.283198Z","shell.execute_reply":"2025-12-17T19:34:25.297070Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for epoch in range(EPOCHS):\n    Head_model.train()\n    tr_loss = 0\n    for batch_features, batch_labels in train_loader:    \n        batch_features, batch_labels = batch_features.to(device), batch_labels.to(device)    \n        outputs = Head_model(batch_features)\n\n        loss = criterion(outputs, batch_labels)\n        tr_loss += loss\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n    print(f\"Epoch : {epoch + 1}\\tTraining Loss : {tr_loss/len(train_loader)}\\t\", end = \"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:25.299414Z","iopub.execute_input":"2025-12-17T19:34:25.299663Z","iopub.status.idle":"2025-12-17T19:34:29.772725Z","shell.execute_reply.started":"2025-12-17T19:34:25.299642Z","shell.execute_reply":"2025-12-17T19:34:29.771988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data_2 = test_data.copy()\nfor i, _ in test_data.iterrows():\n    test_data_2.loc[i, \"sample_id\"] = test_data_2.loc[i, \"sample_id\"].split(\"_\")[0]\ncols = [c for c in test_data_2.columns if c not in [\"target_name\", \"target\"]]\nfinal_test_data = (\n    test_data_2[cols]\n      .drop_duplicates(subset=\"sample_id\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:29.773622Z","iopub.execute_input":"2025-12-17T19:34:29.773947Z","iopub.status.idle":"2025-12-17T19:34:29.781586Z","shell.execute_reply.started":"2025-12-17T19:34:29.773910Z","shell.execute_reply":"2025-12-17T19:34:29.780855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessed_test_images = []\nfor _, row in tqdm(final_test_data.iterrows(), total=len(final_test_data)):\n    image_path = os.path.join(MAIN_PATH, row[\"image_path\"])\n    try:\n        img = Image.open(image_path).convert(\"RGB\")\n    except Exception as e:\n        print(f\"Skipping bad image {image_path}: {e}\")\n        continue\n    img_tensor = preprocess(img).unsqueeze(0).to(device)\n    with torch.no_grad():\n        feature = model(img_tensor)\n\n    preprocessed_test_images.append(feature.cpu())\n    \npreprocessed_test_images = torch.cat(preprocessed_test_images, dim=0).detach().cpu().numpy().squeeze()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:29.782899Z","iopub.execute_input":"2025-12-17T19:34:29.783176Z","iopub.status.idle":"2025-12-17T19:34:30.211325Z","shell.execute_reply.started":"2025-12-17T19:34:29.783153Z","shell.execute_reply":"2025-12-17T19:34:30.210512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessed_test_images.shape[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:30.212390Z","iopub.execute_input":"2025-12-17T19:34:30.213266Z","iopub.status.idle":"2025-12-17T19:34:30.218096Z","shell.execute_reply.started":"2025-12-17T19:34:30.213239Z","shell.execute_reply":"2025-12-17T19:34:30.217306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if preprocessed_test_images.shape[0] == 1024:\n    preprocessed_test_images = preprocessed_test_images.reshape(1, 1024)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:30.219062Z","iopub.execute_input":"2025-12-17T19:34:30.219360Z","iopub.status.idle":"2025-12-17T19:34:30.229499Z","shell.execute_reply.started":"2025-12-17T19:34:30.219337Z","shell.execute_reply":"2025-12-17T19:34:30.228850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Head_model.eval()\noutputs = Head_model(torch.tensor(preprocessed_test_images, dtype = torch.float32).to(device))\nclover = outputs[:, 0]\ndead = outputs[:, 1]\ngreen = outputs[:, 2]\ntotal = clover + dead + green\ngdm = clover + green\nprint(f\"Targets Predicted !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:30.230540Z","iopub.execute_input":"2025-12-17T19:34:30.230772Z","iopub.status.idle":"2025-12-17T19:34:30.264584Z","shell.execute_reply.started":"2025-12-17T19:34:30.230751Z","shell.execute_reply":"2025-12-17T19:34:30.263758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"c, d, g, t, gd = 0, 0, 0, 0, 0\nsubmission = []\nfor _, row in test_data.iterrows():\n    if \"Clover\" in row[\"sample_id\"]:\n        value = clover[c].item()\n        c += 1\n    elif \"Dead\" in row[\"sample_id\"]:\n        value = dead[d].item()\n        d += 1\n    elif \"Green\" in row[\"sample_id\"]:\n        value = green[g].item()\n        g += 1\n    elif \"Total\" in row[\"sample_id\"]:\n        value = total[t].item()\n        t += 1\n    else:\n        value = gdm[gd].item()\n        gd += 1\n    submission_entry = {\n        \"sample_id\" : row[\"sample_id\"],\n        \"target\" : value\n    }\n    submission.append(submission_entry)\nsubmission = pd.DataFrame(submission)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:30.265609Z","iopub.execute_input":"2025-12-17T19:34:30.265880Z","iopub.status.idle":"2025-12-17T19:34:30.273747Z","shell.execute_reply.started":"2025-12-17T19:34:30.265852Z","shell.execute_reply":"2025-12-17T19:34:30.273097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:30.274804Z","iopub.execute_input":"2025-12-17T19:34:30.275119Z","iopub.status.idle":"2025-12-17T19:34:30.288123Z","shell.execute_reply.started":"2025-12-17T19:34:30.275086Z","shell.execute_reply":"2025-12-17T19:34:30.287344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"/kaggle/working/submission.csv\", index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:30.289299Z","iopub.execute_input":"2025-12-17T19:34:30.289558Z","iopub.status.idle":"2025-12-17T19:34:30.307132Z","shell.execute_reply.started":"2025-12-17T19:34:30.289523Z","shell.execute_reply":"2025-12-17T19:34:30.306181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"The entire notebook runs successfully, if error happens, I don't just know why.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T19:34:30.310227Z","iopub.execute_input":"2025-12-17T19:34:30.310483Z","iopub.status.idle":"2025-12-17T19:34:30.318278Z","shell.execute_reply.started":"2025-12-17T19:34:30.310453Z","shell.execute_reply":"2025-12-17T19:34:30.317408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}