{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":285516575,"sourceType":"kernelVersion"},{"sourceId":4533,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3325,"modelId":986},{"sourceId":4537,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3329,"modelId":986}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The notebook below preprocesses the images using the DINOv2 Small model, and the CLS of the preprocessed images is then fed to three different LGBM Regressors, each for dry_clover_g, dry_dead_g and dry_green_g.\n\nIn this case the hyperparameters were chosen using optuna (code commented) to set a baseline for submissions.\n\nConclusion of the current notebook:- (for clover, dead and green respectively)\n\nMSE (Clover): 136.76830688231178\n\nR2 Score (Clover): 0.23254926882145288\n\nMSE (Dead): 136.60008482075835\n\nR2 Score (Dead): 0.055618106953414626\n\nMSE (Green): 465.3614596562113\n\nR2 Score (Green): 0.010596143683867298\n\ntotal = clover + dead + green\n\nGDM = clover + green","metadata":{}},{"cell_type":"code","source":"!pip install --no-index --find-links /kaggle/input/transformers-wheel torch tqdm torchvision optuna-integration[xgboost] keras keras-cv transformers==4.38.2.0 protobuf==3.20.3","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:18:45.046119Z","iopub.execute_input":"2025-12-15T05:18:45.046373Z","iopub.status.idle":"2025-12-15T05:21:49.414015Z","shell.execute_reply.started":"2025-12-15T05:18:45.046354Z","shell.execute_reply":"2025-12-15T05:21:49.413043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport optuna\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score\nimport torch\nimport torchvision.transforms as T\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom transformers import AutoImageProcessor, AutoModel\nfrom tqdm import tqdm\nimport cv2\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nMAIN_PATH = \"/kaggle/input/csiro-biomass\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:21:49.415557Z","iopub.execute_input":"2025-12-15T05:21:49.415909Z","iopub.status.idle":"2025-12-15T05:22:13.759363Z","shell.execute_reply.started":"2025-12-15T05:21:49.415881Z","shell.execute_reply":"2025-12-15T05:22:13.758741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:22:13.760029Z","iopub.execute_input":"2025-12-15T05:22:13.760637Z","iopub.status.idle":"2025-12-15T05:22:13.794540Z","shell.execute_reply.started":"2025-12-15T05:22:13.760616Z","shell.execute_reply":"2025-12-15T05:22:13.793693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, _ in train_data.iterrows():\n    train_data.loc[i, \"sample_id\"] = train_data.loc[i, \"sample_id\"].split(\"_\")[0]\ncols = [c for c in train_data.columns if c not in [\"target_name\", \"target\"]]\ndf_static = (\n    train_data[cols]\n      .drop_duplicates(subset=\"sample_id\")\n)\n\ndf_pivot = (\n    train_data.pivot(index=\"sample_id\", columns=\"target_name\", values=\"target\")\n      .rename(columns=lambda x: f\"{x}_target\")\n      .reset_index()\n)\n\ndf_final = df_static.merge(df_pivot, on=\"sample_id\", how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:22:13.796226Z","iopub.execute_input":"2025-12-15T05:22:13.797108Z","iopub.status.idle":"2025-12-15T05:22:14.234607Z","shell.execute_reply.started":"2025-12-15T05:22:13.797081Z","shell.execute_reply":"2025-12-15T05:22:14.233970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_final","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:22:14.235286Z","iopub.execute_input":"2025-12-15T05:22:14.235541Z","iopub.status.idle":"2025-12-15T05:22:14.260932Z","shell.execute_reply.started":"2025-12-15T05:22:14.235495Z","shell.execute_reply":"2025-12-15T05:22:14.260368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# PreProcessing","metadata":{}},{"cell_type":"code","source":"model_path = \"/kaggle/input/dinov2/pytorch/giant/1\"\nprocessor = AutoImageProcessor.from_pretrained(model_path)\nmodel = AutoModel.from_pretrained(model_path).to(device)\nfor param in model.parameters():\n    param.requires_grad = False\nmodel.eval()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:22:14.261642Z","iopub.execute_input":"2025-12-15T05:22:14.261879Z","iopub.status.idle":"2025-12-15T05:22:37.180781Z","shell.execute_reply.started":"2025-12-15T05:22:14.261857Z","shell.execute_reply":"2025-12-15T05:22:37.180044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessed_images = []\nfor _, row in tqdm(df_final.iterrows(), total=len(df_final)):\n    image_path = os.path.join(MAIN_PATH, row[\"image_path\"])\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    transform = T.Compose([\n        T.ToTensor()\n    ])\n    img = transform(img)\n    img = img.to(device)\n    \n    inputs = processor(images=img, return_tensors=\"pt\")\n    inputs = inputs.to(device)\n    outputs = model(**inputs)\n    features = outputs.last_hidden_state.squeeze(0)\n    preprocessed_images.append(features[0, :])\npreprocessed_images = torch.stack(preprocessed_images, dim=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:22:37.181618Z","iopub.execute_input":"2025-12-15T05:22:37.181897Z","iopub.status.idle":"2025-12-15T05:25:14.305201Z","shell.execute_reply.started":"2025-12-15T05:22:37.181872Z","shell.execute_reply":"2025-12-15T05:25:14.304568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessed_images.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:07.303321Z","iopub.execute_input":"2025-12-15T05:17:07.304298Z","iopub.status.idle":"2025-12-15T05:17:07.322612Z","shell.execute_reply.started":"2025-12-15T05:17:07.304236Z","shell.execute_reply":"2025-12-15T05:17:07.321420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = preprocessed_images.detach().cpu().numpy()\ny_c = df_final.loc[:, \"Dry_Clover_g_target\"]\ny_d = df_final.loc[:, \"Dry_Dead_g_target\"]\ny_g = df_final.loc[:, \"Dry_Green_g_target\"]\nX_train_c, X_test_c, y_train_c, y_test_c = train_test_split(X, y_c, test_size = 0.2, random_state = 42)\nX_train_d, X_test_d, y_train_d, y_test_d = train_test_split(X, y_d, test_size = 0.2, random_state = 42)\nX_train_g, X_test_g, y_train_g, y_test_g = train_test_split(X, y_g, test_size = 0.2, random_state = 42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:13.494513Z","iopub.execute_input":"2025-12-15T05:17:13.494874Z","iopub.status.idle":"2025-12-15T05:17:13.536721Z","shell.execute_reply.started":"2025-12-15T05:17:13.494852Z","shell.execute_reply":"2025-12-15T05:17:13.535788Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training of LGBM Models","metadata":{}},{"cell_type":"code","source":"# def objective(trial):\n#     X_data = X_train_c.values if isinstance(X_train_c, (pd.DataFrame, pd.Series)) else X_train_c\n#     y_data = y_train_c.values if isinstance(y_train_c, (pd.DataFrame, pd.Series)) else y_train_c\n    \n#     params = {\n#         \"task_type\": \"GPU\",\n#         \"devices\": \"0\",\n#         \"loss_function\": \"RMSE\", \n#         \"eval_metric\": \"RMSE\",\n#         \"iterations\": trial.suggest_int(\"iterations\", 100, 1000),\n#         \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-3, 0.3, log=True),\n#         \"depth\": trial.suggest_int(\"depth\", 4, 10),\n#         \"l2_leaf_reg\": trial.suggest_float(\"l2_leaf_reg\", 1e-8, 100.0, log=True),\n#         \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 1, 100),\n#         \"border_count\": 128,\n#         \"verbose\": 0,\n#         \"early_stopping_rounds\": 50\n#     }\n\n#     kf = KFold(n_splits=5, shuffle=True, random_state=42)\n#     mse_scores = []\n\n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X_data, y_data)):\n#         X_train, X_val = X_data[train_idx], X_data[val_idx]\n#         y_train, y_val = y_data[train_idx], y_data[val_idx]\n\n#         model = CatBoostRegressor(**params)\n        \n#         model.fit(\n#             X_train, \n#             y_train, \n#             eval_set=[(X_val, y_val)], \n#             use_best_model=True\n#         )\n\n#         preds = model.predict(X_val)\n#         mse = mean_squared_error(y_val, preds)\n#         mse_scores.append(mse)\n\n#     return np.mean(mse_scores)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T18:40:34.481348Z","iopub.execute_input":"2025-12-11T18:40:34.482026Z","iopub.status.idle":"2025-12-11T18:40:34.488299Z","shell.execute_reply.started":"2025-12-11T18:40:34.481999Z","shell.execute_reply":"2025-12-11T18:40:34.487673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# study = optuna.create_study(direction=\"minimize\")\n# study.optimize(objective, n_trials=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T18:40:35.173734Z","iopub.execute_input":"2025-12-11T18:40:35.174200Z","execution_failed":"2025-12-11T18:41:24.488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(study.best_params, end = \"\\n\\n\")\n# print(study.best_value)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-11T18:41:24.489Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Optuna Results","metadata":{}},{"cell_type":"markdown","source":"#### Clover\n\n\n\n\n\n#### Dead\n\n\n\n\n\n#### Green\n\n\n\n","metadata":{}},{"cell_type":"code","source":"params_c = {'n_estimators': 553, 'max_depth': 9, 'num_leaves': 88, 'learning_rate': 0.0201031389924404, 'reg_alpha': 3.9527465089943645e-05, 'reg_lambda': 1.1244084545207968e-08, 'subsample': 0.5162503045472089, 'colsample_bytree': 0.7707363689715299, 'objective': 'regression', 'metric': 'mse', 'verbosity': -1, 'n_jobs': -1, 'random_state': 42}\nparams_d = {'n_estimators': 411, 'max_depth': 11, 'num_leaves': 90, 'learning_rate': 0.0013659846022125244, 'reg_alpha': 0.0262605931287003, 'reg_lambda': 0.00012303252661984061, 'subsample': 0.8827687063222698, 'colsample_bytree': 0.5119309353652499, 'objective': 'regression', 'metric': 'mse', 'verbosity': -1, 'n_jobs': -1, 'random_state': 42}\nparams_g = {'n_estimators': 1223, 'max_depth': 10, 'num_leaves': 48, 'learning_rate': 0.005974450111604017, 'reg_alpha': 0.398308172872663, 'reg_lambda': 2.585136566238415e-07, 'subsample': 0.8590006376333748, 'colsample_bytree': 0.5365284916847206, 'objective': 'regression', 'metric': 'mse', 'verbosity': -1, 'n_jobs': -1, 'random_state': 42}\n\nlgb_clover = lgb.LGBMRegressor(**params_c)\nlgb_dead = lgb.LGBMRegressor(**params_d)\nlgb_green = lgb.LGBMRegressor(**params_g)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:19.643088Z","iopub.execute_input":"2025-12-15T05:17:19.643411Z","iopub.status.idle":"2025-12-15T05:17:19.656926Z","shell.execute_reply.started":"2025-12-15T05:17:19.643389Z","shell.execute_reply":"2025-12-15T05:17:19.655910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_clover.fit(X, y_c)\nlgb_dead.fit(X, y_d)\nlgb_green.fit(X, y_g)\nprint(f\"All Models Trained !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:23.103393Z","iopub.execute_input":"2025-12-15T05:17:23.103748Z","iopub.status.idle":"2025-12-15T05:17:49.043510Z","shell.execute_reply.started":"2025-12-15T05:17:23.103720Z","shell.execute_reply":"2025-12-15T05:17:49.040738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# pred_c = lgb_clover.predict(X_test_c)\n# pred_d = lgb_dead.predict(X_test_d)\n# pred_g = lgb_green.predict(X_test_g)\n# print(f\"Predictions Calculated !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T17:46:03.177829Z","iopub.execute_input":"2025-12-11T17:46:03.178171Z","iopub.status.idle":"2025-12-11T17:46:03.182434Z","shell.execute_reply.started":"2025-12-11T17:46:03.178140Z","shell.execute_reply":"2025-12-11T17:46:03.181583Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def calculate_r2(preds, targets):\n#     target_mean = np.mean(targets, axis=0)\n#     ss_tot = np.sum((targets - target_mean) ** 2, axis=0)\n#     ss_res = np.sum((targets - preds) ** 2, axis=0)\n#     r2 = 1 - (ss_res / (ss_tot + 1e-8))\n#     return np.mean(r2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T17:46:03.183390Z","iopub.execute_input":"2025-12-11T17:46:03.183692Z","iopub.status.idle":"2025-12-11T17:46:03.200814Z","shell.execute_reply.started":"2025-12-11T17:46:03.183672Z","shell.execute_reply":"2025-12-11T17:46:03.199988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# mse_c = mean_squared_error(pred_c, y_test_c)\n# r2_c = calculate_r2(pred_c, y_test_c)\n# mse_d = mean_squared_error(pred_d, y_test_d)\n# r2_d = calculate_r2(pred_d, y_test_d)\n# mse_g = mean_squared_error(pred_g, y_test_g)\n# r2_g = calculate_r2(pred_g, y_test_g)\n\n# print(f\"MSE (Clover): {mse_c}\\nR2 Score (Clover): {r2_c}\")\n# print(f\"MSE (Dead): {mse_d}\\nR2 Score (Dead): {r2_d}\")\n# print(f\"MSE (Green): {mse_g}\\nR2 Score (Green): {r2_g}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T17:46:03.201711Z","iopub.execute_input":"2025-12-11T17:46:03.202002Z","iopub.status.idle":"2025-12-11T17:46:03.217965Z","shell.execute_reply.started":"2025-12-11T17:46:03.201977Z","shell.execute_reply":"2025-12-11T17:46:03.216975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data_2 = test_data.copy()\nfor i, _ in test_data.iterrows():\n    test_data_2.loc[i, \"sample_id\"] = test_data_2.loc[i, \"sample_id\"].split(\"_\")[0]\ncols = [c for c in test_data_2.columns if c not in [\"target_name\", \"target\"]]\nfinal_test_data = (\n    test_data_2[cols]\n      .drop_duplicates(subset=\"sample_id\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:49.044974Z","iopub.execute_input":"2025-12-15T05:17:49.046977Z","iopub.status.idle":"2025-12-15T05:17:49.071786Z","shell.execute_reply.started":"2025-12-15T05:17:49.046943Z","shell.execute_reply":"2025-12-15T05:17:49.070443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessed_test_images = []\nfor _, row in tqdm(final_test_data.iterrows(), total=len(final_test_data)):\n    image_path = os.path.join(MAIN_PATH, row[\"image_path\"])\n    img = cv2.imread(image_path)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    transform = T.Compose([\n        T.ToTensor()\n    ])\n    img = transform(img)\n    img = img.to(device)\n    \n    inputs = processor(images=img, return_tensors=\"pt\")\n    inputs = inputs.to(device)\n    outputs = model(**inputs)\n    features = outputs.last_hidden_state.squeeze(0)\n    preprocessed_test_images.append(features[0, :])\npreprocessed_test_images = torch.stack(preprocessed_test_images, dim=0).detach().cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:49.073389Z","iopub.execute_input":"2025-12-15T05:17:49.073770Z","iopub.status.idle":"2025-12-15T05:17:55.667166Z","shell.execute_reply.started":"2025-12-15T05:17:49.073737Z","shell.execute_reply":"2025-12-15T05:17:55.666007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessed_test_images.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:55.669288Z","iopub.execute_input":"2025-12-15T05:17:55.669643Z","iopub.status.idle":"2025-12-15T05:17:55.675908Z","shell.execute_reply.started":"2025-12-15T05:17:55.669620Z","shell.execute_reply":"2025-12-15T05:17:55.675043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"clover = lgb_clover.predict(preprocessed_test_images)\ndead = lgb_dead.predict(preprocessed_test_images)\ngreen = lgb_green.predict(preprocessed_test_images)\ntotal = clover + dead + green\ngdm = clover + green\nprint(f\"Targets Predicted !\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:55.677450Z","iopub.execute_input":"2025-12-15T05:17:55.677803Z","iopub.status.idle":"2025-12-15T05:17:55.708472Z","shell.execute_reply.started":"2025-12-15T05:17:55.677781Z","shell.execute_reply":"2025-12-15T05:17:55.706579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"c, d, g, t, gd = 0, 0, 0, 0, 0\nsubmission = []\nfor _, row in test_data.iterrows():\n    if \"Clover\" in row[\"sample_id\"]:\n        value = clover[c]\n        c += 1\n    elif \"Dead\" in row[\"sample_id\"]:\n        value = dead[d]\n        d += 1\n    elif \"Green\" in row[\"sample_id\"]:\n        value = green[g]\n        g += 1\n    elif \"Total\" in row[\"sample_id\"]:\n        value = total[t]\n        t += 1\n    else:\n        value = gdm[gd]\n        gd += 1\n    submission_entry = {\n        \"sample_id\" : row[\"sample_id\"],\n        \"target\" : value\n    }\n    submission.append(submission_entry)\nsubmission = pd.DataFrame(submission)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:55.709752Z","iopub.execute_input":"2025-12-15T05:17:55.710081Z","iopub.status.idle":"2025-12-15T05:17:55.746237Z","shell.execute_reply.started":"2025-12-15T05:17:55.710057Z","shell.execute_reply":"2025-12-15T05:17:55.745065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:55.747500Z","iopub.execute_input":"2025-12-15T05:17:55.748165Z","iopub.status.idle":"2025-12-15T05:17:55.780241Z","shell.execute_reply.started":"2025-12-15T05:17:55.748118Z","shell.execute_reply":"2025-12-15T05:17:55.779015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"/kaggle/working/submission.csv\", index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:55.781385Z","iopub.execute_input":"2025-12-15T05:17:55.781753Z","iopub.status.idle":"2025-12-15T05:17:55.827296Z","shell.execute_reply.started":"2025-12-15T05:17:55.781719Z","shell.execute_reply":"2025-12-15T05:17:55.826092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"The entire notebook runs successfully, if error happens, I don't just know why.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T05:17:55.828350Z","iopub.execute_input":"2025-12-15T05:17:55.828594Z","iopub.status.idle":"2025-12-15T05:17:55.834063Z","shell.execute_reply.started":"2025-12-15T05:17:55.828577Z","shell.execute_reply":"2025-12-15T05:17:55.833065Z"}},"outputs":[],"execution_count":null},{"cell_type":"raw","source":"","metadata":{}}]}