{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":4535,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3327,"modelId":986}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, torch, warnings, numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import r2_score\nfrom transformers import AutoModel\nimport torchvision.transforms as transforms\n\n# Settings\nwarnings.filterwarnings(\"ignore\")\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\nCFG = {\n    \"base_path\": Path(\"/kaggle/input/csiro-biomass\"),\n    \"model_path\": \"/kaggle/input/dinov2/pytorch/large/1\",\n    \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n    \"n_splits\": 5,\n    \"seed\": 42\n}\n\nTARGETS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\nCOMP_WEIGHTS = np.array([0.1, 0.1, 0.1, 0.2, 0.5]) \nMEAN, STD = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T23:00:44.666425Z","iopub.execute_input":"2026-01-21T23:00:44.666701Z","iopub.status.idle":"2026-01-21T23:01:05.641918Z","shell.execute_reply.started":"2026-01-21T23:00:44.666667Z","shell.execute_reply":"2026-01-21T23:01:05.641237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_dense_features(df, model):\n    tta_transforms = [\n        transforms.Compose([transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(MEAN, STD)]),\n        transforms.Compose([transforms.RandomHorizontalFlip(p=1.0), transforms.ToTensor(), transforms.Resize((224, 224)), transforms.Normalize(MEAN, STD)])\n    ]\n    unique_paths = df['image_path'].unique()\n    all_feats = []\n    \n    with torch.no_grad():\n        for path in tqdm(unique_paths, desc=\"GPU Feature Extraction\"):\n            img = Image.open(CFG[\"base_path\"] / path).convert(\"RGB\")\n            tta_results = [model(aug(img).unsqueeze(0).to(CFG[\"device\"])).last_hidden_state[:, 1:, :].mean(dim=1).cpu().numpy() for aug in tta_transforms]\n            all_feats.append(np.mean(tta_results, axis=0))\n            \n    feat_matrix = np.vstack(all_feats)\n    return pd.DataFrame(feat_matrix, columns=[f\"feat_{i}\" for i in range(feat_matrix.shape[1])]), unique_paths","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T23:01:05.643415Z","iopub.execute_input":"2026-01-21T23:01:05.644061Z","iopub.status.idle":"2026-01-21T23:01:05.650786Z","shell.execute_reply.started":"2026-01-21T23:01:05.644032Z","shell.execute_reply":"2026-01-21T23:01:05.650138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load ViT Model\nvit_model = AutoModel.from_pretrained(os.path.abspath(CFG[\"model_path\"]), local_files_only=True, trust_remote_code=True).to(CFG[\"device\"]).eval()\n\n# Train Data\ntrain_df = pd.read_csv(CFG[\"base_path\"] / \"train.csv\")\ntrain_p = train_df.pivot_table(index=\"image_path\", columns=\"target_name\", values=\"target\").reset_index()\nX_train_raw, _ = extract_dense_features(train_p, vit_model)\nY_train_log = np.log1p(train_p[TARGETS].values)\n\n# Test Data\ntest_df = pd.read_csv(CFG[\"base_path\"] / \"test.csv\")\ntest_uniq = pd.DataFrame({'image_path': test_df['image_path'].unique()})\nX_test_raw, test_ids = extract_dense_features(test_uniq, vit_model)\n\n# Scaling\nscaler = StandardScaler()\nX_train = pd.DataFrame(scaler.fit_transform(X_train_raw), columns=X_train_raw.columns)\nX_test = pd.DataFrame(scaler.transform(X_test_raw), columns=X_test_raw.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-21T23:01:05.651772Z","iopub.execute_input":"2026-01-21T23:01:05.652219Z","iopub.status.idle":"2026-01-21T23:02:55.420712Z","shell.execute_reply.started":"2026-01-21T23:01:05.652190Z","shell.execute_reply":"2026-01-21T23:02:55.419789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import r2_score\nimport lightgbm as lgb\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.model_selection import KFold\n\n# Official competition weights\nCOMP_WEIGHTS = np.array([0.1, 0.1, 0.1, 0.2, 0.5]) \n\ndef weighted_r2(y_true, y_pred):\n    # Using the same logic as your CatBoost implementation\n    r2_scores = r2_score(y_true, y_pred, multioutput='raw_values')\n    return np.sum(r2_scores * COMP_WEIGHTS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-22T00:04:25.908398Z","iopub.execute_input":"2026-01-22T00:04:25.908986Z","iopub.status.idle":"2026-01-22T00:04:25.913719Z","shell.execute_reply.started":"2026-01-22T00:04:25.908958Z","shell.execute_reply":"2026-01-22T00:04:25.912963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nkf = KFold(n_splits=CFG[\"n_splits\"], shuffle=True, random_state=CFG[\"seed\"])\ncv_scores = []\n\n# Baseline LightGBM parameters\nlgb_params = {\n    \"n_estimators\": 500,\n    \"learning_rate\": 0.05,\n    \"num_leaves\": 31,\n    \"feature_fraction\": 0.8,\n    \"bagging_fraction\": 0.7,\n    \"bagging_freq\": 5,\n    \"device\": \"gpu\" if torch.cuda.is_available() else \"cpu\",\n    \"random_state\": CFG[\"seed\"],\n    \"verbosity\": -1\n}\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train, Y_train_log)):\n    X_tr, y_tr = X_train.iloc[train_idx], Y_train_log[train_idx]\n    X_vl, y_vl = X_train.iloc[val_idx], Y_train_log[val_idx]\n    \n    # Wrap LGBM in MultiOutputRegressor\n    base_model = lgb.LGBMRegressor(**lgb_params)\n    model = MultiOutputRegressor(base_model)\n    \n    model.fit(X_tr, y_tr)\n    \n    preds = model.predict(X_vl)\n    score = weighted_r2(y_vl, preds)\n    cv_scores.append(score)\n    print(f\"Fold {fold+1} Weighted R^2: {score:.5f}\")\n\nprint(f\"LGBM Baseline Mean Weighted R^2: {np.mean(cv_scores):.5f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-22T00:05:14.156380Z","iopub.execute_input":"2026-01-22T00:05:14.156699Z","iopub.status.idle":"2026-01-22T00:06:33.296438Z","shell.execute_reply.started":"2026-01-22T00:05:14.156673Z","shell.execute_reply":"2026-01-22T00:06:33.295738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\nimport logging\n\n# Set logging to see standard optuna output\noptuna.logging.set_verbosity(optuna.logging.INFO)\n\ndef objective(trial):\n    params = {\n        \"n_estimators\": 1000,\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-3, 0.1, log=True),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 150),\n        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.5, 1.0),\n        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.5, 1.0),\n        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n        \"device\": \"gpu\" if torch.cuda.is_available() else \"cpu\",\n        \"verbosity\": -1\n    }\n    \n    # Selection of fold (using next() to stay consistent with your logic)\n    train_idx, val_idx = next(kf.split(X_train, Y_train_log))\n    X_tr, y_tr = X_train.iloc[train_idx], Y_train_log[train_idx]\n    X_vl, y_vl = X_train.iloc[val_idx], Y_train_log[val_idx]\n    \n    model = MultiOutputRegressor(lgb.LGBMRegressor(**params))\n    model.fit(X_tr, y_tr)\n    \n    preds = model.predict(X_vl)\n    return weighted_r2(y_vl, preds)\n\n# Define custom print logic\ndef callback(study, trial):\n    if study.best_trial.number == trial.number:\n        print(f\"New Best Trial: {trial.number} | Value: {trial.value:.5f}\")\n\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=20, callbacks=[callback])\n\nprint(f\"Best Value: {study.best_value}\")\nprint(f\"Best Params: {study.best_params}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-22T00:06:33.297724Z","iopub.execute_input":"2026-01-22T00:06:33.297960Z","iopub.status.idle":"2026-01-22T00:20:29.586274Z","shell.execute_reply.started":"2026-01-22T00:06:33.297939Z","shell.execute_reply":"2026-01-22T00:20:29.585469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract features from the test images first\n# X_test, test_ids = extract_dense_features(test_df, dino_model)\n\n# Generate predictions (on the log-transformed targets)\npreds_log = model.predict(X_test)\n\n# Inverse transform to get actual biomass values (and ensure no negative values)\nfinal_preds = np.maximum(np.expm1(preds_log), 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-22T00:20:29.587408Z","iopub.execute_input":"2026-01-22T00:20:29.587632Z","iopub.status.idle":"2026-01-22T00:20:29.606348Z","shell.execute_reply.started":"2026-01-22T00:20:29.587611Z","shell.execute_reply":"2026-01-22T00:20:29.605540Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\n\nTARGETS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\nfinal_rows = []\n\n# Loop through each test sample and each of the 5 targets\nfor i, path in enumerate(test_ids):\n    image_id = Path(path).stem # Extract filename without extension\n    for j, target_name in enumerate(TARGETS):\n        final_rows.append({\n            \"sample_id\": f\"{image_id}__{target_name}\", \n            \"target\": final_preds[i, j]\n        })\n\n# Create the final DataFrame and save to CSV\nsubmission_df = pd.DataFrame(final_rows)\nsubmission_df.to_csv(\"submission.csv\", index=False)\n\nprint(f\"Submission file saved with {len(submission_df)} rows.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-22T00:20:29.607519Z","iopub.execute_input":"2026-01-22T00:20:29.607847Z","iopub.status.idle":"2026-01-22T00:20:29.621393Z","shell.execute_reply.started":"2026-01-22T00:20:29.607813Z","shell.execute_reply":"2026-01-22T00:20:29.620578Z"}},"outputs":[],"execution_count":null}]}