{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31259,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CSIRO Image2Biomass - Baseline v4\n## Color Features + LightGBM","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nimport os\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Configuration\nif os.path.exists('/kaggle/input/csiro-biomass'):\n    DATA_PATH = Path('/kaggle/input/csiro-biomass')\n    OUTPUT_PATH = Path('/kaggle/working')\n    print(\"Running on Kaggle\")\nelse:\n    DATA_PATH = Path('data')\n    OUTPUT_PATH = Path('.')\n    print(\"Running locally\")\n\nTARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\nprint(f\"Data path: {DATA_PATH}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:13:36.254181Z","iopub.execute_input":"2026-01-17T02:13:36.254506Z","iopub.status.idle":"2026-01-17T02:13:46.396302Z","shell.execute_reply.started":"2026-01-17T02:13:36.254468Z","shell.execute_reply":"2026-01-17T02:13:46.395362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\ntrain_df = pd.read_csv(DATA_PATH / 'train.csv')\ntest_long = pd.read_csv(DATA_PATH / 'test.csv')  # Keep original for sample_id\n\nprint(f\"Train shape: {train_df.shape}\")\nprint(f\"Test shape: {test_long.shape}\")\nprint(f\"Train columns: {train_df.columns.tolist()}\")\nprint(f\"Test columns: {test_long.columns.tolist()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:13:46.398053Z","iopub.execute_input":"2026-01-17T02:13:46.398727Z","iopub.status.idle":"2026-01-17T02:13:46.430826Z","shell.execute_reply.started":"2026-01-17T02:13:46.398697Z","shell.execute_reply":"2026-01-17T02:13:46.429904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert train to wide format (one row per image)\ntrain_wide = pd.pivot_table(\n    train_df,\n    values='target',\n    index=['image_path', 'Pre_GSHH_NDVI', 'Height_Ave_cm'],\n    columns='target_name'\n).reset_index()\n\n# Convert test to wide format (one row per image)\ntest_wide = test_long.copy()\ntest_wide['target'] = 0\ntest_wide = pd.pivot_table(\n    test_wide,\n    values='target',\n    index='image_path',\n    columns='target_name',\n    aggfunc='mean'\n).reset_index()\n\nprint(f\"Train wide shape: {train_wide.shape}\")\nprint(f\"Test wide shape: {test_wide.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:13:46.432101Z","iopub.execute_input":"2026-01-17T02:13:46.432478Z","iopub.status.idle":"2026-01-17T02:13:46.492536Z","shell.execute_reply.started":"2026-01-17T02:13:46.432450Z","shell.execute_reply":"2026-01-17T02:13:46.491661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_default_features():\n    \"\"\"Return default features when image cannot be loaded.\"\"\"\n    return {\n        # Green detection (3)\n        'green_pixel_ratio': 0.0,\n        'dark_green_ratio': 0.0,\n        'light_green_ratio': 0.0,\n        # Yellow/Dead detection (3)\n        'yellow_pixel_ratio': 0.0,\n        'brown_pixel_ratio': 0.0,\n        'dead_pixel_ratio': 0.0,\n        # Clover detection (1)\n        'clover_proxy_ratio': 0.0,\n        # Vegetation/Soil (2)\n        'vegetation_ratio': 0.0,\n        'soil_ratio': 0.0,\n        # Excess green index (3)\n        'excess_green_mean': 0.0,\n        'excess_green_std': 0.0,\n        'excess_green_positive_ratio': 0.0,\n        # Normalized green (2)\n        'norm_green_mean': 0.33,\n        'norm_green_std': 0.0,\n        # Green-Red index (2)\n        'green_red_idx_mean': 0.0,\n        'green_red_idx_std': 0.0,\n        # RGB statistics (6)\n        'r_mean': 128.0, 'g_mean': 128.0, 'b_mean': 128.0,\n        'r_std': 0.0, 'g_std': 0.0, 'b_std': 0.0,\n        # HSV statistics (6)\n        'h_mean': 0.0, 's_mean': 0.0, 'v_mean': 128.0,\n        'h_std': 0.0, 's_std': 0.0, 'v_std': 0.0,\n    }\n\ndef extract_color_features(image_path):\n    \"\"\"Extract 28 color-based features from image.\"\"\"\n    try:\n        img = cv2.imread(str(image_path))\n        if img is None:\n            return get_default_features()\n    except:\n        return get_default_features()\n    \n    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n    \n    h, s, v = cv2.split(img_hsv)\n    r, g, b = cv2.split(img_rgb)\n    \n    total_pixels = img.shape[0] * img.shape[1]\n    features = {}\n    \n    # === GREEN DETECTION (3 features) ===\n    green_mask = ((h >= 35) & (h <= 85) & (s > 40) & (v > 40))\n    features['green_pixel_ratio'] = np.sum(green_mask) / total_pixels\n    \n    dark_green_mask = ((h >= 35) & (h <= 85) & (s > 80) & (v > 40) & (v < 200))\n    features['dark_green_ratio'] = np.sum(dark_green_mask) / total_pixels\n    \n    light_green_mask = ((h >= 35) & (h <= 85) & (s > 30) & (s <= 80) & (v > 100))\n    features['light_green_ratio'] = np.sum(light_green_mask) / total_pixels\n    \n    # === YELLOW/DEAD DETECTION (3 features) ===\n    yellow_mask = ((h >= 15) & (h <= 35) & (s > 30) & (v > 50))\n    features['yellow_pixel_ratio'] = np.sum(yellow_mask) / total_pixels\n    \n    brown_mask = ((h >= 10) & (h <= 30) & (s > 20) & (s < 100) & (v > 40) & (v < 180))\n    features['brown_pixel_ratio'] = np.sum(brown_mask) / total_pixels\n    \n    dead_mask = ((s < 50) & (v > 80) & (v < 200))\n    features['dead_pixel_ratio'] = np.sum(dead_mask) / total_pixels\n    \n    # === CLOVER DETECTION (1 feature) ===\n    clover_mask = ((h >= 40) & (h <= 80) & (s > 50) & (v > 30) & (v < 150))\n    features['clover_proxy_ratio'] = np.sum(clover_mask) / total_pixels\n    \n    # === VEGETATION/SOIL (2 features) ===\n    veg_mask = ((h >= 20) & (h <= 85) & (s > 30) & (v > 40))\n    features['vegetation_ratio'] = np.sum(veg_mask) / total_pixels\n    \n    soil_mask = ((s < 40) & (v > 60) & (v < 180))\n    features['soil_ratio'] = np.sum(soil_mask) / total_pixels\n    \n    # === RGB-BASED INDICES (7 features) ===\n    g_float = g.astype(np.float32)\n    r_float = r.astype(np.float32)\n    b_float = b.astype(np.float32)\n    \n    # Excess green index\n    excess_green = 2 * g_float - r_float - b_float\n    features['excess_green_mean'] = float(np.mean(excess_green))\n    features['excess_green_std'] = float(np.std(excess_green))\n    features['excess_green_positive_ratio'] = np.sum(excess_green > 0) / total_pixels\n    \n    # Normalized green\n    rgb_sum = r_float + g_float + b_float + 1e-6\n    norm_green = g_float / rgb_sum\n    features['norm_green_mean'] = float(np.mean(norm_green))\n    features['norm_green_std'] = float(np.std(norm_green))\n    \n    # Green-Red index\n    gr_sum = g_float + r_float + 1e-6\n    green_red_idx = (g_float - r_float) / gr_sum\n    features['green_red_idx_mean'] = float(np.mean(green_red_idx))\n    features['green_red_idx_std'] = float(np.std(green_red_idx))\n    \n    # === RGB STATISTICS (6 features) ===\n    features['r_mean'] = float(np.mean(r))\n    features['g_mean'] = float(np.mean(g))\n    features['b_mean'] = float(np.mean(b))\n    features['r_std'] = float(np.std(r))\n    features['g_std'] = float(np.std(g))\n    features['b_std'] = float(np.std(b))\n    \n    # === HSV STATISTICS (6 features) ===\n    features['h_mean'] = float(np.mean(h))\n    features['s_mean'] = float(np.mean(s))\n    features['v_mean'] = float(np.mean(v))\n    features['h_std'] = float(np.std(h))\n    features['s_std'] = float(np.std(s))\n    features['v_std'] = float(np.std(v))\n    \n    return features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:13:46.493745Z","iopub.execute_input":"2026-01-17T02:13:46.494316Z","iopub.status.idle":"2026-01-17T02:13:46.512361Z","shell.execute_reply.started":"2026-01-17T02:13:46.494249Z","shell.execute_reply":"2026-01-17T02:13:46.511180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_features_batch(image_paths, data_path, desc=\"Extracting\"):\n    \"\"\"Extract features for list of image paths.\"\"\"\n    results = []\n    for path in tqdm(image_paths, desc=desc):\n        full_path = data_path / path\n        feat = extract_color_features(full_path)\n        feat['image_path'] = path\n        results.append(feat)\n    return pd.DataFrame(results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:13:46.513385Z","iopub.execute_input":"2026-01-17T02:13:46.513694Z","iopub.status.idle":"2026-01-17T02:13:46.532244Z","shell.execute_reply.started":"2026-01-17T02:13:46.513661Z","shell.execute_reply":"2026-01-17T02:13:46.531160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract train features\ntrain_feat = extract_features_batch(train_wide['image_path'].values, DATA_PATH, \"Train\")\ntrain_data = train_wide.merge(train_feat, on='image_path')\nprint(f\"Train data shape: {train_data.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:13:46.533309Z","iopub.execute_input":"2026-01-17T02:13:46.533596Z","iopub.status.idle":"2026-01-17T02:15:13.603629Z","shell.execute_reply.started":"2026-01-17T02:13:46.533572Z","shell.execute_reply":"2026-01-17T02:15:13.602621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract test features\ntest_feat = extract_features_batch(test_wide['image_path'].values, DATA_PATH, \"Test\")\ntest_data = test_wide.merge(test_feat, on='image_path')\nprint(f\"Test data shape: {test_data.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:15:13.606163Z","iopub.execute_input":"2026-01-17T02:15:13.606533Z","iopub.status.idle":"2026-01-17T02:15:13.866399Z","shell.execute_reply.started":"2026-01-17T02:15:13.606505Z","shell.execute_reply":"2026-01-17T02:15:13.865314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define feature columns\nFEAT_COLS = [c for c in train_feat.columns if c != 'image_path']\n\n# Add NDVI and Height if available in train\nif 'Pre_GSHH_NDVI' in train_data.columns:\n    FEAT_COLS.append('Pre_GSHH_NDVI')\n    # For test, use train mean\n    test_data['Pre_GSHH_NDVI'] = train_data['Pre_GSHH_NDVI'].mean()\n\nif 'Height_Ave_cm' in train_data.columns:\n    FEAT_COLS.append('Height_Ave_cm')\n    test_data['Height_Ave_cm'] = train_data['Height_Ave_cm'].mean()\n\nprint(f\"Feature columns ({len(FEAT_COLS)}): {FEAT_COLS}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:15:13.867679Z","iopub.execute_input":"2026-01-17T02:15:13.868023Z","iopub.status.idle":"2026-01-17T02:15:13.875674Z","shell.execute_reply.started":"2026-01-17T02:15:13.867974Z","shell.execute_reply":"2026-01-17T02:15:13.874663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train LightGBM models\nlgb_params = {\n    'objective': 'regression',\n    'metric': 'rmse',\n    'boosting_type': 'gbdt',\n    'learning_rate': 0.05,\n    'num_leaves': 31,\n    'min_child_samples': 20,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 0.1,\n    'reg_lambda': 0.1,\n    'random_state': 42,\n    'verbose': -1,\n}\n\nX_train = train_data[FEAT_COLS].fillna(0).values\nX_test = test_data[FEAT_COLS].fillna(0).values\n\nn_folds = 5\nkf = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n\nmodels = {}\ntest_preds = {}\n\nfor target in TARGET_NAMES:\n    print(f\"\\nTraining: {target}\")\n    y = train_data[target].values\n    fold_models = []\n    fold_test_preds = np.zeros(len(X_test))\n    \n    for fold, (tr_idx, va_idx) in enumerate(kf.split(X_train)):\n        tr_data = lgb.Dataset(X_train[tr_idx], label=y[tr_idx])\n        va_data = lgb.Dataset(X_train[va_idx], label=y[va_idx])\n        \n        model = lgb.train(\n            lgb_params, tr_data,\n            num_boost_round=500,\n            valid_sets=[va_data],\n            callbacks=[lgb.early_stopping(50), lgb.log_evaluation(0)]\n        )\n        fold_models.append(model)\n        fold_test_preds += model.predict(X_test) / n_folds\n    \n    models[target] = fold_models\n    # Clean predictions\n    fold_test_preds = np.nan_to_num(fold_test_preds, nan=0.0, posinf=0.0, neginf=0.0)\n    fold_test_preds = np.maximum(fold_test_preds, 0.0)\n    test_preds[target] = fold_test_preds\n    print(f\"  {target}: mean={fold_test_preds.mean():.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:15:13.877098Z","iopub.execute_input":"2026-01-17T02:15:13.877445Z","iopub.status.idle":"2026-01-17T02:15:15.482479Z","shell.execute_reply.started":"2026-01-17T02:15:13.877407Z","shell.execute_reply":"2026-01-17T02:15:15.481784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create submission\n# Step 1: Create wide predictions DataFrame\npred_df = pd.DataFrame({'image_path': test_data['image_path'].values})\nfor target in TARGET_NAMES:\n    pred_df[target] = test_preds[target]\n\nprint(f\"Predictions (wide) shape: {pred_df.shape}\")\npred_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:15:15.483161Z","iopub.execute_input":"2026-01-17T02:15:15.483402Z","iopub.status.idle":"2026-01-17T02:15:15.513915Z","shell.execute_reply.started":"2026-01-17T02:15:15.483379Z","shell.execute_reply":"2026-01-17T02:15:15.513356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 2: Melt to long format\npred_long = pred_df.melt(\n    id_vars=['image_path'],\n    value_vars=TARGET_NAMES,\n    var_name='target_name',\n    value_name='target'\n)\nprint(f\"Predictions (long) shape: {pred_long.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:15:15.514705Z","iopub.execute_input":"2026-01-17T02:15:15.514941Z","iopub.status.idle":"2026-01-17T02:15:15.524129Z","shell.execute_reply.started":"2026-01-17T02:15:15.514920Z","shell.execute_reply":"2026-01-17T02:15:15.522926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 3: Merge with original test_long to get correct sample_id\nsubmission = pd.merge(\n    test_long[['sample_id', 'image_path', 'target_name']],\n    pred_long,\n    on=['image_path', 'target_name'],\n    how='left'\n)[['sample_id', 'target']]\n\n# Final cleanup\nsubmission['target'] = submission['target'].fillna(0.0)\nsubmission['target'] = np.nan_to_num(submission['target'].values, nan=0.0, posinf=0.0, neginf=0.0)\nsubmission['target'] = np.maximum(submission['target'].values, 0.0)\n\nprint(f\"Submission shape: {submission.shape}\")\nprint(f\"NaN count: {submission['target'].isna().sum()}\")\nprint(f\"Negative count: {(submission['target'] < 0).sum()}\")\nsubmission.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:15:15.525289Z","iopub.execute_input":"2026-01-17T02:15:15.526077Z","iopub.status.idle":"2026-01-17T02:15:15.550161Z","shell.execute_reply.started":"2026-01-17T02:15:15.526044Z","shell.execute_reply":"2026-01-17T02:15:15.549399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save submission\nsubmission.to_csv(OUTPUT_PATH / 'submission.csv', index=False)\nprint(f\"Saved to {OUTPUT_PATH / 'submission.csv'}\")\nprint(f\"Total rows: {len(submission)}\")\n\n# Verify\ncheck = pd.read_csv(OUTPUT_PATH / 'submission.csv')\nprint(f\"\\nVerification:\")\nprint(check.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T02:15:15.551156Z","iopub.execute_input":"2026-01-17T02:15:15.551495Z","iopub.status.idle":"2026-01-17T02:15:15.576508Z","shell.execute_reply.started":"2026-01-17T02:15:15.551466Z","shell.execute_reply":"2026-01-17T02:15:15.575505Z"}},"outputs":[],"execution_count":null}]}