{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom PIL import Image, ImageCms\nfrom typing import Dict\nfrom skimage.feature import graycomatrix, graycoprops\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import KFold\nfrom catboost import CatBoostRegressor\nfrom xgboost import XGBRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:57.610477Z","iopub.execute_input":"2026-01-19T06:12:57.610881Z","iopub.status.idle":"2026-01-19T06:12:59.250538Z","shell.execute_reply.started":"2026-01-19T06:12:57.610852Z","shell.execute_reply":"2026-01-19T06:12:59.249928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:59.251733Z","iopub.execute_input":"2026-01-19T06:12:59.252071Z","iopub.status.idle":"2026-01-19T06:12:59.279203Z","shell.execute_reply.started":"2026-01-19T06:12:59.252048Z","shell.execute_reply":"2026-01-19T06:12:59.278726Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta_cols = [\n    \"image_path\",\n    \"Sampling_Date\",\n    \"State\",\n    \"Species\",\n    \"Pre_GSHH_NDVI\",\n    \"Height_Ave_cm\",\n]\n\nmeta_df = (\n    train_df[meta_cols]\n    .groupby(\"image_path\", as_index=False)\n    .agg(\"first\")\n)\n\npivot_df = (\n    train_df\n    .pivot_table(\n        index=\"image_path\",\n        columns=\"target_name\",\n        values=\"target\",\n        aggfunc=\"first\"  # safe since each type appears once per image_path\n    )\n    .reset_index()\n)\n\ntrain_df = meta_df.merge(pivot_df, on=\"image_path\")\n\n# Convert to datetime (your format is YYYY/M/D)\ntrain_df[\"Sampling_Date\"] = pd.to_datetime(train_df[\"Sampling_Date\"], format=\"%Y/%m/%d\")\n\n# Extract year, month, day\ntrain_df[\"sampling_year\"]  = train_df[\"Sampling_Date\"].dt.year\ntrain_df[\"sampling_month\"] = train_df[\"Sampling_Date\"].dt.month\ntrain_df[\"sampling_day\"]   = train_df[\"Sampling_Date\"].dt.day\n\ntrain_df.drop(columns=[\"Sampling_Date\"], inplace=True)\n\ntrain_df = pd.get_dummies(train_df, columns=['State'], prefix='State', dtype=\"int\")\ntrain_df = pd.get_dummies(train_df, columns=['Species'], prefix='Species', dtype=\"int\")\n\ntrain_df['Height_Ave_cm'] = (train_df.Height_Ave_cm - train_df.Height_Ave_cm.mean()) / train_df.Height_Ave_cm.std()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:59.279995Z","iopub.execute_input":"2026-01-19T06:12:59.280177Z","iopub.status.idle":"2026-01-19T06:12:59.333436Z","shell.execute_reply.started":"2026-01-19T06:12:59.280159Z","shell.execute_reply":"2026-01-19T06:12:59.332717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:59.335029Z","iopub.execute_input":"2026-01-19T06:12:59.335596Z","iopub.status.idle":"2026-01-19T06:12:59.359290Z","shell.execute_reply.started":"2026-01-19T06:12:59.335572Z","shell.execute_reply":"2026-01-19T06:12:59.358776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = train_df[[\"image_path\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"GDM_g\", \"Dry_Total_g\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:59.360060Z","iopub.execute_input":"2026-01-19T06:12:59.360301Z","iopub.status.idle":"2026-01-19T06:12:59.364091Z","shell.execute_reply.started":"2026-01-19T06:12:59.360281Z","shell.execute_reply":"2026-01-19T06:12:59.363555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:59.364788Z","iopub.execute_input":"2026-01-19T06:12:59.365021Z","iopub.status.idle":"2026-01-19T06:12:59.382088Z","shell.execute_reply.started":"2026-01-19T06:12:59.364993Z","shell.execute_reply":"2026-01-19T06:12:59.381549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nclass PreProcess:\n    SUPPORTED_COLOR_SPACES = {\"RGB\", \"L\", \"HSV\", \"LAB\"}\n\n    def __init__(self):\n        pass\n\n    @staticmethod\n    def load_image_rgb(image_path: str) -> Image.Image:\n        with Image.open(image_path) as img:\n            return img.convert(\"RGB\")\n\n    def convert_color_space(\n        self,\n        img: Image.Image,\n        target_color_space: str\n    ) -> Image.Image:\n\n        target_color_space = target_color_space.upper()\n\n        if target_color_space not in self.SUPPORTED_COLOR_SPACES:\n            raise ValueError(\n                f\"Unsupported color space '{target_color_space}'. \"\n                f\"Supported values: {sorted(self.SUPPORTED_COLOR_SPACES)}\"\n            )\n\n        if target_color_space == \"LAB\":\n            return self.rgb_to_lab(img)\n\n        return img.convert(target_color_space)\n\n    @staticmethod\n    def rgb_to_lab(img: Image.Image) -> Image.Image:\n        srgb = ImageCms.createProfile(\"sRGB\")\n        lab = ImageCms.createProfile(\"LAB\")\n\n        transform = ImageCms.buildTransformFromOpenProfiles(\n            srgb, lab, \"RGB\", \"LAB\"\n        )\n\n        return ImageCms.applyTransform(img, transform)\n\n\n    def extract_rgb_features(self, rgb_image):\n        features = {}\n        R, G, B = rgb_image[:, :, 0], rgb_image[:, :, 1], rgb_image[:, :, 2]\n        features[\"red_mean\"] = R.mean() / 255\n        features[\"red_std\"] = R.std() / 255\n\n        features[\"green_mean\"] = G.mean() / 255\n        features[\"green_std\"] = G.std() / 255\n\n        features[\"blue_mean\"] = B.mean() / 255\n        features[\"blue_std\"] = B.std() / 255\n\n        features[\"red_percent\"] = (R>125).sum() / (R.shape[0]*R.shape[1])\n        features[\"green_percent\"] = (G>125).sum() / (G.shape[0]*G.shape[1])\n        features[\"blue_percent\"] = (B>125).sum() / (B.shape[0]*B.shape[1])\n\n        # Excess Green\n        ExG = ((2*G) - R) - B\n        features[\"exg_mean\"] = ExG.mean()\n        features[\"exg_std\"] = ExG.std()\n        features[\"exg_percent\"] = (ExG>0).sum() / (ExG.shape[0]*ExG.shape[1])\n\n        # Normalized Green Red Diff Index\n        ngdri = (G/255-R/255) / ((G/255+R/255) + 1e-6)\n        features[\"ngdri_mean\"] = ngdri.mean()\n        features[\"ngdri_std\"] = ngdri.std()\n        features[\"ngdri_+percent\"] = (ngdri>0).sum() / (ngdri.shape[0]*ngdri.shape[1])\n        features[\"ngdri_-percent\"] = (ngdri<0).sum() / (ngdri.shape[0]*ngdri.shape[1])\n        \n        return features\n\n\n    def extract_grayscale_features(self, grayscale_image):\n        features = {}\n        features[\"gray_mean\"] = grayscale_image.mean() / 255\n        features[\"gray_std\"] = grayscale_image.std() / 255\n        # features[\"gray_entropy\"] = -np.sum(\n        #     (grayscale_image/255) * np.log2((grayscale_image/255) + 1e-6)\n        # )\n        return features\n        \n\n    def extract_hsv_features(self, hsv_image):\n        H, S, V = hsv_image[:, :, 0], hsv_image[:, :, 1], hsv_image[:, :, 2]\n        features = {}\n\n        features[\"hue_mean\"] = H.mean() / 255\n        features[\"hue_std\"] = H.std() / 255\n        features[\"hue_var\"] = np.var(H/255)\n\n        features[\"sat_mean\"] = S.mean() / 255\n        features[\"sat_std\"] = S.std() / 255\n        features[\"sat_var\"] = np.var(S/255)\n\n        features[\"hsvalue_mean\"] = V.mean() / 255\n        features[\"hsvalue_std\"] = V.std() / 255\n        features[\"hsvalue_var\"] = np.var(V/255)\n\n        \n        # Green hue\n        green_mask = (H >= 35) & (H <= 85)\n        features[\"green_hue_mean\"] = H[green_mask].mean() / 255 if green_mask.any() else 0\n        features[\"green_hue_std\"]  = H[green_mask].std() / 255  if green_mask.any() else 0\n        features[\"green_hue_var\"] = np.var(green_mask/255)\n        features[\"green_hue_percent\"] = (green_mask>0).sum() / (green_mask.shape[0]*green_mask.shape[1])\n\n        yellowing_grass = (H >= 20) & (H <= 35)\n        features[\"yellowing_grass_hue_mean\"] = yellowing_grass.mean()\n        features[\"yellowing_grass_hue_std\"] = yellowing_grass.std()\n        features[\"yellowing_grass_hue_var\"] = np.var(yellowing_grass/255)\n        features[\"yellowing_grass_hue_percent\"] = (yellowing_grass>0).sum() / (yellowing_grass.shape[0]*yellowing_grass.shape[1])\n\n        \n        brown_dry = (H >= 10) & (H <= 20)\n        features[\"brown_dry_hue_mean\"] = brown_dry.mean()\n        features[\"brown_dry_hue_std\"] = brown_dry.std()\n        features[\"brown_dry_hue_var\"] = np.var(brown_dry/255)\n        features[\"brown_dry_hue_percent\"] = (brown_dry>0).sum() / (brown_dry.shape[0]*brown_dry.shape[1])\n\n        \n        soild_dead = H <= 10\n        features[\"soild_dead_hue_mean\"] = soild_dead.mean()\n        features[\"soild_dead_hue_std\"] = soild_dead.std()\n        features[\"soild_dead_hue_var\"] = np.var(soild_dead/255)\n        features[\"soild_dead_hue_percent\"] = (soild_dead>0).sum() / (soild_dead.shape[0]*soild_dead.shape[1])\n\n        # More saturation -> More vegetation\n        features[\"less_saturated_percent\"] = (S<50).sum() / (S.shape[0]*S.shape[1])\n        features[\"more_saturated_percent\"] = (S>50).sum() / (S.shape[0]*S.shape[1])\n\n        # Shadow in image\n        features[\"shadow_percent\"] = (V<50).sum() / (V.shape[0]*V.shape[1])\n\n        masked_v = V.copy()\n        masked_v[~green_mask] = 0\n        \n        masked_v = masked_v.astype(np.float32)\n        masked_v = (masked_v / 255.0 * (32 - 1)).astype(np.uint8)\n\n        distances = [1]\n        angles = [0, np.pi/4, np.pi/2, 3*np.pi/4]\n        \n        # 3. Calculate the GLCM\n        # levels=256 specifies the number of gray levels\n        glcm = graycomatrix(masked_v, \n                            distances=distances, \n                            angles=angles, \n                            levels=32, \n                            symmetric=True, \n                            normed=True) #\n        \n        # 4. Extract texture features (e.g., contrast and homogeneity)\n        contrast = graycoprops(glcm, 'contrast')\n        homogeneity = graycoprops(glcm, 'homogeneity')\n        energy = graycoprops(glcm, 'energy')\n        correlation = graycoprops(glcm, 'correlation')\n        \n        eps = 1e-10\n        biomass_entropy = -np.sum(glcm * np.log2(glcm + eps))\n\n        features[\"glcm_contrast\"] = contrast.mean()\n        features[\"glcm_homogeneity\"] = homogeneity.mean()\n        features[\"glcm_energy\"] = energy.mean()\n        features[\"glcm_correlation\"] = correlation.mean()\n        features[\"biomass_entropy\"] = biomass_entropy\n\n        return features\n        \n\n    def extract_lab_features(self, lab_image):\n        L, A, B = lab_image[:, :, 0], lab_image[:, :, 1], lab_image[:, :, 2]\n        features = {}\n\n        features[\"L_mean\"] = L.mean() / 255\n        features[\"L_std\"] = L.std() / 255\n        features[\"L_var\"] = np.var(L/255)\n\n        features[\"A_mean\"] = A.mean() / 255\n        features[\"A_std\"] = A.std() / 255\n        features[\"A_var\"] = np.var(A/255)\n\n        features[\"B_mean\"] = B.mean() / 255\n        features[\"B_std\"] = B.std() / 255\n        features[\"B_var\"] = np.var(B/255)\n\n        features[\"A_B_ratio\"] = A.mean() / (B.mean() + 1e-6)\n        \n        A_signed = A.astype(np.float32) - 128\n        B_signed = B.astype(np.float32) - 128\n        \n        features[\"A_mean\"] = A_signed.mean()\n        features[\"B_mean\"] = B_signed.mean()\n        features[\"AB_ratio\"] = A_signed.mean() / (B_signed.mean() + 1e-6)\n\n        return features    \n        \n\n    def process(self, image_path: str) -> Dict[str, Image.Image]:\n        # Load Image\n        rgb_image = self.load_image_rgb(image_path)\n        grayscale_image = self.convert_color_space(rgb_image, \"L\")\n        hsv_image = self.convert_color_space(rgb_image, \"HSV\")\n        lab_image = self.convert_color_space(rgb_image, \"LAB\")\n\n        # Image to Numpy\n        rgb_image = np.asarray(rgb_image)\n        grayscale_image = np.asarray(grayscale_image)\n        hsv_image = np.asarray(hsv_image)\n        lab_image = np.asarray(lab_image)\n\n        rgb_features = self.extract_rgb_features(rgb_image)\n        hsv_features = self.extract_hsv_features(hsv_image)\n        lab_features = self.extract_lab_features(lab_image)\n        grayscale_features = self.extract_grayscale_features(grayscale_image)\n\n        all_features = {**rgb_features, **hsv_features, **lab_features, **grayscale_features}\n        all_features[\"green_to_total\"] = all_features[\"green_percent\"] / (all_features[\"exg_percent\"] + 1e-6)\n        all_features[\"brown_to_green\"] = all_features[\"brown_dry_hue_percent\"] / (all_features[\"green_hue_percent\"] + 1e-6)\n\n        all_features[\"exg_x_sat\"] = all_features[\"exg_mean\"] * all_features[\"sat_mean\"]\n        all_features[\"entropy_x_exg\"] = all_features[\"biomass_entropy\"] * all_features[\"exg_mean\"]\n\n\n        \n        return all_features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:59.383095Z","iopub.execute_input":"2026-01-19T06:12:59.383365Z","iopub.status.idle":"2026-01-19T06:12:59.478578Z","shell.execute_reply.started":"2026-01-19T06:12:59.383337Z","shell.execute_reply":"2026-01-19T06:12:59.477999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessor = PreProcess()\n\ndef extract_features_safe(image_path):\n    return preprocessor.process(f\"/kaggle/input/csiro-biomass/{image_path}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:59.479423Z","iopub.execute_input":"2026-01-19T06:12:59.479696Z","iopub.status.idle":"2026-01-19T06:12:59.495975Z","shell.execute_reply.started":"2026-01-19T06:12:59.479674Z","shell.execute_reply":"2026-01-19T06:12:59.495386Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntqdm.pandas()\n\ntrain_feature_df = train_data[\"image_path\"].progress_apply(extract_features_safe)\n\n# Convert list/dict → DataFrame\ntrain_feature_df = pd.DataFrame(train_feature_df.tolist())\nfor col in train_feature_df.columns:\n    train_feature_df[col + \"_rank\"] = train_feature_df[col].rank(pct=True)\n    \nprint(train_feature_df.shape)\ntrain_feature_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:12:59.496804Z","iopub.execute_input":"2026-01-19T06:12:59.497139Z","iopub.status.idle":"2026-01-19T06:17:21.547447Z","shell.execute_reply.started":"2026-01-19T06:12:59.497111Z","shell.execute_reply":"2026-01-19T06:17:21.546723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LOG_FEATURES = [\n    \"glcm_contrast\",\n    \"biomass_entropy\",\n    \"exg_mean\",\n    \"exg_x_sat\",\n    \"entropy_x_exg\",\n]\n\nfor col in LOG_FEATURES:\n    if col in train_feature_df.columns:\n        train_feature_df[col] = np.log1p(train_feature_df[col])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:17:21.549734Z","iopub.execute_input":"2026-01-19T06:17:21.550136Z","iopub.status.idle":"2026-01-19T06:17:21.555763Z","shell.execute_reply.started":"2026-01-19T06:17:21.550114Z","shell.execute_reply":"2026-01-19T06:17:21.555123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_cols = [\n    \"Dry_Clover_g\",\n    \"Dry_Dead_g\",\n    \"Dry_Green_g\",\n    \"GDM_g\",\n    \"Dry_Total_g\",\n]\n\ntrain_data = pd.concat(\n    [train_feature_df, train_data[target_cols].reset_index(drop=True)],\n    axis=1\n)\n\nprint(train_data.shape)\ntrain_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:17:21.556455Z","iopub.execute_input":"2026-01-19T06:17:21.556702Z","iopub.status.idle":"2026-01-19T06:17:21.588920Z","shell.execute_reply.started":"2026-01-19T06:17:21.556678Z","shell.execute_reply":"2026-01-19T06:17:21.588185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\ntarget_cols = [\n    \"Dry_Clover_g\",\n    \"Dry_Dead_g\",\n    \"Dry_Green_g\",\n    \"GDM_g\",\n    \"Dry_Total_g\",\n]\n\ncorr = train_data.corr(numeric_only=True)\n\n# Only feature → target correlations\ncorr_targets = corr[target_cols].drop(target_cols)\ncorr_targets = np.abs(corr_targets)\n\n# Filter high corr\ncorr_targets = corr_targets[\n    (corr_targets.Dry_Clover_g > 0.3) |\n    (corr_targets.Dry_Green_g > 0.3) |\n    (corr_targets.GDM_g > 0.3) |\n    (corr_targets.Dry_Total_g > 0.3)\n]\n\nplt.figure(figsize=(10, max(6, len(corr_targets)//3)))\nsns.heatmap(\n    corr_targets,\n    cmap=\"coolwarm\",\n    center=0,\n    linewidths=0.3\n)\n\nplt.title(\"Feature vs Target Correlation\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:24:48.427521Z","iopub.execute_input":"2026-01-19T06:24:48.427857Z","iopub.status.idle":"2026-01-19T06:24:49.095637Z","shell.execute_reply.started":"2026-01-19T06:24:48.427833Z","shell.execute_reply":"2026-01-19T06:24:49.095013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_targets = train_data[corr_targets.columns]\ntrain_features = train_data[corr_targets.index.tolist()]\n\ntrain_data = pd.concat([train_features, train_targets], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:30:59.575339Z","iopub.execute_input":"2026-01-19T06:30:59.575861Z","iopub.status.idle":"2026-01-19T06:30:59.581176Z","shell.execute_reply.started":"2026-01-19T06:30:59.575834Z","shell.execute_reply":"2026-01-19T06:30:59.580564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:31:01.305401Z","iopub.execute_input":"2026-01-19T06:31:01.306252Z","iopub.status.idle":"2026-01-19T06:31:01.323244Z","shell.execute_reply.started":"2026-01-19T06:31:01.306226Z","shell.execute_reply":"2026-01-19T06:31:01.322592Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport numpy as np\nimport pandas as pd\n\ntarget_cols = [\n    \"Dry_Clover_g\",\n    \"Dry_Dead_g\",\n    \"Dry_Green_g\",\n    \"GDM_g\",\n    \"Dry_Total_g\",\n]\n\nX = train_data.drop(columns=target_cols)\nY = train_data[target_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:31:06.788478Z","iopub.execute_input":"2026-01-19T06:31:06.789240Z","iopub.status.idle":"2026-01-19T06:31:06.794870Z","shell.execute_reply.started":"2026-01-19T06:31:06.789206Z","shell.execute_reply":"2026-01-19T06:31:06.794185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TARGET_WEIGHTS = {\n    \"Dry_Green_g\": 0.1,\n    \"Dry_Dead_g\": 0.1,\n    \"Dry_Clover_g\": 0.1,\n    \"GDM_g\": 0.2,\n    \"Dry_Total_g\": 0.5,\n}\n\n\ndef weighted_r2_score(y_true_df, y_pred_df, target_weights):\n    \"\"\"\n    y_true_df, y_pred_df: DataFrames with same columns (targets)\n    \"\"\"\n    y_true = []\n    y_pred = []\n    weights = []\n\n    for col in y_true_df.columns:\n        y_true.append(y_true_df[col].values)\n        y_pred.append(y_pred_df[col].values)\n        weights.append(\n            np.full(len(y_true_df), target_weights[col])\n        )\n\n    y_true = np.concatenate(y_true)\n    y_pred = np.concatenate(y_pred)\n    weights = np.concatenate(weights)\n\n    y_weighted_mean = np.average(y_true, weights=weights)\n\n    numerator = np.sum(weights * (y_true - y_pred) ** 2)\n    denominator = np.sum(weights * (y_true - y_weighted_mean) ** 2)\n\n    return 1.0 - numerator / (denominator + 1e-12)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:31:08.723023Z","iopub.execute_input":"2026-01-19T06:31:08.723606Z","iopub.status.idle":"2026-01-19T06:31:08.729208Z","shell.execute_reply.started":"2026-01-19T06:31:08.723579Z","shell.execute_reply":"2026-01-19T06:31:08.728468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nos.makedirs(\"models/catboost\", exist_ok=True)\nos.makedirs(\"models/xgboost\", exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:31:13.045570Z","iopub.execute_input":"2026-01-19T06:31:13.046120Z","iopub.status.idle":"2026-01-19T06:31:13.049979Z","shell.execute_reply.started":"2026-01-19T06:31:13.046096Z","shell.execute_reply":"2026-01-19T06:31:13.049271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CAT_LOSS_MAP = {\n    \"Dry_Total_g\": \"Huber:delta=20\",\n    \"GDM_g\": \"Huber:delta=10\",\n    \"Dry_Green_g\": \"RMSE\",\n    \"Dry_Dead_g\": \"MAE\",\n    \"Dry_Clover_g\": \"MAE\",\n}\n\nCAT_PARAMS = dict(\n    iterations=3000,          # ↓ from 4000\n    learning_rate=0.03,       # ↑ slightly\n    depth=7,                  # ↓ from 8\n    l2_leaf_reg=5.0,          # ADD regularization\n    loss_function=\"RMSE\",\n    eval_metric=\"RMSE\",\n    random_seed=42,\n    task_type=\"GPU\",\n    devices=\"0\",\n    verbose=False,\n    early_stopping_rounds=200\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:31:17.994978Z","iopub.execute_input":"2026-01-19T06:31:17.995494Z","iopub.status.idle":"2026-01-19T06:31:17.999736Z","shell.execute_reply.started":"2026-01-19T06:31:17.995467Z","shell.execute_reply":"2026-01-19T06:31:17.999061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"XGB_PARAMS = dict(\n    n_estimators=5000,\n    learning_rate=0.02,\n    max_depth=6,\n    min_child_weight=5,\n    subsample=0.8,\n    colsample_bytree=0.8,\n    reg_alpha=0.1,     # L1\n    reg_lambda=1.0,    # L2\n    objective=\"reg:squarederror\",\n    tree_method=\"hist\",\n    predictor=\"gpu_predictor\",\n    random_state=42,\n)\n\n\nXGB_LOSS_MAP = {\n    \"Dry_Clover_g\": \"reg:squarederror\",\n    \"Dry_Dead_g\": \"reg:squarederror\",\n    \"Dry_Green_g\": \"reg:squarederror\",\n    \"GDM_g\": \"reg:squarederror\",\n    \"Dry_Total_g\": \"reg:squarederror\",\n}\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:31:18.231151Z","iopub.execute_input":"2026-01-19T06:31:18.231885Z","iopub.status.idle":"2026-01-19T06:31:18.236032Z","shell.execute_reply.started":"2026-01-19T06:31:18.231858Z","shell.execute_reply":"2026-01-19T06:31:18.235369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost.callback import EarlyStopping\nimport xgboost as xgb","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:31:22.874047Z","iopub.execute_input":"2026-01-19T06:31:22.874695Z","iopub.status.idle":"2026-01-19T06:31:22.878040Z","shell.execute_reply.started":"2026-01-19T06:31:22.874667Z","shell.execute_reply":"2026-01-19T06:31:22.877384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=42)\nfold_scores = []\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n    print(f\"\\n===== Fold {fold} =====\")\n\n    X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n    Y_train, Y_val = Y.iloc[train_idx], Y.iloc[val_idx]\n\n    val_preds = pd.DataFrame(index=Y_val.index, columns=target_cols)\n\n    for target in target_cols:\n\n        # ---------- CatBoost ----------\n        cat_model = CatBoostRegressor(\n            **CAT_PARAMS,\n            # loss_function=CAT_LOSS_MAP[target]\n        )\n\n        cat_model.fit(\n            X_train,\n            Y_train[target],\n            eval_set=(X_val, Y_val[target]),\n            use_best_model=True\n        )\n\n        cat_model.save_model(\n            f\"models/catboost/fold_{fold}_{target}.cbm\"\n        )\n\n        cat_pred = cat_model.predict(X_val)\n\n        # ---------- XGBoost ----------\n        xgb_params = XGB_PARAMS.copy()\n        xgb_params[\"objective\"] = \"reg:squarederror\"\n        xgb_params[\"eval_metric\"] = \"rmse\"\n        \n        # dtrain = xgb.DMatrix(X_train, label=Y_train[target])\n        # dval   = xgb.DMatrix(X_val,   label=Y_val[target])\n        \n        # xgb_model = xgb.train(\n        #     params=xgb_params,\n        #     dtrain=dtrain,\n        #     num_boost_round=5000,\n        #     evals=[(dval, \"val\")],\n        #     early_stopping_rounds=300,\n        #     verbose_eval=False\n        # )\n\n        xgb_model = XGBRegressor(**xgb_params)\n\n        xgb_model.fit(\n            X_train,\n            Y_train[target],\n            eval_set=[(X_val, Y_val[target])],\n            verbose=False\n        )\n\n        xgb_model.save_model(\n            f\"models/xgboost/fold_{fold}_{target}.json\"\n        )\n\n        xgb_pred = xgb_model.predict(X_val)\n        xgb_pred = np.maximum(xgb_pred, 0.0)\n\n        # ---------- Ensemble ----------\n        val_preds[target] = 0.6 * cat_pred + 0.4 * xgb_pred\n\n    # ---------- Fold metric ----------\n    fold_score = weighted_r2_score(Y_val, val_preds, TARGET_WEIGHTS)\n    fold_scores.append(fold_score)\n\n    print(\n        f\"Fold {fold} weighted R² : {fold_score:.5f} | \"\n        f\"Running mean: {np.mean(fold_scores):.5f}\"\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:31:23.377357Z","iopub.execute_input":"2026-01-19T06:31:23.378114Z","iopub.status.idle":"2026-01-19T06:41:58.599901Z","shell.execute_reply.started":"2026-01-19T06:31:23.378089Z","shell.execute_reply":"2026-01-19T06:41:58.599234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n==============================\")\nprint(f\"Mean CV weighted R² : {np.mean(fold_scores):.5f}\")\nprint(f\"Std  CV weighted R² : {np.std(fold_scores):.5f}\")\nprint(\"==============================\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T06:41:58.600860Z","iopub.execute_input":"2026-01-19T06:41:58.601075Z","iopub.status.idle":"2026-01-19T06:41:58.609057Z","shell.execute_reply.started":"2026-01-19T06:41:58.601054Z","shell.execute_reply":"2026-01-19T06:41:58.608595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}