{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":293690062,"sourceType":"kernelVersion"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nimport glob\nfrom tqdm import tqdm\n\nimport math\nimport random\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn import model_selection","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:04:34.806571Z","iopub.execute_input":"2026-02-02T09:04:34.806940Z","iopub.status.idle":"2026-02-02T09:04:35.960738Z","shell.execute_reply.started":"2026-02-02T09:04:34.806914Z","shell.execute_reply":"2026-02-02T09:04:35.959442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\nembed_df = pd.read_csv(\"/kaggle/input/cs-embeddings/train_siglip_embeddings.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:04:35.962187Z","iopub.execute_input":"2026-02-02T09:04:35.962738Z","iopub.status.idle":"2026-02-02T09:04:36.327356Z","shell.execute_reply.started":"2026-02-02T09:04:35.962711Z","shell.execute_reply":"2026-02-02T09:04:36.326069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pivot_table(df: pd.DataFrame)->pd.DataFrame:\n\n    if 'target' in df.columns.tolist():\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'], \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    else:\n        df['target'] = 0\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index='image_path', \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    return df_pt\n\ntrain_pvt = pivot_table(train_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:04:36.328516Z","iopub.execute_input":"2026-02-02T09:04:36.328915Z","iopub.status.idle":"2026-02-02T09:04:36.384337Z","shell.execute_reply.started":"2026-02-02T09:04:36.328883Z","shell.execute_reply":"2026-02-02T09:04:36.383008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[['sample_id', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:04:36.387078Z","iopub.execute_input":"2026-02-02T09:04:36.387457Z","iopub.status.idle":"2026-02-02T09:04:36.421993Z","shell.execute_reply.started":"2026-02-02T09:04:36.387422Z","shell.execute_reply":"2026-02-02T09:04:36.419285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.path.basename(train_pvt.iloc[0]['image_path']).split('.')[0]\n\nextract_image_id = lambda img_path: os.path.basename(img_path).split('.')[0]\n\ntrain_pvt['event_id'] = train_pvt['image_path'].map(extract_image_id)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:04:36.423765Z","iopub.execute_input":"2026-02-02T09:04:36.424168Z","iopub.status.idle":"2026-02-02T09:04:36.433767Z","shell.execute_reply.started":"2026-02-02T09:04:36.424135Z","shell.execute_reply":"2026-02-02T09:04:36.432298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_pvt[['image_path', 'event_id']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:04:36.434541Z","iopub.execute_input":"2026-02-02T09:04:36.435029Z","iopub.status.idle":"2026-02-02T09:04:36.477952Z","shell.execute_reply.started":"2026-02-02T09:04:36.434995Z","shell.execute_reply":"2026-02-02T09:04:36.476618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.cluster import KMeans, BisectingKMeans, AgglomerativeClustering\nfrom sklearn.model_selection import StratifiedGroupKFold, GroupKFold\nfrom sklearn.decomposition import PCA\nfrom sklearn import model_selection\nfrom sklearn import preprocessing\n\ndef create_robust_cv(train_df, \n                     target_names, \n                     weights, \n                     n_splits=4, \n                     n_clusters=30, \n                     seed=42):\n    \"\"\"\n    Creates a robust CV scheme that simulates unseen test environments.\n    \"\"\"\n    df = train_df.copy()\n\n    # ----------------------------\n    # 1. Select Embedding Columns\n    # ----------------------------\n    # Robust selector for \"emb\", \"embedding\", or numeric columns\n    emb_cols = [c for c in df.columns if \"emb\" in str(c).lower()]\n    if not emb_cols:\n        exclude = set(target_names + list(weights.keys()) + ['fold', 'id', 'image_path', 'State', 'Species'])\n        emb_cols = [c for c in df.columns if c not in exclude and str(c).isdigit()]\n    \n    print(f\"Using {len(emb_cols)} embedding columns for clustering.\")\n\n    # ----------------------------\n    # 2. Visual Clustering (Define the Groups)\n    # ----------------------------\n    # We create ~50 groups. The model must learn to generalize *across* groups.\n    # X_emb = df[emb_cols].values\n    \n    pca = PCA(n_components=0.85, random_state=seed)\n    X_emb = pca.fit_transform(df[emb_cols].values)\n    \n    kmeans = KMeans(n_clusters=n_clusters, random_state=seed, n_init=10)\n    # kmeans = BisectingKMeans(n_clusters=n_clusters, random_state=seed, n_init=10)\n    # kmeans = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward')\n    df[\"visual_cluster\"] = kmeans.fit_predict(X_emb)\n\n    # ----------------------------\n    # 3. Create Stratification Target\n    # ----------------------------\n    # We want folds to be balanced by BIOMASS, not by cluster ID.\n    composite = np.zeros(len(df))\n    for t in target_names:\n        composite += df[t] * weights.get(t, 0)\n        \n    # Bin the continuous target into 10 bins for stratification\n    try:\n        df['target_bins'] = pd.qcut(composite, q=10, labels=False, duplicates='drop')\n    except ValueError:\n        df['target_bins'] = pd.qcut(composite, q=5, labels=False, duplicates='drop')\n\n    # ----------------------------\n    # 4. Stratified Group K-Fold\n    # ----------------------------\n    # CRITICAL: \n    #   y      = target_bins    (Balance the biomass difficulty)\n    #   groups = visual_cluster (Prevent leakage of similar images)\n    sgkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n\n    df[\"fold\"] = -1\n    for fold, (train_idx, val_idx) in enumerate(sgkf.split(df, df['target_bins'], groups=df[\"visual_cluster\"])):\n        df.loc[val_idx, \"fold\"] = fold\n\n    return df\n\n# ==========================================\n# CONFIGURATION\n# ==========================================\nTARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\nweights = {\n    'Dry_Green_g': 0.1, 'Dry_Dead_g': 0.1, 'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2, 'Dry_Total_g': 0.5,\n}\n\n# # LOAD AND CREATE FOLDS\n# train_df = pd.read_csv(\"/kaggle/input/cs-embeddings/train_siglip_embeddings.csv\")\n# train_df = create_robust_cv(train_df, TARGET_NAMES, weights, n_splits=5, n_clusters=50)\n\n# # VERIFICATION\n# print(\"\\nFold Distribution (Will be uneven - this is expected):\")\n# print(train_df['fold'].value_counts())\n\n# print(\"\\nVerify Leakage (Must be 0):\")\n# leakage_check = train_df.groupby('visual_cluster')['fold'].nunique()\n# print(f\"Clusters appearing in multiple folds: {sum(leakage_check > 1)}\")\n\n# train_df['Sampling_Date_Month'] = train_df.Sampling_Date.apply(lambda x: x.split('/')[1].strip())\n# kfold = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=0)\n\n# agg_train_df = train_df.copy()\n# agg_train_df['fold'] = None\n# for i, (trn_idx, val_idx) in enumerate(kfold.split(agg_train_df.index, y=agg_train_df.State, groups=agg_train_df.Sampling_Date)):\n#     agg_train_df.loc[val_idx, 'fold'] = i\n    \n#     trn_df = agg_train_df[agg_train_df.fold != i]\n#     val_df = agg_train_df[agg_train_df.fold == i]\n    \n#     trn_months = sorted(int(x) for x in trn_df.Sampling_Date_Month.unique())\n#     val_months = sorted(int(x) for x in val_df.Sampling_Date_Month.unique())\n#     print(f'trn({trn_df.shape[0]}) -> val({val_df.shape[0]}): {trn_months} -> {val_months}')\n\n# train_df = agg_train_df.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:04:36.479319Z","iopub.execute_input":"2026-02-02T09:04:36.479700Z","iopub.status.idle":"2026-02-02T09:04:36.970095Z","shell.execute_reply.started":"2026-02-02T09:04:36.479669Z","shell.execute_reply":"2026-02-02T09:04:36.968914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_stratified_folds(df, n_splits=5, seed=42):\n    # Reset index to ensure positional indices from skf match the dataframe\n    df = df.copy().reset_index(drop=True)\n\n    # # 1. Feature Engineering for Stratification\n    # df['bin_total'] = pd.qcut(df['Dry_Total_g'], q=4, labels=False)\n\n    # df['dead_frac'] = df['Dry_Dead_g'] / (df['Dry_Total_g'] + 1e-6)\n    # df['bin_dead'] = pd.qcut(df['dead_frac'], q=3, labels=False, duplicates='drop')\n\n    # living_mass = df['Dry_Clover_g'] + df['Dry_Green_g']\n    # df['clover_frac'] = df['Dry_Clover_g'] / (living_mass + 1e-6)\n\n    # try:\n    #     df['bin_species'] = pd.qcut(df['clover_frac'], q=3, labels=False, duplicates='drop')\n    # except ValueError:\n    #     df['bin_species'] = pd.cut(df['clover_frac'], bins=[-0.1, 0.1, 0.5, 1.1], labels=[0, 1, 2])\n\n    # # 2. Create Stratification Key\n    # df['stratify_key'] = (\n    #     df['bin_total'].astype(str) + \"_\" +\n    #     df['bin_dead'].astype(str) + \"_\" +\n    #     df['bin_species'].astype(str)\n    # )\n\n    # # 3. Handle Rare Classes (Aggressive Misc Bucketing)\n    # counts = df['stratify_key'].value_counts()\n    # drop_keys = counts[counts < n_splits].index # Threshold = n_splits\n    \n    # df.loc[df['stratify_key'].isin(drop_keys), 'stratify_key'] = 'misc'\n\n    # # Safety check for the 'misc' bucket itself\n    # misc_count = len(df[df['stratify_key'] == 'misc'])\n    # if 0 < misc_count < n_splits:\n    #     most_common = df['stratify_key'].mode()[0]\n    #     df.loc[df['stratify_key'] == 'misc', 'stratify_key'] = most_common\n\n    # 4. Perform the Split (THIS WAS MISSING)\n    skf = model_selection.StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n\n    df['fold'] = -1\n    # Iterate through the split and assign the fold number\n    for fold, (train_idx, val_idx) in enumerate(skf.split(df, df['State'])):\n        df.loc[val_idx, 'fold'] = fold\n\n    return df\n\n\n# def get_robust_stratified_folds(df, n_splits=5, seed=42):\n#     df = df.copy().reset_index(drop=True)\n\n#     # --- 1. Feature Engineering ---\n    \n#     # A. Total Mass: 3 Bins (Low, Mid, High)\n#     df['bin_total'] = pd.qcut(df['Dry_Total_g'], q=3, labels=[\"L\", \"M\", \"H\"])\n\n#     # B. Clover Fraction: 2 Bins (Low vs High) - Keep it simple\n#     living_mass = df['Dry_Clover_g'] + df['Dry_Green_g']\n#     df['clover_frac'] = df['Dry_Clover_g'] / (living_mass + 1e-6)\n#     # Threshold at 20% clover\n#     df['bin_clover'] = pd.cut(df['clover_frac'], bins=[-0.1, 0.2, 1.1], labels=[\"Lo\", \"Hi\"])\n\n#     # C. State\n#     df['state_key'] = df['State'].astype(str)\n\n#     # --- 2. Define Hierarchy Keys ---\n    \n#     # Level 1: Most detailed\n#     df['key_L1'] = df['state_key'] + \"_\" + df['bin_total'].astype(str) + \"_\" + df['bin_clover'].astype(str)\n    \n#     # Level 2: Ignore Clover\n#     df['key_L2'] = df['state_key'] + \"_\" + df['bin_total'].astype(str)\n    \n#     # Level 3: State Only (Guaranteed > 5 samples per State)\n#     df['key_L3'] = df['state_key']\n\n#     # --- 3. The Waterfall Collapse ---\n    \n#     # Start with Level 1\n#     df['final_stratify'] = df['key_L1']\n    \n#     # Check counts\n#     counts = df['final_stratify'].value_counts()\n#     rare_keys = counts[counts < n_splits].index\n    \n#     # FALLBACK 1: If Level 1 is rare, replace with Level 2\n#     mask_rare_L1 = df['final_stratify'].isin(rare_keys)\n#     df.loc[mask_rare_L1, 'final_stratify'] = df.loc[mask_rare_L1, 'key_L2']\n    \n#     # Re-check counts after first merge\n#     counts = df['final_stratify'].value_counts()\n#     rare_keys = counts[counts < n_splits].index\n    \n#     # FALLBACK 2: If Level 2 is rare, replace with Level 3 (State)\n#     mask_rare_L2 = df['final_stratify'].isin(rare_keys)\n#     df.loc[mask_rare_L2, 'final_stratify'] = df.loc[mask_rare_L2, 'key_L3']\n    \n#     # (Optional) Final Safety: If State somehow had < 5 samples (impossible in your data, but good practice)\n#     counts = df['final_stratify'].value_counts()\n#     rare_keys = counts[counts < n_splits].index\n#     if len(rare_keys) > 0:\n#         print(f\"Warning: Dropping {len(rare_keys)} extremely rare buckets to 'Misc'\")\n#         df.loc[df['final_stratify'].isin(rare_keys), 'final_stratify'] = 'Misc'\n\n#     # --- 4. Perform Split ---\n#     skf = model_selection.StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    \n#     df['fold'] = -1\n#     for fold, (train_idx, val_idx) in enumerate(skf.split(df, df['final_stratify'])):\n#         df.loc[val_idx, 'fold'] = fold\n        \n#     return df\n\n# LOAD AND CREATE FOLDS\ntrain_df = pd.read_csv(\"/kaggle/input/cs-embeddings/train_siglip_embeddings.csv\")\ntrain_df = get_stratified_folds(train_df, n_splits=3)\n\n# Verify Balance\nprint(\"--- Mean Values per Fold ---\")\nprint(train_df.groupby('fold')[['Dry_Total_g', 'Dry_Dead_g', 'Dry_Clover_g', 'Dry_Green_g']].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:06:39.987609Z","iopub.execute_input":"2026-02-02T09:06:39.987977Z","iopub.status.idle":"2026-02-02T09:06:40.125716Z","shell.execute_reply.started":"2026-02-02T09:06:39.987952Z","shell.execute_reply":"2026-02-02T09:06:40.124307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.to_csv(\"csiro_data_split.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:06:51.833176Z","iopub.execute_input":"2026-02-02T09:06:51.833527Z","iopub.status.idle":"2026-02-02T09:06:52.408421Z","shell.execute_reply.started":"2026-02-02T09:06:51.833501Z","shell.execute_reply":"2026-02-02T09:06:52.407285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv(\"csiro_data_split.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-02T09:06:52.409776Z","iopub.execute_input":"2026-02-02T09:06:52.410142Z","iopub.status.idle":"2026-02-02T09:06:52.548063Z","shell.execute_reply.started":"2026-02-02T09:06:52.410112Z","shell.execute_reply":"2026-02-02T09:06:52.546867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}