{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":288520982,"sourceType":"kernelVersion"}],"dockerImageVersionId":31239,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom sklearn.model_selection import StratifiedGroupKFold\n\n# =========================\n# CONFIG (DO NOT CHANGE)\n# =========================\n\nTRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"\nSIGLIP_EMB_CSV = \"/kaggle/input/csiro-train-embedings/train_siglip_embeddings_new.csv\"\nOUT_CSV = \"/kaggle/working/csiro_data_split.csv\"\n\nN_FOLDS = 5\nN_CLUSTERS = 20\nPCA_VARIANCE = 0.85\nRANDOM_STATE = 42\n\nTARGET_NAMES = [\n    \"Dry_Clover_g\",\n    \"Dry_Dead_g\",\n    \"Dry_Green_g\",\n    \"Dry_Total_g\",\n    \"GDM_g\"\n]\n\nweights = {\n    \"Dry_Green_g\": 0.1,\n    \"Dry_Dead_g\": 0.1,\n    \"Dry_Clover_g\": 0.1,\n    \"GDM_g\": 0.2,\n    \"Dry_Total_g\": 0.5\n}\n\n# =========================\n# HELPERS\n# =========================\n\ndef pivot_train_long_to_wide(df):\n    wide = (\n        df.pivot_table(\n            index=[\n                \"image_path\",\n                \"Sampling_Date\",\n                \"State\",\n                \"Species\",\n                \"Pre_GSHH_NDVI\",\n                \"Height_Ave_cm\",\n            ],\n            columns=\"target_name\",\n            values=\"target\",\n            aggfunc=\"mean\"\n        )\n        .reset_index()\n    )\n    wide.columns.name = None\n    return wide\n\n# =========================\n# MAIN\n# =========================\n\ndef main():\n    # Load train (LONG format)\n    train_long = pd.read_csv(TRAIN_CSV)\n\n    # Pivot to WIDE (one row per image)\n    train_df = pivot_train_long_to_wide(train_long)\n\n    # Normalize image_path (safety)\n    train_df[\"image_path\"] = train_df[\"image_path\"].apply(lambda x: x.split(\"/\")[-1])\n\n    # Load SigLIP embeddings (one row per image)\n    emb_df = pd.read_csv(SIGLIP_EMB_CSV)\n    emb_df[\"image_path\"] = emb_df[\"image_path\"].apply(lambda x: x.split(\"/\")[-1])\n\n    # Merge embeddings\n    df = train_df.merge(emb_df, on=\"image_path\", how=\"left\")\n\n    # Embedding columns\n    emb_cols = [c for c in df.columns if c.startswith(\"emb\")]\n\n    # Hard safety check\n    if df[emb_cols].isna().any().any():\n        raise RuntimeError(\"NaNs found in embeddings after merge\")\n\n    # PCA\n    X = df[emb_cols].values\n    pca = PCA(n_components=PCA_VARIANCE, random_state=RANDOM_STATE)\n    X_pca = pca.fit_transform(X)\n\n    # KMeans clustering\n    kmeans = KMeans(\n        n_clusters=N_CLUSTERS,\n        random_state=RANDOM_STATE,\n        n_init=10\n    )\n    df[\"visual_cluster\"] = kmeans.fit_predict(X_pca)\n\n    # Weighted biomass target\n    weighted_target = np.zeros(len(df), dtype=np.float32)\n    for t in TARGET_NAMES:\n        weighted_target += df[t].values * weights[t]\n\n    # Bin for stratification\n    y_binned = pd.qcut(\n        weighted_target,\n        q=10,\n        labels=False,\n        duplicates=\"drop\"\n    )\n\n    # StratifiedGroupKFold\n    df[\"fold\"] = -1\n    sgkf = StratifiedGroupKFold(\n        n_splits=N_FOLDS,\n        shuffle=True,\n        random_state=RANDOM_STATE\n    )\n\n    for fold, (_, val_idx) in enumerate(\n        sgkf.split(df, y_binned, groups=df[\"visual_cluster\"])\n    ):\n        df.loc[val_idx, \"fold\"] = fold\n\n    # Final safety check\n    assert (df[\"fold\"] >= 0).all()\n    assert df.groupby(\"visual_cluster\")[\"fold\"].nunique().max() == 1\n\n    # Save\n    df.to_csv(OUT_CSV, index=False)\n    print(\"✅ csiro_data_split.csv created successfully\")\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-26T11:08:15.483285Z","iopub.execute_input":"2025-12-26T11:08:15.484127Z","iopub.status.idle":"2025-12-26T11:08:17.973409Z","shell.execute_reply.started":"2025-12-26T11:08:15.484095Z","shell.execute_reply":"2025-12-26T11:08:17.972372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}