{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedGroupKFold\n\ndef visualize_folds(train_df, config):\n    \"\"\"\n    Create visualizations for fold distribution, species balance, \n    temporal distribution, and state/fold heatmap.\n    \n    Args:\n        train_df (pd.DataFrame): Processed dataframe with 'fold', 'State',\n                                 'Species', and 'Sampling_Date'.\n        config (dict): Configuration containing 'n_folds'.\n    \"\"\"\n\n    n_folds = config.get(\"n_folds\", 5)\n\n    # ---------------------------------------------------------\n    # 1. FOLD DISTRIBUTION\n    # ---------------------------------------------------------\n    plt.figure(figsize=(8,5))\n    train_df['fold'].value_counts().sort_index().plot(kind='bar')\n    plt.title(\"Fold Distribution\")\n    plt.xlabel(\"Fold\")\n    plt.ylabel(\"Number of Samples\")\n    plt.tight_layout()\n    plt.show()\n\n    # ---------------------------------------------------------\n    # 2. SPECIES PER FOLD (TOP 20 for readability)\n    # ---------------------------------------------------------\n    top_species = train_df['Species'].value_counts().head(20).index\n    species_df = train_df[train_df['Species'].isin(top_species)]\n    species_pivot = pd.crosstab(species_df['Species'], species_df['fold'])\n\n    plt.figure(figsize=(12,7))\n    species_pivot.plot(kind='bar', stacked=True, figsize=(12,7))\n    plt.title(\"Top 20 Species Distribution Across Folds\")\n    plt.xlabel(\"Species\")\n    plt.ylabel(\"Count\")\n    plt.legend(title=\"Fold\")\n    plt.tight_layout()\n    plt.show()\n\n    # ---------------------------------------------------------\n    # 3. TEMPORAL DISTRIBUTION FOR EACH FOLD\n    # ---------------------------------------------------------\n    plt.figure(figsize=(10,5))\n    for f in range(n_folds):\n        sub = train_df[train_df['fold'] == f]\n        plt.hist(\n            sub['Sampling_Date'].dropna(),\n            bins=30,\n            alpha=0.5,\n            label=f\"Fold {f}\"\n        )\n\n    plt.title(\"Sampling Date Distribution by Fold\")\n    plt.xlabel(\"Date\")\n    plt.ylabel(\"Frequency\")\n    plt.legend()\n    plt.tight_layout()\n    plt.show()\n\n    # ---------------------------------------------------------\n    # 4. STATE × FOLD HEATMAP\n    # ---------------------------------------------------------\n    state_fold = pd.crosstab(train_df[\"State\"], train_df[\"fold\"])\n\n    plt.figure(figsize=(10,6))\n    plt.imshow(state_fold, aspect='auto')\n    plt.colorbar(label=\"Counts\")\n    plt.xticks(range(n_folds), state_fold.columns)\n    plt.yticks(range(len(state_fold.index)), state_fold.index)\n    plt.title(\"State × Fold Heatmap\")\n    plt.xlabel(\"Fold\")\n    plt.ylabel(\"State\")\n    plt.tight_layout()\n    plt.show()\n\n    print(\"✔ All fold visualizations generated.\")\n\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:04.839362Z","iopub.execute_input":"2025-12-08T02:27:04.839645Z","iopub.status.idle":"2025-12-08T02:27:09.238017Z","shell.execute_reply.started":"2025-12-08T02:27:04.839621Z","shell.execute_reply":"2025-12-08T02:27:09.236792Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------------------------------------------------------\n# LOAD DATA\n# ---------------------------------------------------------\nprint(\"\\nLoading data...\")\ntrain_df_raw = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntrain_df_raw['Sampling_Date'] = pd.to_datetime(train_df_raw.get('Sampling_Date'), errors='coerce')\n\n# Pivot target columns\ntarget_pivot = pd.pivot_table(train_df_raw, index='image_path',\n                              columns='target_name', values='target').reset_index()\n\n# Metadata columns\nmeta_df = train_df_raw[['image_path', 'State', 'Sampling_Date',\n                        'Species', 'Height_Ave_cm', 'Pre_GSHH_NDVI']].drop_duplicates(subset=['image_path'])\n\n# Merge\nbase_df = target_pivot.merge(meta_df, on='image_path', how='left')\n\n\n# ---------------------------------------------------------\n# FUNCTION TO CREATE FOLDS\n# ---------------------------------------------------------\ndef create_folds(df, n_folds):\n    df = df.sort_values(['State', 'Sampling_Date']).reset_index(drop=True)\n    df['fold'] = -1\n\n    for state in df['State'].dropna().unique():\n        sub = df[df['State'] == state]\n        dates = sub['Sampling_Date'].dropna().unique()\n\n        if len(dates) == 0:\n            continue\n\n        date_folds = np.array_split(dates, n_folds)\n        for fold_idx, date_chunk in enumerate(date_folds):\n            idx = sub[sub['Sampling_Date'].isin(date_chunk)].index\n            df.loc[idx, 'fold'] = fold_idx\n\n    # Assign leftover samples\n    nofold_idx = df[df['fold'] == -1].index\n    for i, idx in enumerate(nofold_idx):\n        df.at[idx, 'fold'] = i % n_folds\n\n    return df\n\n \nfor n_folds in [5, 10]:\n    print(f\"\\nCreating {n_folds} folds...\")\n    df_folds = create_folds(base_df.copy(), n_folds)\n\n    print(f\"Fold distribution for {n_folds} folds:\")\n    print(df_folds['fold'].value_counts().sort_index())\n\n    # Save CSV\n    out_name = f\"folds_train_v1_{n_folds}.csv\"\n    df_folds.to_csv(out_name, index=False)\n    print(f\"✔ Saved: {out_name}\")\n    visualize_folds(df_folds, {\"n_folds\": n_folds})","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:09.240031Z","iopub.execute_input":"2025-12-08T02:27:09.240515Z","iopub.status.idle":"2025-12-08T02:27:13.216999Z","shell.execute_reply.started":"2025-12-08T02:27:09.240486Z","shell.execute_reply":"2025-12-08T02:27:13.216038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nprint(\"\\nLoading data...\")\ntrain_df_raw = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntrain_df_raw['Sampling_Date'] = pd.to_datetime(train_df_raw.get('Sampling_Date'), errors='coerce')\n\n# Pivot target columns\ntarget_pivot = pd.pivot_table(train_df_raw, index='image_path',\n                              columns='target_name', values='target').reset_index()\nmeta_df = train_df_raw[['image_path', 'State', 'Sampling_Date',\n                        'Species', 'Height_Ave_cm', 'Pre_GSHH_NDVI']].drop_duplicates(subset=['image_path'])\n\nbase_df = target_pivot.merge(meta_df, on='image_path', how='left')\ndef create_state_temporal_folds(train_df, n_folds):\n    train_df = train_df.sort_values(['State', 'Sampling_Date']).reset_index(drop=True)\n    train_df['fold'] = -1\n\n    for state in train_df['State'].dropna().unique():\n\n        state_mask = train_df['State'] == state\n        state_df = train_df[state_mask]\n\n        # Group by date → count samples\n        date_groups = state_df.groupby('Sampling_Date').size().sort_index()\n\n        # If there are fewer dates than folds, limit folds\n        effective_folds = min(len(date_groups), n_folds)\n\n        total_samples = date_groups.sum()\n        samples_per_fold = total_samples / effective_folds\n\n        cumsum = 0\n        current_fold = 0\n\n        for date, count in date_groups.items():\n\n            # Assign all samples from that date to the current fold\n            idx = state_df[state_df['Sampling_Date'] == date].index\n            train_df.loc[idx, 'fold'] = current_fold\n\n            cumsum += count\n\n            # Move to next fold when threshold reached\n            if (cumsum >= samples_per_fold * (current_fold + 1) and\n                current_fold < effective_folds - 1):\n                current_fold += 1\n\n    # If any leftover, assign round-robin\n    nofold_idx = train_df[train_df['fold'] == -1].index\n    for i, idx in enumerate(nofold_idx):\n        train_df.at[idx, 'fold'] = i % n_folds\n\n    return train_df\nfor n_folds in [5, 10]:\n    print(f\"\\nCreating temporal folds with {n_folds} folds...\")\n    df_folds = create_state_temporal_folds(base_df.copy(), n_folds) \n    print(f\"\\nFold distribution ({n_folds} folds):\")\n    print(df_folds['fold'].value_counts().sort_index())\n    visualize_folds(df_folds, {\"n_folds\": n_folds})\n    out_name = f\"folds_train_v2_{n_folds}.csv\"\n    df_folds.to_csv(out_name, index=False)\n    print(f\"✔ Saved: {out_name}\")\n\nprint(\"\\n✔ Done! Both 5-fold and 10-fold temporal grouped datasets created.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:13.218044Z","iopub.execute_input":"2025-12-08T02:27:13.218310Z","iopub.status.idle":"2025-12-08T02:27:16.689910Z","shell.execute_reply.started":"2025-12-08T02:27:13.218281Z","shell.execute_reply":"2025-12-08T02:27:16.688929Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## v3","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nprint(\"\\nCreating stratified temporal folds...\")\n\ntrain_df_raw = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntrain_df_raw['Sampling_Date'] = pd.to_datetime(train_df_raw.get('Sampling_Date'), errors='coerce')\n\ntarget_pivot = pd.pivot_table(train_df_raw, index='image_path',\n                              columns='target_name', values='target').reset_index()\n\nmeta_df = train_df_raw[['image_path', 'State', 'Sampling_Date',\n                        'Species', 'Height_Ave_cm', 'Pre_GSHH_NDVI']].drop_duplicates(subset=['image_path'])\n\nbase_df = target_pivot.merge(meta_df, on='image_path', how='left')\n\n\ndef create_stratified_temporal_folds(df, n_folds):\n    df = df.sort_values(['State', 'Species', 'Sampling_Date']).reset_index(drop=True)\n    df['fold'] = -1\n\n    for (state, species), sub in df.groupby(['State', 'Species']):\n        sub = sub.sort_values('Sampling_Date')\n        idxs = sub.index.tolist()\n        for i, idx in enumerate(idxs):\n            df.at[idx, 'fold'] = i % n_folds\n\n    nofold_idx = df[df['fold'] == -1].index\n    for i, idx in enumerate(nofold_idx):\n        df.at[idx, 'fold'] = i % n_folds\n\n    return df \n\nfor n_folds in [5, 10]:\n    df_folds = create_stratified_temporal_folds(base_df.copy(), n_folds)\n    print(f\"\\nFold distribution ({n_folds} folds):\")\n    print(df_folds['fold'].value_counts().sort_index())\n\n    out_name = f\"folds_train_v3_{n_folds}.csv\"\n    df_folds.to_csv(out_name, index=False)\n    print(\"Saved:\", out_name)\n    visualize_folds(df_folds, {\"n_folds\": n_folds})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:16.690917Z","iopub.execute_input":"2025-12-08T02:27:16.691228Z","iopub.status.idle":"2025-12-08T02:27:20.159485Z","shell.execute_reply.started":"2025-12-08T02:27:16.691198Z","shell.execute_reply":"2025-12-08T02:27:20.158629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_folds.columns\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:20.160451Z","iopub.execute_input":"2025-12-08T02:27:20.161307Z","iopub.status.idle":"2025-12-08T02:27:20.167181Z","shell.execute_reply.started":"2025-12-08T02:27:20.161275Z","shell.execute_reply":"2025-12-08T02:27:20.166449Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## VERSION 5 folds with stratifying on 3 labels and group on state ad sampling date","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport random\n\n# ----------------------------\n# Load + Build Base DF\n# ----------------------------\ntrain_df_raw = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntrain_df_raw['Sampling_Date'] = pd.to_datetime(train_df_raw['Sampling_Date'], errors='coerce')\n\ntarget_pivot = pd.pivot_table(\n    train_df_raw, index='image_path',\n    columns='target_name', values='target'\n).reset_index()\n\nmeta_df = train_df_raw[['image_path','State','Sampling_Date',\n                        'Species','Height_Ave_cm','Pre_GSHH_NDVI']].drop_duplicates('image_path')\n\nbase_df = target_pivot.merge(meta_df, on='image_path', how='left')\n\nTARGET_COLS = ['Dry_Clover_g','Dry_Dead_g','Dry_Green_g']\n\n# ----------------------------\n# Label Signature\n# ----------------------------\ndef build_label_signature(df, bins=4):\n    df = df.copy()\n    for col in TARGET_COLS:\n        df[col+\"_bin\"] = pd.qcut(df[col], q=bins, labels=False, duplicates=\"drop\")\n    df[\"label_sig\"] = df[[c+\"_bin\" for c in TARGET_COLS]].astype(str).agg(\"_\".join, axis=1)\n    return df\n\n# ----------------------------\n# Fold Assignment\n# ----------------------------\ndef create_stratified_temporal_folds(df, n_folds):\n    df = build_label_signature(df)\n    df = df.sort_values(['State','Species','Sampling_Date']).reset_index(drop=True)\n    df['fold'] = -1\n\n    for _, sub in df.groupby(['State','Species','label_sig']):\n        sub = sub.sort_values('Sampling_Date')\n        idxs = sub.index.tolist()\n        start = random.randint(0, n_folds - 1)\n        for i, idx in enumerate(idxs):\n            df.at[idx, 'fold'] = (start + i) % n_folds\n\n    nofold = df[df['fold'] == -1].index\n    for i, idx in enumerate(nofold):\n        df.at[idx, 'fold'] = i % n_folds\n\n    return df\n\n# ----------------------------\n# Visualizations\n# ----------------------------\ndef visualize_folds_n(df, cfg):\n    n_folds = cfg[\"n_folds\"]\n\n    # 1. Fold distribution\n    plt.figure(figsize=(7,4))\n    df['fold'].value_counts().sort_index().plot(kind='bar')\n    plt.title(\"Fold Distribution\")\n    plt.tight_layout()\n    plt.show()\n\n    # 2. Top 20 species\n    top_species = df['Species'].value_counts().head(20).index\n    species_pivot = pd.crosstab(df[df['Species'].isin(top_species)]['Species'], df['fold'])\n    species_pivot.plot(kind='bar', stacked=True, figsize=(12,6))\n    plt.title(\"Top 20 Species per Fold\")\n    plt.tight_layout()\n    plt.show()\n\n    # 3. Temporal sampling distribution\n    plt.figure(figsize=(10,4))\n    for f in range(n_folds):\n        plt.hist(df[df['fold']==f]['Sampling_Date'].dropna(), bins=30, alpha=0.5, label=f\"Fold {f}\")\n    plt.title(\"Sampling Date Distribution per Fold\")\n    plt.legend()\n    plt.tight_layout()\n    plt.show()\n\n    # 4. State × Fold heatmap\n    state_fold = pd.crosstab(df[\"State\"], df[\"fold\"])\n    plt.figure(figsize=(10,5))\n    plt.imshow(state_fold, aspect='auto')\n    plt.colorbar()\n    plt.xticks(range(n_folds), state_fold.columns)\n    plt.yticks(range(len(state_fold.index)), state_fold.index)\n    plt.title(\"State × Fold\")\n    plt.tight_layout()\n    plt.show()\n\n    # 5. Target value distribution per fold\n    for col in TARGET_COLS:\n        plt.figure(figsize=(8,4))\n        for f in range(n_folds):\n            plt.hist(df[df['fold']==f][col], bins=30, alpha=0.4, label=f\"Fold {f}\")\n        plt.title(f\"{col} Distribution per Fold\")\n        plt.legend()\n        plt.tight_layout()\n        plt.show()\n\n    # 6. Label signature distribution per fold\n    sig_pivot = pd.crosstab(df['label_sig'], df['fold'])\n    sig_pivot.plot(kind=\"bar\", stacked=True, figsize=(14,6))\n    plt.title(\"Label Signature Distribution per Fold\")\n    plt.tight_layout()\n    plt.show()\n\n# ----------------------------\n# Main Loop\n# ----------------------------\nfor n_folds in [5, 10]:\n    df_folds = create_stratified_temporal_folds(base_df.copy(), n_folds)\n    print(f\"\\nFold distribution ({n_folds} folds):\")\n    print(df_folds['fold'].value_counts().sort_index())\n\n    out = f\"folds_train_v5_{n_folds}.csv\"\n    df_folds.to_csv(out, index=False)\n    print(\"Saved:\", out)\n\n    visualize_folds_n(df_folds, {\"n_folds\": n_folds})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:20.168410Z","iopub.execute_input":"2025-12-08T02:27:20.168759Z","iopub.status.idle":"2025-12-08T02:27:29.273463Z","shell.execute_reply.started":"2025-12-08T02:27:20.168736Z","shell.execute_reply":"2025-12-08T02:27:29.272576Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## AYAAN FOLDS V4","metadata":{}},{"cell_type":"code","source":"def build_or_load_groupkfold_splits(\n    train_df: pd.DataFrame,\n    groups: np.ndarray,\n    stratify_labels: np.ndarray,\n    n_splits: int,\n    cache_dir: str,\n    seed: int = 42,\n) -> pd.DataFrame:\n    \"\"\"\n    Group + Stratify KFold:\n      - groups : e.g. Sampling_Date + State (group_key)\n      - stratify_labels : Dry_Total_g quantile bin (int label)\n    구현은 StratifiedGroupKFold 사용.\n    \"\"\"\n    os.makedirs(cache_dir, exist_ok=True)\n    cache_fp = os.path.join(cache_dir, f\"splits_sgkf{n_splits}_seed{seed}.csv\")\n\n    if os.path.exists(cache_fp):\n        try:\n            cached = pd.read_csv(cache_fp)\n            cur_ids = set(train_df[\"image_path\"].astype(str).tolist())\n            old_ids = set(cached[\"image_path\"].astype(str).tolist())\n            if cur_ids == old_ids and \"fold\" in cached.columns:\n                train_df = train_df.merge(\n                    cached[[\"image_path\", \"fold\"]], on=\"image_path\", how=\"left\"\n                )\n                if train_df[\"fold\"].notna().all():\n                    print(f(\"[SplitCache] Load cached Group+Stratified splits...\"))\n                    train_df[\"fold\"] = train_df[\"fold\"].astype(int)\n                    return train_df\n            else:\n                print(\"[SplitCache] Dataset changed. Rebuilding Group+Stratified splits...\")\n        except Exception as e:\n            print(f\"[SplitCache] Failed to read cache ({e}). Rebuilding...\")\n\n    sgkf = StratifiedGroupKFold(\n        n_splits=n_splits, shuffle=True, random_state=seed\n    )\n    fold_arr = np.full(len(train_df), -1, dtype=int)\n    for f, (_, valid_index) in enumerate(\n        sgkf.split(train_df, stratify_labels, groups=groups)\n    ):\n        fold_arr[valid_index] = f\n\n    train_df = train_df.copy()\n    train_df[\"fold\"] = fold_arr\n\n    train_df[[\"image_path\", \"fold\"]].to_csv(cache_fp, index=False)\n    print(f\"[SplitCache] Saved Group+Stratified splits: {cache_fp}\")\n    return train_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:29.275532Z","iopub.execute_input":"2025-12-08T02:27:29.275871Z","iopub.status.idle":"2025-12-08T02:27:29.285952Z","shell.execute_reply.started":"2025-12-08T02:27:29.275849Z","shell.execute_reply":"2025-12-08T02:27:29.284827Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    # Paths\n    seed = 42\n    n_splits = 4\n    stratify_n_bins = 5\n    cache_dir = \"cache\"\n\nDATA_ROOT = \"/kaggle/input/csiro-biomass\"\n\nbase_train_df = pd.read_csv(os.path.join(DATA_ROOT, \"train.csv\"))\npivot_df = base_train_df.pivot(index=\"image_path\", columns=\"target_name\", values=\"target\").reset_index()\nmeta_keep = [\"image_path\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\", \"Sampling_Date\", \"State\", \"Species\"]\navailable_meta = [c for c in meta_keep if c in base_train_df.columns]\nother_cols = base_train_df[available_meta].drop_duplicates(\"image_path\").set_index(\"image_path\")\ntrain_df = pivot_df.merge(other_cols, on=\"image_path\", how=\"left\")\n# === group_key (Sampling_Date + State) : SGKF용 ===\nmeta_group = base_train_df[[\"image_path\", \"Sampling_Date\", \"State\"]].drop_duplicates(\"image_path\")\nmeta_group[\"group_key\"] = meta_group[\"Sampling_Date\"].astype(str) + \" \" + meta_group[\"State\"].astype(str)\ntrain_df = train_df.merge(meta_group[[\"image_path\", \"group_key\"]], on=\"image_path\", how=\"left\")\n\ndt = train_df[\"Dry_Total_g\"].astype(float)\ntrain_df[\"stratify_bin\"] = pd.qcut(\n    dt,\n    q=CFG.stratify_n_bins,\n    labels=False,\n    duplicates=\"drop\"\n).astype(int)\ngroups = train_df[\"group_key\"].values\nstratify_labels = train_df[\"stratify_bin\"].values\ntrain_df = build_or_load_groupkfold_splits(\n    train_df,\n    groups=groups,\n    stratify_labels=stratify_labels,\n    n_splits=CFG.n_splits,\n    cache_dir=CFG.cache_dir,\n    seed=CFG.seed,\n)\n\nout_name = f\"folds_train_v4_{CFG.n_splits}.csv\"\ntrain_df.to_csv(out_name, index=False)\nvisualize_folds(train_df, {\"n_folds\": 4})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:29.287243Z","iopub.execute_input":"2025-12-08T02:27:29.287663Z","iopub.status.idle":"2025-12-08T02:27:30.823942Z","shell.execute_reply.started":"2025-12-08T02:27:29.287626Z","shell.execute_reply":"2025-12-08T02:27:30.822945Z"},"collapsed":true,"jupyter":{"source_hidden":true,"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## V6 state sampling date","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport random\n\n# ----------------------------\n# Load + Build Base DF\n# ----------------------------\ntrain_df_raw = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntrain_df_raw['Sampling_Date'] = pd.to_datetime(train_df_raw['Sampling_Date'], errors='coerce')\n\ntarget_pivot = pd.pivot_table(\n    train_df_raw, index='image_path',\n    columns='target_name', values='target'\n).reset_index()\n\nmeta_df = train_df_raw[['image_path','State','Sampling_Date',\n                        'Species','Height_Ave_cm','Pre_GSHH_NDVI']].drop_duplicates('image_path')\n\nbase_df = target_pivot.merge(meta_df, on='image_path', how='left')\n\nTARGET_COLS = ['Dry_Clover_g','Dry_Dead_g','Dry_Green_g']\n\n# ----------------------------\n# Label Signature\n# ----------------------------\ndef build_label_signature(df, bins=4):\n    df = df.copy()\n    for col in TARGET_COLS:\n        df[col+\"_bin\"] = pd.qcut(df[col], q=bins, labels=False, duplicates=\"drop\")\n    df[\"label_sig\"] = df[[c+\"_bin\" for c in TARGET_COLS]].astype(str).agg(\"_\".join, axis=1)\n    return df\n\n# ----------------------------\n# Fold Assignment\n# ----------------------------\ndef create_stratified_temporal_folds(df, n_folds):\n    df = build_label_signature(df)\n    df = df.sort_values(['State','Sampling_Date']).reset_index(drop=True)\n    df['fold'] = -1\n\n    for _, sub in df.groupby(['State','label_sig']):\n        sub = sub.sort_values('Sampling_Date')\n        idxs = sub.index.tolist()\n        start = random.randint(0, n_folds - 1)\n        for i, idx in enumerate(idxs):\n            df.at[idx, 'fold'] = (start + i) % n_folds\n\n    nofold = df[df['fold'] == -1].index\n    for i, idx in enumerate(nofold):\n        df.at[idx, 'fold'] = i % n_folds\n\n    return df\n\n# ----------------------------\n# Visualizations\n# ----------------------------\ndef visualize_folds_n(df, cfg):\n    n_folds = cfg[\"n_folds\"]\n\n    # 1. Fold distribution\n    plt.figure(figsize=(7,4))\n    df['fold'].value_counts().sort_index().plot(kind='bar')\n    plt.title(\"Fold Distribution\")\n    plt.tight_layout()\n    plt.show()\n\n    # 2. Top 20 species\n    top_species = df['Species'].value_counts().head(20).index\n    species_pivot = pd.crosstab(df[df['Species'].isin(top_species)]['Species'], df['fold'])\n    species_pivot.plot(kind='bar', stacked=True, figsize=(12,6))\n    plt.title(\"Top 20 Species per Fold\")\n    plt.tight_layout()\n    plt.show()\n\n    # 3. Temporal sampling distribution\n    plt.figure(figsize=(10,4))\n    for f in range(n_folds):\n        plt.hist(df[df['fold']==f]['Sampling_Date'].dropna(), bins=30, alpha=0.5, label=f\"Fold {f}\")\n    plt.title(\"Sampling Date Distribution per Fold\")\n    plt.legend()\n    plt.tight_layout()\n    plt.show()\n\n    # 4. State × Fold heatmap\n    state_fold = pd.crosstab(df[\"State\"], df[\"fold\"])\n    plt.figure(figsize=(10,5))\n    plt.imshow(state_fold, aspect='auto')\n    plt.colorbar()\n    plt.xticks(range(n_folds), state_fold.columns)\n    plt.yticks(range(len(state_fold.index)), state_fold.index)\n    plt.title(\"State × Fold\")\n    plt.tight_layout()\n    plt.show()\n\n    # 5. Target value distribution per fold\n    for col in TARGET_COLS:\n        plt.figure(figsize=(8,4))\n        for f in range(n_folds):\n            plt.hist(df[df['fold']==f][col], bins=30, alpha=0.4, label=f\"Fold {f}\")\n        plt.title(f\"{col} Distribution per Fold\")\n        plt.legend()\n        plt.tight_layout()\n        plt.show()\n\n    # 6. Label signature distribution per fold\n    sig_pivot = pd.crosstab(df['label_sig'], df['fold'])\n    sig_pivot.plot(kind=\"bar\", stacked=True, figsize=(14,6))\n    plt.title(\"Label Signature Distribution per Fold\")\n    plt.tight_layout()\n    plt.show()\n\n# ----------------------------\n# Main Loop\n# ----------------------------\nfor n_folds in [5, 10]:\n    df_folds = create_stratified_temporal_folds(base_df.copy(), n_folds)\n    print(f\"\\nFold distribution ({n_folds} folds):\")\n    print(df_folds['fold'].value_counts().sort_index())\n\n    out = f\"folds_train_v6_{n_folds}.csv\"\n    df_folds.to_csv(out, index=False)\n    print(\"Saved:\", out)\n\n    visualize_folds_n(df_folds, {\"n_folds\": n_folds})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T02:27:30.825013Z","iopub.execute_input":"2025-12-08T02:27:30.825316Z","iopub.status.idle":"2025-12-08T02:27:39.923745Z","shell.execute_reply.started":"2025-12-08T02:27:30.825293Z","shell.execute_reply":"2025-12-08T02:27:39.922698Z"}},"outputs":[],"execution_count":null}]}