{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14197987,"sourceType":"datasetVersion","datasetId":9054328},{"sourceId":14198331,"sourceType":"datasetVersion","datasetId":9054617},{"sourceId":14204319,"sourceType":"datasetVersion","datasetId":9059370},{"sourceId":14207298,"sourceType":"datasetVersion","datasetId":9061693},{"sourceId":14210833,"sourceType":"datasetVersion","datasetId":9064416},{"sourceId":14219969,"sourceType":"datasetVersion","datasetId":9070859},{"sourceId":14222560,"sourceType":"datasetVersion","datasetId":9072749},{"sourceId":14241859,"sourceType":"datasetVersion","datasetId":9086189},{"sourceId":14241912,"sourceType":"datasetVersion","datasetId":9086219}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import KFold, GroupKFold\n\n# ---------------- 配置 ----------------\n# 请确保这里是你数据的真实路径\nDATA_ROOT = '/kaggle/input/csiro-biomass' \nTRAIN_CSV = os.path.join(DATA_ROOT, 'train.csv')\nSEED = 42\nN_FOLD = 5\n\ndef analyze_data():\n    if not os.path.exists(TRAIN_CSV):\n        print(f\"❌ 找不到文件: {TRAIN_CSV}\")\n        return\n\n    print(\"正在读取并处理数据...\")\n    df = pd.read_csv(TRAIN_CSV)\n    \n    # 1. 数据预处理 (转为宽表)\n    df['id_prefix'] = df['sample_id'].apply(lambda x: x.split('__')[0])\n    pivot_df = df.pivot_table(index='id_prefix', columns='target_name', values='target').reset_index()\n    \n    # 提取元数据 (关键是 Sampling_Date)\n    meta_cols = ['Sampling_Date', 'State']\n    meta_df = df.groupby('id_prefix')[meta_cols].first().reset_index()\n    \n    agg_df = pd.merge(pivot_df, meta_df, on='id_prefix')\n    print(f\"样本总数: {len(agg_df)}\")\n\n    # ==========================================\n    # 2. 揭秘：为什么 CV 虚高？(检查时间泄露)\n    # ==========================================\n    print(\"\\n\" + \"=\"*50)\n    print(\"🕵️‍♀️ 侦探报告：检查随机打乱(Random KFold)的泄露情况\")\n    print(\"=\"*50)\n    \n    kf = KFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)\n    \n    for fold, (train_idx, val_idx) in enumerate(kf.split(agg_df)):\n        train_data = agg_df.iloc[train_idx]\n        val_data = agg_df.iloc[val_idx]\n        \n        # 检查日期重叠\n        train_dates = set(train_data['Sampling_Date'].unique())\n        val_dates = set(val_data['Sampling_Date'].unique())\n        overlap = train_dates.intersection(val_dates)\n        \n        leakage_pct = len(overlap) / len(val_dates) * 100\n        \n        print(f\"[Fold {fold+1}]\")\n        print(f\"  - 验证集共有 {len(val_dates)} 个拍摄日期\")\n        print(f\"  - 其中 {len(overlap)} 个日期也在训练集中出现过\")\n        print(f\"  - 🔴 泄露比例: {leakage_pct:.1f}%\")\n        \n        if leakage_pct > 50:\n            print(\"    -> 结论：严重泄露！模型在'背答案'而不是'预测'。\")\n            break # 只看一折就够了\n\n    # ==========================================\n    # 3. 可视化：类别分布难度\n    # ==========================================\n    print(\"\\n\" + \"=\"*50)\n    print(\"📊 可视化：各个类别的预测难度\")\n    print(\"=\"*50)\n    \n    targets = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g']\n    fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n    \n    for i, col in enumerate(targets):\n        data = agg_df[col]\n        # 画分布图\n        sns.histplot(data, bins=50, kde=True, ax=axes[i], color='skyblue')\n        \n        # 统计零值 (难点)\n        zeros = (data < 0.1).sum()\n        zero_pct = zeros / len(data) * 100\n        mean_val = data.mean()\n        max_val = data.max()\n        \n        axes[i].set_title(f\"{col}\\nMean: {mean_val:.1f} | Max: {max_val:.1f}\")\n        axes[i].set_xlabel(f\"Biomass (g)\\n(Zeros: {zero_pct:.1f}%)\")\n        \n        print(f\"分析 {col}:\")\n        print(f\"  - 平均值: {mean_val:.2f}g, 最大值: {max_val:.2f}g\")\n        print(f\"  - 零值样本占比: {zero_pct:.1f}%\")\n        if zero_pct > 30:\n            print(\"    -> 难点：这是一个'零膨胀'问题，模型很容易为了稳妥预测一个小正数而导致误差。\")\n            \n    plt.tight_layout()\n    plt.show()\n\nif __name__ == '__main__':\n    analyze_data()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T16:06:35.209514Z","iopub.execute_input":"2025-12-19T16:06:35.209853Z","iopub.status.idle":"2025-12-19T16:06:37.585971Z","shell.execute_reply.started":"2025-12-19T16:06:35.209828Z","shell.execute_reply":"2025-12-19T16:06:37.585111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# ==========================================\n# 配置\n# ==========================================\nDATA_ROOT = '/kaggle/input/csiro-biomass'\n# 如果在本地运行，请修改为你的路径\nif not os.path.exists(DATA_ROOT):\n    DATA_ROOT = r'D:\\Downloads\\CSIRO - Image2生物质预测\\csiro-biomass'\n\nTRAIN_CSV = os.path.join(DATA_ROOT, 'train.csv')\n\ndef perform_deep_eda():\n    if not os.path.exists(TRAIN_CSV):\n        print(f\"❌ 找不到文件: {TRAIN_CSV}\")\n        return\n\n    print(\"正在加载全量数据...\")\n    df = pd.read_csv(TRAIN_CSV)\n    \n    # 1. 数据重构 (Long -> Wide)\n    # 原始数据是长格式，我们需要把 targets 变成列\n    df['id_prefix'] = df['sample_id'].apply(lambda x: x.split('__')[0])\n    \n    # 提取 Target 数值\n    pivot_df = df.pivot_table(index='id_prefix', columns='target_name', values='target').reset_index()\n    \n    # 提取并合并元数据 (Date, State, Species, Height, NDVI)\n    # 注意：Height 和 NDVI 可能有些行是 NaN，需要注意\n    meta_cols = ['Sampling_Date', 'State', 'Species', 'Height_Ave_cm', 'Pre_GSHH_NDVI']\n    # 仅保留存在的列\n    valid_meta = [c for c in meta_cols if c in df.columns]\n    \n    meta_df = df.groupby('id_prefix')[valid_meta].first().reset_index()\n    full_df = pd.merge(pivot_df, meta_df, on='id_prefix')\n    \n    # 解析日期\n    if 'Sampling_Date' in full_df.columns:\n        full_df['Sampling_Date'] = pd.to_datetime(full_df['Sampling_Date'], format='mixed', dayfirst=True)\n        full_df['Month'] = full_df['Sampling_Date'].dt.month\n        full_df['Year'] = full_df['Sampling_Date'].dt.year\n\n    print(f\"✅ 数据重构完成，共 {len(full_df)} 个唯一样本。\")\n    \n    # ==========================================\n    # 1. 目标变量的相关性 (Correlation)\n    # ==========================================\n    print(\"\\n📊 1. 变量相关性分析\")\n    # 选取数值型列\n    num_cols = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'Height_Ave_cm', 'Pre_GSHH_NDVI']\n    # 过滤掉不存在的列\n    num_cols = [c for c in num_cols if c in full_df.columns]\n    \n    corr = full_df[num_cols].corr()\n    \n    plt.figure(figsize=(8, 6))\n    sns.heatmap(corr, annot=True, cmap='coolwarm', fmt=\".2f\", linewidths=0.5)\n    plt.title(\"Correlation Heatmap: Biomass vs Metadata\")\n    plt.show()\n    \n    # ==========================================\n    # 2. 时间分布 (Seasonality)\n    # ==========================================\n    print(\"\\n📅 2. 时间分布分析\")\n    if 'Month' in full_df.columns:\n        fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n        \n        # 月份分布\n        sns.countplot(data=full_df, x='Month', hue='Year', ax=axes[0], palette='viridis')\n        axes[0].set_title(\"Sample Count by Month & Year\")\n        \n        # 生物量随月份的变化\n        # 把数据融化以便画图\n        melted = full_df.melt(id_vars=['Month'], value_vars=['Dry_Green_g', 'Dry_Dead_g'], var_name='Type', value_name='Biomass')\n        sns.boxplot(data=melted, x='Month', y='Biomass', hue='Type', ax=axes[1], showfliers=False)\n        axes[1].set_title(\"Green vs Dead Biomass by Month\")\n        \n        plt.tight_layout()\n        plt.show()\n    \n    # ==========================================\n    # 3. 三叶草 (Clover) 的特殊性\n    # ==========================================\n    print(\"\\n🍀 3. 三叶草深度分析\")\n    plt.figure(figsize=(12, 5))\n    \n    # 三叶草与草高的关系\n    plt.subplot(1, 2, 1)\n    sns.scatterplot(data=full_df, x='Height_Ave_cm', y='Dry_Clover_g', alpha=0.6, color='green')\n    plt.title(\"Clover Biomass vs Grass Height\")\n    plt.xlabel(\"Height (cm)\")\n    plt.ylabel(\"Clover (g)\")\n    \n    # 不同州的三叶草分布\n    plt.subplot(1, 2, 2)\n    sns.boxplot(data=full_df, x='State', y='Dry_Clover_g', showfliers=False)\n    plt.title(\"Clover Biomass by State\")\n    plt.xticks(rotation=45)\n    \n    plt.tight_layout()\n    plt.show()\n    \n    # ==========================================\n    # 4. 极端值检查 (Outliers)\n    # ==========================================\n    print(\"\\n⚠️ 4. 极端值 (Top 5 Samples)\")\n    for col in ['Dry_Green_g', 'Dry_Total_g']:\n        if col in full_df.columns:\n            print(f\"--- Top 5 {col} ---\")\n            print(full_df.nlargest(5, col)[['id_prefix', 'State', col]])\n\nif __name__ == '__main__':\n    perform_deep_eda()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T16:12:28.471561Z","iopub.execute_input":"2025-12-19T16:12:28.471905Z","iopub.status.idle":"2025-12-19T16:12:29.806596Z","shell.execute_reply.started":"2025-12-19T16:12:28.471875Z","shell.execute_reply":"2025-12-19T16:12:29.805661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import GroupKFold\nimport matplotlib.dates as mdates\n\n# ==========================================\n# 配置\n# ==========================================\nDATA_ROOT = '/kaggle/input/csiro-biomass'\n# 本地路径备用\nif not os.path.exists(DATA_ROOT):\n    DATA_ROOT = r'D:\\Downloads\\CSIRO - Image2生物质预测\\csiro-biomass'\n    \nTRAIN_CSV = os.path.join(DATA_ROOT, 'train.csv')\nN_FOLD = 5\n\ndef check_group_split():\n    if not os.path.exists(TRAIN_CSV):\n        print(f\"❌ 找不到文件: {TRAIN_CSV}\")\n        return\n\n    print(\"正在加载数据...\")\n    df = pd.read_csv(TRAIN_CSV)\n    \n    # 1. 数据预处理\n    df['id_prefix'] = df['sample_id'].apply(lambda x: x.split('__')[0])\n    pivot_df = df.pivot_table(index='id_prefix', columns='target_name', values='target').reset_index()\n    \n    # 提取 Sampling_Date\n    meta_df = df.groupby('id_prefix')[['Sampling_Date']].first().reset_index()\n    agg_df = pd.merge(pivot_df, meta_df, on='id_prefix')\n    \n    # 解析日期\n    agg_df['Sampling_Date'] = pd.to_datetime(agg_df['Sampling_Date'], format='mixed', dayfirst=True)\n    # 按日期排序，方便观察\n    agg_df = agg_df.sort_values('Sampling_Date')\n    \n    print(f\"样本总数: {len(agg_df)}\")\n    print(f\"日期范围: {agg_df['Sampling_Date'].min().date()} 到 {agg_df['Sampling_Date'].max().date()}\")\n\n    # ==========================================\n    # 2. 执行 GroupKFold\n    # ==========================================\n    gkf = GroupKFold(n_splits=N_FOLD)\n    groups = agg_df['Sampling_Date']\n    \n    # 保存每一折的统计信息\n    fold_stats = []\n    \n    # 创建画布：左边画时间轴，右边画分布图\n    fig = plt.figure(figsize=(20, 4 * N_FOLD))\n    \n    for fold, (train_idx, val_idx) in enumerate(gkf.split(agg_df, groups=groups)):\n        train_data = agg_df.iloc[train_idx]\n        val_data = agg_df.iloc[val_idx]\n        \n        # --- 统计信息 ---\n        train_dates = set(train_data['Sampling_Date'].unique())\n        val_dates = set(val_data['Sampling_Date'].unique())\n        overlap = train_dates.intersection(val_dates)\n        \n        # 记录关键指标\n        stats = {\n            'Fold': fold + 1,\n            'Train_Samples': len(train_data),\n            'Val_Samples': len(val_data),\n            'Leakage_Days': len(overlap),\n            'Val_Mean_Green': val_data['Dry_Green_g'].mean(),\n            'Train_Mean_Green': train_data['Dry_Green_g'].mean()\n        }\n        fold_stats.append(stats)\n        \n        # --- 可视化 1: 时间轴分布 (Timeline) ---\n        ax1 = plt.subplot(N_FOLD, 2, fold * 2 + 1)\n        \n        # 绘制散点：y轴是Fold编号(其实只有1个)，x轴是时间\n        # 训练集用蓝色小点，验证集用红色大点\n        ax1.scatter(train_data['Sampling_Date'], [1]*len(train_data), \n                   c='blue', alpha=0.1, s=20, label='Train')\n        ax1.scatter(val_data['Sampling_Date'], [1]*len(val_data), \n                   c='red', alpha=0.8, s=50, label='Validation', marker='x')\n        \n        ax1.set_title(f\"Fold {fold+1}: Temporal Split (Time Axis)\")\n        ax1.set_yticks([])\n        ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))\n        ax1.legend(loc='upper right')\n        \n        # --- 可视化 2: 生物量分布对比 (Distribution) ---\n        ax2 = plt.subplot(N_FOLD, 2, fold * 2 + 2)\n        \n        sns.kdeplot(train_data['Dry_Green_g'], ax=ax2, color='blue', label=f'Train (Mean: {stats[\"Train_Mean_Green\"]:.1f})', fill=True, alpha=0.1)\n        sns.kdeplot(val_data['Dry_Green_g'], ax=ax2, color='red', label=f'Val (Mean: {stats[\"Val_Mean_Green\"]:.1f})', fill=True, alpha=0.1)\n        \n        ax2.set_title(f\"Fold {fold+1}: Green Biomass Distribution\")\n        ax2.set_xlabel(\"Biomass (g)\")\n        ax2.legend()\n\n    plt.tight_layout()\n    plt.show()\n    \n    # ==========================================\n    # 3. 输出诊断报告\n    # ==========================================\n    stats_df = pd.DataFrame(fold_stats)\n    print(\"\\n📊 GroupKFold 诊断报告:\")\n    print(stats_df)\n    \n    print(\"\\n💡 判读指南:\")\n    print(\"1. Leakage_Days 必须全是 0。如果不是0，代码有错。\")\n    print(\"2. 观察 Val_Mean_Green 是否与 Train_Mean_Green 差异过大？\")\n    print(\"   - 如果 Fold X 的验证集均值是 100g，而训练集是 20g，说明这折非常难(Hard Fold)。\")\n    print(\"   - 这种分布不一致会导致该折的 CV 分数很低，这是正常的，但也提醒我们要提升模型的泛化能力。\")\n\nif __name__ == '__main__':\n    check_group_split()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T16:23:14.533140Z","iopub.execute_input":"2025-12-19T16:23:14.533839Z","iopub.status.idle":"2025-12-19T16:23:16.749037Z","shell.execute_reply.started":"2025-12-19T16:23:14.533811Z","shell.execute_reply":"2025-12-19T16:23:16.748365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold\n\n# ================= 配置 =================\nDATA_ROOT = '/kaggle/input/csiro-biomass'\nif not os.path.exists(DATA_ROOT):\n    DATA_ROOT = r'D:\\Downloads\\CSIRO - Image2生物质预测\\csiro-biomass'\nTRAIN_CSV = os.path.join(DATA_ROOT, 'train.csv')\nN_FOLD = 5\nSEED = 42\n\ndef compare_splitting_strategies():\n    if not os.path.exists(TRAIN_CSV):\n        print(f\"❌ 找不到文件: {TRAIN_CSV}\")\n        return\n\n    print(\"正在加载数据...\")\n    df = pd.read_csv(TRAIN_CSV)\n    \n    # 1. 预处理\n    df['id_prefix'] = df['sample_id'].apply(lambda x: x.split('__')[0])\n    pivot_df = df.pivot_table(index='id_prefix', columns='target_name', values='target').reset_index()\n    \n    # 提取日期用于 Group\n    meta_df = df.groupby('id_prefix')[['Sampling_Date']].first().reset_index()\n    agg_df = pd.merge(pivot_df, meta_df, on='id_prefix')\n    \n    # 制作分层用的 Bins (用于 StratifiedKFold)\n    # 将连续的 Dry_Green_g 切分成 10 个等级\n    agg_df['bin'] = pd.qcut(agg_df['Dry_Green_g'], q=10, labels=False, duplicates='drop')\n    \n    # 定义三种策略\n    strategies = {\n        \"1. Random KFold (Shuffle=True)\": KFold(n_splits=N_FOLD, shuffle=True, random_state=SEED),\n        \"2. Stratified KFold (By Biomass Bin)\": StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED),\n        \"3. Group KFold (By Date)\": GroupKFold(n_splits=N_FOLD)\n    }\n    \n    # ================= 开始对比可视化 =================\n    fig, axes = plt.subplots(3, N_FOLD, figsize=(20, 12), sharex=True)\n    plt.subplots_adjust(hspace=0.4, wspace=0.2)\n    \n    target_col = 'Dry_Green_g'\n    \n    for row_idx, (name, splitter) in enumerate(strategies.items()):\n        print(f\"\\n正在分析策略: {name} ...\")\n        \n        # 准备 split 参数\n        X = agg_df\n        y = agg_df['bin'] # Stratified 需要 label\n        groups = agg_df['Sampling_Date'] # Group 需要 groups\n        \n        split_gen = splitter.split(X, y, groups=groups)\n        \n        for fold, (train_idx, val_idx) in enumerate(split_gen):\n            train_data = agg_df.iloc[train_idx]\n            val_data = agg_df.iloc[val_idx]\n            \n            # 检查泄露\n            train_dates = set(train_data['Sampling_Date'].unique())\n            val_dates = set(val_data['Sampling_Date'].unique())\n            leakage = len(train_dates.intersection(val_dates))\n            \n            # 绘图\n            ax = axes[row_idx, fold]\n            sns.kdeplot(train_data[target_col], ax=ax, color='blue', label='Train', fill=True, alpha=0.1)\n            sns.kdeplot(val_data[target_col], ax=ax, color='red', label='Val', fill=True, alpha=0.1)\n            \n            # 设置标题 (红色标题表示有危险)\n            title_color = 'red' if leakage > 0 else 'green'\n            ax.set_title(f\"Fold {fold+1}\\nLeak Days: {leakage}\", color=title_color, fontweight='bold')\n            \n            if fold == 0:\n                ax.set_ylabel(name, fontsize=12, fontweight='bold')\n            if row_idx == 0 and fold == N_FOLD-1:\n                ax.legend()\n\n    print(\"\\n✅ 可视化完成！请看图表。\")\n    print(\"蓝色=训练集分布，红色=验证集分布\")\n    print(\"如果红色曲线和蓝色曲线完全重合，CV 分数通常会很高。\")\n    print(\"如果 Leak Days > 0，说明这个高 CV 是'作弊'得来的。\")\n    plt.show()\n\nif __name__ == '__main__':\n    compare_splitting_strategies()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T16:34:30.028055Z","iopub.execute_input":"2025-12-19T16:34:30.028811Z","iopub.status.idle":"2025-12-19T16:34:32.812705Z","shell.execute_reply.started":"2025-12-19T16:34:30.028782Z","shell.execute_reply":"2025-12-19T16:34:32.811834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\n# ==========================================\n# 配置\n# ==========================================\nDATA_ROOT = '/kaggle/input/csiro-biomass-prediction-data'\n# 本地路径备用 (如果需要)\nif not os.path.exists(DATA_ROOT):\n    DATA_ROOT = '/kaggle/input/csiro-biomass'\n    \nTRAIN_CSV = os.path.join(DATA_ROOT, 'train.csv')\n\ndef check_binding():\n    if not os.path.exists(TRAIN_CSV):\n        print(f\"❌ 找不到文件: {TRAIN_CSV}\")\n        return\n\n    print(\"正在加载数据...\")\n    df = pd.read_csv(TRAIN_CSV)\n    \n    # 1. 提取 id_prefix\n    df['id_prefix'] = df['sample_id'].apply(lambda x: x.split('__')[0])\n    \n    # 2. 提取元数据 (Sampling_Date 和 State)\n    # 我们按 id_prefix 分组，取每张图的第一条记录即可\n    meta_cols = ['Sampling_Date', 'State']\n    # 过滤掉不存在的列 (以防万一)\n    valid_cols = [c for c in meta_cols if c in df.columns]\n    \n    if len(valid_cols) < 2:\n        print(\"❌ 错误: 数据集中缺少 Sampling_Date 或 State 列，无法验证绑定关系。\")\n        print(f\"当前列名: {df.columns.tolist()}\")\n        return\n\n    agg_df = df.groupby('id_prefix')[valid_cols].first().reset_index()\n    \n    # 3. 解析日期\n    agg_df['Sampling_Date'] = pd.to_datetime(agg_df['Sampling_Date'], format='mixed', dayfirst=True)\n    \n    print(f\"✅ 数据准备完成，共 {len(agg_df)} 个唯一样本。\")\n\n    # ==========================================\n    # 4. 核心验证：交叉表\n    # ==========================================\n    print(\"\\n📊 正在生成 [日期 vs 州] 交叉表...\")\n    \n    # 生成交叉表：行是日期，列是州，值是样本数量\n    cross_tab = pd.crosstab(agg_df['Sampling_Date'], agg_df['State'])\n    \n    # 打印前 20 行看看\n    print(\"\\n--- 交叉表预览 (前20个日期) ---\")\n    print(cross_tab.head(20))\n    \n    # ==========================================\n    # 5. 深度统计：绑定强度\n    # ==========================================\n    # 统计每一天有几个州的数据 (计算每一行有多少个非零值)\n    states_per_day = (cross_tab > 0).sum(axis=1)\n    \n    print(\"\\n\\n🕵️‍♀️ 绑定关系深度分析报告:\")\n    print(\"=\"*40)\n    \n    # 统计有多少天是“单州独占”的\n    single_state_days = (states_per_day == 1).sum()\n    total_days = len(states_per_day)\n    binding_strength = single_state_days / total_days * 100\n    \n    print(f\"总采样天数: {total_days} 天\")\n    print(f\"只在一个州采样的天数: {single_state_days} 天\")\n    print(f\"跨多个州采样的天数: {total_days - single_state_days} 天\")\n    print(f\"🔗 绑定强度: {binding_strength:.1f}%\")\n    \n    if binding_strength > 90:\n        print(\"\\n👉 结论：强绑定！\")\n        print(\"绝大多数日期只对应一个州。这意味着 GroupKFold(Date) 确实几乎等同于按地点分组。\")\n        print(\"模型必须学会'域泛化'才能得分。多任务学习（预测State）在这里会非常关键。\")\n    else:\n        print(\"\\n👉 结论：弱绑定。\")\n        print(\"很多日期同时在多个州采样。GroupKFold(Date) 主要是为了防止时间泄露，而不是地点泄露。\")\n\n    # 如果有跨州的天数，打印出来看看是哪天\n    if (states_per_day > 1).any():\n        print(\"\\n--- ⚠️ 跨州采样的日期 (Multi-State Days) ---\")\n        multi_days = states_per_day[states_per_day > 1].index\n        print(cross_tab.loc[multi_days])\n\nif __name__ == '__main__':\n    check_binding()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T16:45:35.366536Z","iopub.execute_input":"2025-12-19T16:45:35.367750Z","iopub.status.idle":"2025-12-19T16:45:35.409095Z","shell.execute_reply.started":"2025-12-19T16:45:35.367709Z","shell.execute_reply":"2025-12-19T16:45:35.408258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import GroupKFold, StratifiedGroupKFold\n\n# ================= 配置 =================\nDATA_ROOT = '/kaggle/input/csiro-biomass'\n# 本地路径兼容\nif not os.path.exists(DATA_ROOT):\n    DATA_ROOT = r'D:\\Downloads\\CSIRO - Image2生物质预测\\csiro-biomass'\nTRAIN_CSV = os.path.join(DATA_ROOT, 'train.csv')\nN_FOLD = 5\nSEED = 42\n\ndef compare_skgf_vs_gkf():\n    if not os.path.exists(TRAIN_CSV):\n        print(f\"❌ 找不到文件: {TRAIN_CSV}\")\n        return\n\n    print(\"正在加载数据...\")\n    df = pd.read_csv(TRAIN_CSV)\n    \n    # 1. 预处理\n    df['id_prefix'] = df['sample_id'].apply(lambda x: x.split('__')[0])\n    # 提取元数据\n    meta_df = df.groupby('id_prefix')[['Sampling_Date', 'State']].first().reset_index()\n    \n    # 填补缺失的 State (作为 'Unknown' 类)\n    meta_df['State'] = meta_df['State'].fillna('Unknown')\n    \n    # 2. 定义两种策略\n    # A: 你的旧方案 (SKGF by State)\n    skgf = StratifiedGroupKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)\n    \n    # B: 推荐方案 (GroupKFold by Date)\n    gkf = GroupKFold(n_splits=N_FOLD)\n    \n    # 准备绘图数据容器\n    results = []\n\n    print(\"\\n🚀 开始模拟划分...\")\n    \n    # --- 模拟 SKGF ---\n    # SKGF 需要 y 参数来进行分层，这里 y = State\n    for fold, (train_idx, val_idx) in enumerate(skgf.split(meta_df, y=meta_df['State'], groups=meta_df['Sampling_Date'])):\n        val_data = meta_df.iloc[val_idx]\n        # 统计验证集里各个州的占比\n        counts = val_data['State'].value_counts(normalize=True).to_dict()\n        for state, ratio in counts.items():\n            results.append({'Strategy': '1. SKGF (Stratify=State)', 'Fold': f'Fold {fold+1}', 'State': state, 'Ratio': ratio})\n\n    # --- 模拟 GroupKFold ---\n    for fold, (train_idx, val_idx) in enumerate(gkf.split(meta_df, groups=meta_df['Sampling_Date'])):\n        val_data = meta_df.iloc[val_idx]\n        counts = val_data['State'].value_counts(normalize=True).to_dict()\n        for state, ratio in counts.items():\n            results.append({'Strategy': '2. GroupKFold (Group=Date)', 'Fold': f'Fold {fold+1}', 'State': state, 'Ratio': ratio})\n\n    # ================= 可视化 =================\n    res_df = pd.DataFrame(results)\n    \n    plt.figure(figsize=(16, 8))\n    \n    # 使用条形堆叠图或者分面图来展示\n    g = sns.catplot(\n        data=res_df, kind=\"bar\",\n        x=\"Fold\", y=\"Ratio\", hue=\"State\", col=\"Strategy\",\n        palette=\"tab10\", alpha=0.9, height=5, aspect=1.2\n    )\n    \n    g.fig.suptitle('Validation Set State Distribution Comparison', fontsize=16, y=1.05)\n    g.set_axis_labels(\"\", \"Ratio of Samples in Validation Set\")\n    \n    print(\"\\n📊 分析指南:\")\n    print(\"1. 看左图 (SKGF): 每一折是不是'五颜六色'？如果是，说明验证集被强行均匀化了，降低了难度。\")\n    print(\"2. 看右图 (GroupKFold): 是不是有些折'颜色很单调'？比如 Fold 2 只有一种颜色？这代表了真实的极端分布。\")\n    \n    plt.show()\n\nif __name__ == '__main__':\n    compare_skgf_vs_gkf()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T16:54:16.285751Z","iopub.execute_input":"2025-12-19T16:54:16.286162Z","iopub.status.idle":"2025-12-19T16:54:16.978618Z","shell.execute_reply.started":"2025-12-19T16:54:16.286133Z","shell.execute_reply":"2025-12-19T16:54:16.977868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport timm\nfrom tqdm import tqdm\n\n# ==========================================\n# 1. 配置 (必须与训练一致)\n# ==========================================\nclass Config:\n    # ⚠️ 必须与训练时的设置完全一致\n    model_name = 'efficientnet_b4'\n    img_size = 380  # 切分后的半张图尺寸\n    batch_size = 32 # 推理时可以用大一点的batch size\n    num_workers = 4\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    # ⚠️ 模型权重路径\n    # 请修改为您上传到 Kaggle Dataset 的实际路径\n    # 假设你把 trained_models_final_sgkf 文件夹上传了\n    MODEL_DIR = '/kaggle/input/b475-rrr' \n    \n    # 如果使用了5折交叉验证\n    n_fold = 5 \n\n# ==========================================\n# 2. 模型定义 (必须完全复制训练代码)\n# ==========================================\nclass BiomassModel3(nn.Module):\n    def __init__(self, model_name, pretrained=False):\n        super().__init__()\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg',\n        )\n        \n        in_features = self.backbone.num_features * 2\n        \n        def make_head(out_dim=1):\n            return nn.Sequential(\n                nn.Linear(in_features, 256),\n                \n                nn.ReLU(inplace=True),\n                nn.Dropout(0.3),\n                nn.Linear(256, out_dim)\n            )\n        \n        self.head_green = make_head()\n        self.head_clover = make_head()\n        self.head_dead = make_head()\n        \n        self.softplus = nn.Softplus(beta=1.0)\n\n    def forward(self, left_x, right_x):\n        left_feat = self.backbone(left_x)\n        right_feat = self.backbone(right_x)\n        \n        combined = torch.cat([left_feat, right_feat], dim=1)\n        \n        green = self.softplus(self.head_green(combined))\n        clover = self.softplus(self.head_clover(combined))\n        dead = self.softplus(self.head_dead(combined))\n        \n        return torch.cat([green, clover, dead], dim=1)\n\n# ==========================================\n# 3. 数据处理 (切分 + Resize)\n# ==========================================\ndef get_transforms(img_size):\n    return transforms.Compose([\n        transforms.Resize((img_size, img_size)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n\nclass TestDataset(Dataset):\n    def __init__(self, df, data_root, transform=None):\n        self.df = df\n        self.data_root = data_root\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.data_root, row['image_path'])\n        \n        try:\n            image = Image.open(img_path).convert('RGB')\n        except:\n            # 极少数坏图处理\n            image = Image.new('RGB', (2000, 1000))\n            \n        # --- 核心：左右切分 ---\n        w, h = image.size\n        mid = w // 2\n        \n        left_image = image.crop((0, 0, mid, h))\n        right_image = image.crop((mid, 0, w, h))\n        \n        if self.transform:\n            left_image = self.transform(left_image)\n            right_image = self.transform(right_image)\n            \n        # 返回 ID 前缀以便后续组装\n        return left_image, right_image, row['id_prefix']\n\n# ==========================================\n# 4. 推理函数 (含 TTA)\n# ==========================================\ndef run_inference():\n    # 1. 准备数据\n    DATA_ROOT = '/kaggle/input/csiro-biomass'\n    test_csv = os.path.join(DATA_ROOT, 'test.csv')\n    test_df = pd.read_csv(test_csv)\n    \n    # 提取 id_prefix (例如 ID12345) 和 image_path\n    # Test集格式为 ID12345__Dry_Green_g, 我们需要把它聚合为一张图一行\n    test_df['id_prefix'] = test_df['sample_id'].apply(lambda x: x.split('__')[0])\n    \n    # 去重，每张图只保留一行\n    unique_test_df = test_df.drop_duplicates(subset=['id_prefix']).reset_index(drop=True)\n    print(f\"Total test images: {len(unique_test_df)}\")\n    \n    dataset = TestDataset(unique_test_df, DATA_ROOT, transform=get_transforms(Config.img_size))\n    loader = DataLoader(dataset, batch_size=Config.batch_size, shuffle=False, num_workers=Config.num_workers)\n    \n    # 2. 加载模型 (5折集成)\n    models = []\n    for fold in range(Config.n_fold):\n        # ⚠️ 注意：这里的文件名必须和你上传的文件名匹配\n        # 可能是 fold1_best.pth 或 fold0_best.pth，取决于你训练时怎么命名的\n        model_path = os.path.join(Config.MODEL_DIR, f'fold{fold+1}_best.pth')\n        \n        if not os.path.exists(model_path):\n            print(f\"⚠️ Warning: Model not found at {model_path}, skipping...\")\n            continue\n            \n        model = BiomassModel3(Config.model_name, pretrained=False)\n        model.load_state_dict(torch.load(model_path, map_location=Config.device))\n        model.to(Config.device)\n        model.eval()\n        models.append(model)\n        print(f\"Loaded Fold {fold+1}\")\n        \n    if len(models) == 0:\n        print(\"❌ Error: No models loaded!\")\n        return\n\n    # 3. 开始预测\n    results = {} # {id_prefix: [green, clover, dead]}\n    \n    print(\"Running Inference...\")\n    with torch.no_grad():\n        for left_imgs, right_imgs, ids in tqdm(loader):\n            left_imgs = left_imgs.to(Config.device)\n            right_imgs = right_imgs.to(Config.device)\n            \n            # --- TTA 1: 原始图像 ---\n            batch_preds = torch.zeros((left_imgs.size(0), 3)).to(Config.device)\n            \n            for model in models:\n                pred = model(left_imgs, right_imgs)\n                batch_preds += pred\n                \n            # --- TTA 2: 水平翻转 (Horizontal Flip) ---\n            # 增强鲁棒性：把左右图都做水平翻转\n            left_flip = torch.flip(left_imgs, [3]) \n            right_flip = torch.flip(right_imgs, [3])\n            \n            for model in models:\n                pred_flip = model(left_flip, right_flip)\n                batch_preds += pred_flip\n            \n            # 计算平均值 (5个模型 * 2次TTA = 除以10)\n            batch_preds /= (len(models) * 2)\n            \n            # 存入结果\n            batch_preds = batch_preds.cpu().numpy()\n            for i, prefix in enumerate(ids):\n                results[prefix] = batch_preds[i]\n\n    # 4. 生成提交文件\n    submission_rows = []\n    \n    for prefix, preds in results.items():\n        p_green, p_clover, p_dead = preds[0], preds[1], preds[2]\n        \n        # 物理计算\n        p_gdm = p_green + p_clover\n        p_total = p_gdm + p_dead\n        \n        # 严格按照 sample_id 格式写入\n        submission_rows.append({'sample_id': f\"{prefix}__Dry_Green_g\", 'target': p_green})\n        submission_rows.append({'sample_id': f\"{prefix}__Dry_Clover_g\", 'target': p_clover})\n        submission_rows.append({'sample_id': f\"{prefix}__Dry_Dead_g\", 'target': p_dead})\n        submission_rows.append({'sample_id': f\"{prefix}__GDM_g\", 'target': p_gdm})\n        submission_rows.append({'sample_id': f\"{prefix}__Dry_Total_g\", 'target': p_total})\n        \n    sub_df = pd.DataFrame(submission_rows)\n    \n    # 确保排序正确（虽然不是强制的，但好习惯）\n    sub_df.to_csv('submission.csv', index=False)\n    print(\"✅ Submission.csv generated successfully!\")\n    print(sub_df.head(10))\n\nif __name__ == '__main__':\n    run_inference()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}