{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":668443,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":506095,"modelId":520886},{"sourceId":699025,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":530293,"modelId":544238}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nfrom torchvision.io import read_image\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# ======================\n# 全局配置（新增多模态相关配置）\n# ======================\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nBATCH_SIZE = 16\nEPOCHS = 300\nLEARNING_RATE = 1e-4\nSEED = 42\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n\n# 图像目录配置\nTRAIN_IMG_DIR = '/kaggle/input/csiro-biomass/train/'\nTEST_IMG_DIR = '/kaggle/input/csiro-biomass/test/'\nTEST_CSV_PATH = '/kaggle/input/csiro-biomass/test.csv'\n\n# ConvNeXt图像预处理（适配224x224）\nCONVNEXT_TRANSFORM = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n])\n\n# 文本嵌入配置\nSTATE_VOCAB = ['NSW', 'Tas', 'Vic', 'QLD', 'SA', 'WA', 'NT', 'ACT', 'Unknown']\nSPECIES_VOCAB = ['Ryegrass_Clover', 'Perennial_Ryegrass', 'Annual_Ryegrass', 'Clover', 'Unknown']\nTEXT_EMBED_DIM = 32  # 文本嵌入维度\n\n# ======================\n# 工具函数（新增多模态相关工具）\n# ======================\ndef check_train_data_format(train_df):\n    \"\"\"检查train.csv格式\"\"\"\n    print(\"=== 训练数据格式检查 ===\")\n    print(f\"train.csv列名：{train_df.columns.tolist()}\")\n    print(f\"train.csv形状：{train_df.shape}\")\n    \n    required_cols = ['target_name', 'target', 'sample_id']\n    missing_cols = [col for col in required_cols if col not in train_df.columns]\n    if missing_cols:\n        raise ValueError(f\"train.csv缺少关键列：{missing_cols}\")\n    return\n\ndef check_test_data_format(test_df):\n    \"\"\"检查test.csv格式\"\"\"\n    print(\"\\n=== 测试数据格式检查 ===\")\n    print(f\"test.csv列名：{test_df.columns.tolist()}\")\n    print(f\"test.csv形状：{test_df.shape}\")\n    \n    required_cols = ['sample_id', 'target_name']\n    missing_cols = [col for col in required_cols if col not in test_df.columns]\n    if missing_cols:\n        raise ValueError(f\"test.csv缺少关键列：{missing_cols}\")\n    \n    test_targets = test_df['target_name'].unique()\n    expected_targets = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'GDM_g', 'Dry_Total_g']\n    if len(test_targets) != 5 or not set(test_targets).issubset(set(expected_targets)):\n        print(f\"警告：test.csv的target_name不完整，当前为：{test_targets}\")\n    return\n\ndef index_to_series(df):\n    \"\"\"将Index转换为Series\"\"\"\n    return df.index.to_series().astype(int)\n\ndef extract_image_id(sample_id):\n    \"\"\"增强容错的ID提取\"\"\"\n    import re\n    match = re.search(r'[Ii][Dd](\\d+)', str(sample_id))\n    if match:\n        return match.group(1)\n    else:\n        digits = re.findall(r'\\d+', str(sample_id))\n        default_id = digits[0] if digits else '0'\n        print(f\"警告：sample_id {sample_id} 未提取到有效ID，使用默认ID {default_id}\")\n        return default_id\n\ndef calculate_vegetation_indices(img_tensor):\n    \"\"\"\n    从RGB图像计算植被指数（多模态核心：新增植被指数模态）\n    输入：(3, H, W)的float张量（0-1范围）\n    输出：6维植被指数特征\n    \"\"\"\n    # 分离RGB通道\n    r = img_tensor[0, :, :]\n    g = img_tensor[1, :, :]\n    b = img_tensor[2, :, :]\n    \n    # 避免除零\n    eps = 1e-8\n    \n    # 1. NDVI（归一化植被指数）：近似计算（用R通道反相模拟NIR）\n    nir = 1.0 - r  # RGB图像无NIR通道，用R反相近似\n    ndvi = (nir - r) / (nir + r + eps)\n    \n    # 2. GNDVI（绿度归一化植被指数）\n    gndvi = (nir - g) / (nir + g + eps)\n    \n    # 3. SAVI（土壤调整植被指数）\n    L = 0.5  # 土壤调整系数\n    savi = (nir - r) * (1 + L) / (nir + r + L + eps)\n    \n    # 4. EVI（增强植被指数）\n    C1, C2 = 6.0, 7.5\n    evi = 2.5 * (nir - r) / (nir + C1 * r - C2 * b + 1 + eps)\n    \n    # 5. RGBVI（RGB植被指数）\n    rgbvi = (g **2 - r * b) / (g** 2 + r * b + eps)\n    \n    # 6. 绿红比\n    gr_ratio = g / (r + eps)\n    \n    # 计算统计特征（均值+标准差），输出12维特征\n    vi_features = []\n    for vi in [ndvi, gndvi, savi, evi, rgbvi, gr_ratio]:\n        vi_features.append(torch.mean(vi).item())\n        vi_features.append(torch.std(vi).item())\n    \n    return np.array(vi_features, dtype=np.float32)\n\ndef preprocess_test_data(test_csv_path, test_img_dir):\n    \"\"\"预处理test.csv（适配多模态）\"\"\"\n    test_df = pd.read_csv(test_csv_path)\n    check_test_data_format(test_df)\n    \n    # 提取图片ID\n    test_df['image_id'] = test_df['sample_id'].apply(extract_image_id)\n    total_image_ids = test_df['image_id'].nunique()\n    print(f\"从train.csv提取到的总图片ID数量：{total_image_ids}（预期357个）\")\n    \n    # 校验每个ID对应5种生物量\n    id_count = test_df['image_id'].value_counts()\n    abnormal_ids = [img_id for img_id, count in id_count.items() if count != 5]\n    if abnormal_ids:\n        raise ValueError(f\"以下图片ID未对应5种生物量：{abnormal_ids[:5]}...\")\n    \n    # 构建图片路径\n    test_df['full_image_path'] = test_df['image_id'].apply(\n        lambda x: os.path.join(test_img_dir, f'ID{x}.jpg')\n    )\n    missing_imgs = [path for path in test_df['full_image_path'].unique() if not os.path.exists(path)]\n    if missing_imgs:\n        print(f\"警告：{len(missing_imgs)}张图片缺失，路径示例：{missing_imgs[:3]}\")\n    \n    print(f\"\\n=== test.csv预处理结果 ===\")\n    print(f\"测试数据总行数：{len(test_df)}（预期{total_image_ids}×5={total_image_ids*5}行）\")\n    print(f\"唯一图片ID数量：{total_image_ids}（预期357个）\")\n    return test_df\n\n# ======================\n# 1. 数据预处理（适配多模态）\n# ======================\ndef preprocess_structured_data(train_path, train_img_dir):\n    train_df = pd.read_csv(train_path)\n    check_train_data_format(train_df)\n    \n    train_df['image_id'] = train_df['sample_id'].apply(extract_image_id)\n    index_series = index_to_series(train_df)\n    train_df['image_id'] = train_df['image_id'].fillna(index_series).astype(int)\n    \n    print(f\"\\n=== image_id提取结果 ===\")\n    print(f\"唯一image_id数量：{train_df['image_id'].nunique()}\")\n    \n    train_df['full_image_path'] = train_df['image_id'].apply(\n        lambda x: os.path.join(train_img_dir, f'ID{x}.jpg')\n    )\n    \n    # 构建宽表\n    wide_df = train_df.pivot_table(\n        index='image_id',\n        columns='target_name',\n        values='target',\n        aggfunc='first',\n        fill_value=0\n    ).reset_index(drop=False)\n    \n    # 合并结构化特征\n    struct_cols = ['image_id', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']\n    struct_cols = [col for col in struct_cols if col in train_df.columns]\n    struct_df = train_df[struct_cols].drop_duplicates(subset=['image_id']).reset_index(drop=True)\n    wide_df = pd.merge(wide_df, struct_df, on='image_id', how='left')\n    img_info = train_df[['image_id', 'full_image_path']].drop_duplicates()\n    wide_df = pd.merge(wide_df, img_info, on='image_id', how='left')\n    \n    # 类别特征编码（用于文本嵌入）\n    wide_df['State'] = wide_df['State'].fillna('Unknown')\n    wide_df['Species'] = wide_df['Species'].fillna('Unknown')\n    # 限制在预定义词汇表内\n    wide_df['State'] = wide_df['State'].apply(lambda x: x if x in STATE_VOCAB else 'Unknown')\n    wide_df['Species'] = wide_df['Species'].apply(lambda x: x if x in SPECIES_VOCAB else 'Unknown')\n    \n    # 数值特征标准化\n    scaler_struct = StandardScaler()\n    num_struct_cols = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n    num_struct_cols = [col for col in num_struct_cols if col in wide_df.columns]\n    print(f\"\\n=== 标准化列信息 ===\")\n    print(f\"实际用于标准化的列：{num_struct_cols}\")\n    \n    if len(num_struct_cols) > 0:\n        wide_df[num_struct_cols] = wide_df[num_struct_cols].fillna(0)\n        wide_df[num_struct_cols] = scaler_struct.fit_transform(wide_df[num_struct_cols])\n    else:\n        scaler_struct = None\n    \n    # 定义目标列和特征列\n    target_cols = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'GDM_g', 'Dry_Total_g']\n    target_cols = [col for col in target_cols if col in wide_df.columns]\n    if not target_cols:\n        raise ValueError(\"未找到目标列\")\n    \n    struct_feat_cols = []\n    if 'Pre_GSHH_NDVI' in wide_df.columns:\n        struct_feat_cols.append('Pre_GSHH_NDVI')\n    if 'Height_Ave_cm' in wide_df.columns:\n        struct_feat_cols.append('Height_Ave_cm')\n    \n    # 文本特征列（新增）\n    text_feat_cols = ['State', 'Species']\n    \n    print(f\"\\n=== 数据预处理结果 ===\")\n    print(f\"生成的目标列：{target_cols}\")\n    print(f\"生成的结构化特征列：{struct_feat_cols}\")\n    print(f\"生成的文本特征列：{text_feat_cols}\")\n    return wide_df, target_cols, struct_feat_cols, text_feat_cols, scaler_struct, num_struct_cols\n\n# ======================\n# 2. 多模态特征提取（核心升级）\n# ======================\ndef init_convnext_feature_extractor():\n    \"\"\"初始化ConvNeXt-Large特征提取器（视觉模态）\"\"\"\n    try:\n        convnext = models.convnext_large(pretrained=False)\n        local_weight_path = \"/kaggle/input/convnext/pytorch/default/1/convnext_large-ea097f82.pth\"\n        convnext.load_state_dict(torch.load(local_weight_path, map_location=DEVICE))\n    except Exception as e:\n        print(f\"加载本地ConvNeXt权重失败：{str(e)[:50]}...，自动下载权重\")\n        convnext = models.convnext_large(pretrained=True)\n    \n    # 冻结参数，保留特征提取部分\n    for param in convnext.parameters():\n        param.requires_grad = False\n    feature_extractor = nn.Sequential(*list(convnext.children())[:-1],\n                                      nn.AdaptiveAvgPool2d(1),\n                                      nn.Flatten()).to(DEVICE)\n    feature_extractor.eval()\n    return feature_extractor\n\ndef extract_multimodal_features(image_paths, convnext_extractor, img_type=\"train\"):\n    \"\"\"\n    提取多模态特征：\n    - 视觉模态：ConvNeXt-Large特征（1536维）\n    - 植被指数模态：6种VI的统计特征（12维）\n    \"\"\"\n    visual_features = []    # 视觉模态：1536维\n    vi_features = []        # 植被指数模态：12维\n    missing_imgs = []\n    \n    with torch.no_grad():\n        for img_path in image_paths:\n            if not os.path.exists(img_path):\n                missing_imgs.append(img_path)\n                visual_features.append(np.zeros(1536, dtype=np.float32))\n                vi_features.append(np.zeros(12, dtype=np.float32))\n                continue\n            try:\n                # 读取并预处理图像\n                img_tensor = read_image(img_path)\n                img_tensor = img_tensor.float() / 255.0  # 0-1归一化\n                \n                # 1. 计算植被指数模态\n                vi_feat = calculate_vegetation_indices(img_tensor)\n                vi_features.append(vi_feat)\n                \n                # 2. 提取ConvNeXt视觉特征\n                img_processed = CONVNEXT_TRANSFORM(img_tensor).unsqueeze(0).to(DEVICE)\n                vis_feat = convnext_extractor(img_processed).squeeze().cpu().numpy()\n                visual_features.append(vis_feat)\n                \n            except Exception as e:\n                missing_imgs.append(f\"{img_path}（错误：{str(e)[:20]}...）\")\n                visual_features.append(np.zeros(1536, dtype=np.float32))\n                vi_features.append(np.zeros(12, dtype=np.float32))\n    \n    # 标准化\n    visual_features = np.array(visual_features, dtype=np.float32)\n    vi_features = np.array(vi_features, dtype=np.float32)\n    \n    scaler_visual = StandardScaler()\n    scaler_vi = StandardScaler()\n    \n    visual_features_scaled = scaler_visual.fit_transform(visual_features)\n    vi_features_scaled = scaler_vi.fit_transform(vi_features)\n    \n    if len(missing_imgs) > 0:\n        print(f\"警告：{img_type}集缺失/损坏图像 {len(missing_imgs)} 张\")\n    \n    return visual_features_scaled, vi_features_scaled, scaler_visual, scaler_vi, missing_imgs\n\n# ======================\n# 3. 多模态融合模块（核心创新）\n# ======================\nclass TextEmbeddingLayer(nn.Module):\n    \"\"\"文本模态嵌入层：对State/Species进行嵌入\"\"\"\n    def __init__(self):\n        super().__init__()\n        self.state_emb = nn.Embedding(len(STATE_VOCAB), TEXT_EMBED_DIM)\n        self.species_emb = nn.Embedding(len(SPECIES_VOCAB), TEXT_EMBED_DIM)\n        # 词汇表到索引的映射\n        self.state2idx = {s:i for i,s in enumerate(STATE_VOCAB)}\n        self.species2idx = {s:i for i,s in enumerate(SPECIES_VOCAB)}\n    \n    def forward(self, state_names, species_names):\n        \"\"\"\n        输入：state/species名称列表（batch_size）\n        输出：拼接后的文本嵌入（batch_size, 2*TEXT_EMBED_DIM）\n        \"\"\"\n        # 转换为索引\n        state_idx = torch.tensor([self.state2idx[s] for s in state_names], device=DEVICE)\n        species_idx = torch.tensor([self.species2idx[s] for s in species_names], device=DEVICE)\n        \n        # 嵌入\n        state_emb = self.state_emb(state_idx)\n        species_emb = self.species_emb(species_idx)\n        \n        # 拼接\n        text_emb = torch.cat([state_emb, species_emb], dim=1)\n        return text_emb\n\nclass CrossModalAttentionFusion(nn.Module):\n    \"\"\"跨模态注意力融合模块：替代简单拼接\"\"\"\n    def __init__(self, modal_dims, hidden_dim=256):\n        \"\"\"\n        modal_dims: 各模态维度列表 [视觉, 植被指数, 结构化, 文本]\n        \"\"\"\n        super().__init__()\n        self.modal_dims = modal_dims\n        self.num_modals = len(modal_dims)\n        \n        # 模态投影层（统一到隐藏维度）\n        self.proj_layers = nn.ModuleList([\n            nn.Linear(dim, hidden_dim) for dim in modal_dims\n        ])\n        \n        # 注意力权重计算\n        self.attention = nn.Sequential(\n            nn.Linear(self.num_modals * hidden_dim, self.num_modals),\n            nn.Softmax(dim=1)\n        )\n        \n        # 融合后投影\n        self.fusion_proj = nn.Linear(self.num_modals * hidden_dim, hidden_dim * 2)\n        self.norm = nn.LayerNorm(hidden_dim * 2)\n        self.dropout = nn.Dropout(0.2)\n    \n    def forward(self, modal_features):\n        \"\"\"\n        输入：各模态特征列表 [vis, vi, struct, text]\n              每个特征形状：(batch_size, dim)\n        输出：融合后的特征 (batch_size, hidden_dim*2)\n        \"\"\"\n        # 投影到统一维度\n        proj_features = []\n        for i, feat in enumerate(modal_features):\n            proj = self.proj_layers[i](feat)\n            proj_features.append(proj)\n        \n        # 拼接所有投影特征\n        concat_proj = torch.cat(proj_features, dim=1)\n        \n        # 计算注意力权重\n        attn_weights = self.attention(concat_proj)  # (batch, num_modals)\n        \n        # 加权融合\n        fused = 0\n        for i in range(self.num_modals):\n            fused += attn_weights[:, i:i+1] * proj_features[i]\n        \n        # 最终投影\n        out = self.fusion_proj(torch.cat(proj_features, dim=1))\n        out = self.norm(out)\n        out = torch.relu(out)\n        out = self.dropout(out)\n        \n        return out, attn_weights\n\n# ======================\n# 4. 多模态数据集（适配）\n# ======================\nclass MultimodalBiomassDataset(Dataset):\n    def __init__(self, fused_df, visual_feat, vi_feat, struct_feat_cols, text_feat_cols, target_cols):\n        self.visual_feat = torch.tensor(visual_feat, dtype=torch.float32)  # 先放CPU，批量时再移到DEVICE\n        self.vi_feat = torch.tensor(vi_feat, dtype=torch.float32)\n        self.struct_feat = torch.tensor(\n            fused_df[struct_feat_cols].fillna(0).values, dtype=torch.float32\n        )\n        # 提前将文本名称转换为列表（便于collate_fn处理）\n        self.state_names = fused_df[text_feat_cols[0]].fillna('Unknown').tolist()\n        self.species_names = fused_df[text_feat_cols[1]].fillna('Unknown').tolist()\n        self.targets = torch.tensor(\n            fused_df[target_cols].values, dtype=torch.float32\n        )\n        self.struct_feat_cols = struct_feat_cols\n        self.device = DEVICE\n\n    def __len__(self):\n        return len(self.visual_feat)\n\n    def __getitem__(self, idx):\n        return (\n            self.visual_feat[idx],    # 视觉模态（CPU张量）\n            self.vi_feat[idx],        # 植被指数模态（CPU张量）\n            self.struct_feat[idx],    # 结构化模态（CPU张量）\n            self.state_names[idx],    # 地区名称（字符串）\n            self.species_names[idx],  # 物种名称（字符串）\n            self.targets[idx]         # 目标值（CPU张量）\n        )\n\n# 自定义Collate函数（核心修复：处理字符串类型）\ndef multimodal_collate_fn(batch):\n    \"\"\"\n    自定义批次整理函数，处理包含字符串的多模态数据\n    batch结构：[(visual, vi, struct, state, species, target), ...]\n    \"\"\"\n    # 拆分批次中的各个字段\n    visuals = torch.stack([item[0] for item in batch])\n    vis = torch.stack([item[1] for item in batch])\n    structs = torch.stack([item[2] for item in batch])\n    states = [item[3] for item in batch]  # 字符串列表，直接保留\n    species = [item[4] for item in batch] # 字符串列表，直接保留\n    targets = torch.stack([item[5] for item in batch])\n    \n    # 移到目标设备（统一在这里移，避免数据集里提前移导致的问题）\n    visuals = visuals.to(DEVICE)\n    vis = vis.to(DEVICE)\n    structs = structs.to(DEVICE)\n    targets = targets.to(DEVICE)\n    \n    return (visuals, vis, structs, states, species, targets)\n\n# ======================\n# 5. 多模态生物量回归模型（核心升级）\n# ======================\nclass MultimodalBiomassRegressor(nn.Module):\n    \"\"\"多模态生物量回归模型：ConvNeXt + 跨模态注意力 + 残差网络\"\"\"\n    def __init__(self, modal_dims, output_dim=5, hidden_dims=[512, 256, 128]):\n        super().__init__()\n        \n        # 文本嵌入层\n        self.text_emb_layer = TextEmbeddingLayer()\n        \n        # 跨模态融合层\n        self.fusion_layer = CrossModalAttentionFusion(modal_dims, hidden_dim=256)\n        fusion_dim = 512  # 融合层输出维度\n        \n        # 残差回归头\n        residual_blocks = []\n        prev_dim = fusion_dim\n        for dim in hidden_dims:\n            residual_blocks.append(self._make_residual_block(prev_dim, dim))\n            prev_dim = dim\n        \n        self.residual_backbone = nn.Sequential(*residual_blocks)\n        \n        # 输出层\n        self.output_layer = nn.Sequential(\n            nn.Linear(hidden_dims[-1], output_dim),\n            nn.Softplus()  # 非负约束\n        )\n        \n        # 权重初始化\n        self._initialize_weights()\n    \n    def _make_residual_block(self, in_dim, out_dim, dropout=0.3):\n        \"\"\"构建残差块\"\"\"\n        return nn.Sequential(\n            nn.Linear(in_dim, out_dim),\n            nn.BatchNorm1d(out_dim),\n            nn.LeakyReLU(0.01),\n            nn.Dropout(dropout),\n            nn.Linear(out_dim, out_dim),\n            nn.BatchNorm1d(out_dim),\n            nn.LeakyReLU(0.01),\n            # 残差连接\n            nn.Identity() if in_dim == out_dim else nn.Linear(in_dim, out_dim)\n        )\n    \n    def _initialize_weights(self):\n        \"\"\"权重初始化\"\"\"\n        for m in self.modules():\n            if isinstance(m, nn.Linear):\n                nn.init.xavier_uniform_(m.weight)\n                if m.bias is not None:\n                    nn.init.constant_(m.bias, 0.1)\n            elif isinstance(m, nn.BatchNorm1d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n    \n    def forward(self, visual, vi, struct, text_names):\n        \"\"\"\n        前向传播：多模态输入\n        text_names: (state_name, species_name) 元组，每个元素是batch_size的列表\n        \"\"\"\n        # 文本嵌入\n        text_emb = self.text_emb_layer(text_names[0], text_names[1])\n        \n        # 多模态融合\n        modal_features = [visual, vi, struct, text_emb]\n        fused_feat, attn_weights = self.fusion_layer(modal_features)\n        \n        # 残差回归\n        x = fused_feat\n        for block in self.residual_backbone:\n            residual = x\n            x = block[:-1](x)\n            residual = block[-1](residual)\n            x = x + residual\n            x = torch.relu(x)\n        \n        # 输出\n        out = self.output_layer(x)\n        return out, attn_weights\n\n# ======================\n# 6. 加权R²评估（保持不变）\n# ======================\ndef weighted_r2_score(y_true, y_pred, weights=[0.1, 0.1, 0.1, 0.2, 0.5]):\n    weights = np.array(weights)\n    weights = weights[:y_true.shape[1]]\n    assert y_true.shape == y_pred.shape, \"真实值与预测值形状不匹配\"\n    \n    ss_total = 0.0\n    ss_res = 0.0\n    for i in range(y_true.shape[1]):\n        y_true_i = y_true[:, i]\n        y_pred_i = y_pred[:, i]\n        mean_true_i = np.mean(y_true_i)\n        \n        ss_total += weights[i] * np.sum((y_true_i - mean_true_i) ** 2)\n        ss_res += weights[i] * np.sum((y_true_i - y_pred_i) ** 2)\n    \n    return 1 - (ss_res / ss_total) if ss_total != 0 else 0.0\n\n# ======================\n# 7. 加权损失函数（保持不变）\n# ======================\nclass WeightedMSELoss(nn.Module):\n    \"\"\"加权MSE损失：与加权R²评估指标一致\"\"\"\n    def __init__(self, weights=[0.1, 0.1, 0.1, 0.2, 0.5]):\n        super(WeightedMSELoss, self).__init__()\n        self.weights = torch.tensor(weights, dtype=torch.float32).to(DEVICE)\n    \n    def forward(self, pred, target):\n        squared_error = (pred - target) ** 2\n        weighted_error = squared_error * self.weights.unsqueeze(0)\n        return torch.mean(weighted_error)\n\n# ======================\n# 8. 多模态模型训练（适配）\n# ======================\ndef train_multimodal_model(model, train_loader, val_loader, criterion, optimizer, scheduler, epochs):\n    best_val_r2 = -float('inf')\n    train_losses = []\n    val_losses = []\n    val_r2s = []\n    lrs = []\n    \n    for epoch in range(epochs):\n        model.train()\n        train_loss = 0.0\n        for batch in train_loader:\n            # 批次结构适配自定义collate_fn的输出\n            visual, vi, struct, state_names, species_names, target = batch\n            \n            # 前向传播\n            output, _ = model(visual, vi, struct, (state_names, species_names))\n            loss = criterion(output, target)\n            \n            # 反向传播\n            optimizer.zero_grad()\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            optimizer.step()\n            \n            train_loss += loss.item() * visual.size(0)\n        \n        train_loss_avg = train_loss / len(train_loader.dataset)\n        train_losses.append(train_loss_avg)\n        lrs.append(optimizer.param_groups[0]['lr'])\n        \n        # 验证\n        model.eval()\n        val_loss = 0.0\n        y_true_all = []\n        y_pred_all = []\n        with torch.no_grad():\n            for batch in val_loader:\n                visual, vi, struct, state_names, species_names, target = batch\n                \n                output, _ = model(visual, vi, struct, (state_names, species_names))\n                loss = criterion(output, target)\n                val_loss += loss.item() * visual.size(0)\n                \n                y_true_all.append(target.cpu().numpy())\n                y_pred_all.append(output.cpu().numpy())\n        \n        val_loss_avg = val_loss / len(val_loader.dataset)\n        y_true = np.concatenate(y_true_all, axis=0)\n        y_pred = np.concatenate(y_pred_all, axis=0)\n        val_r2 = weighted_r2_score(y_true, y_pred)\n        \n        val_losses.append(val_loss_avg)\n        val_r2s.append(val_r2)\n        scheduler.step()\n        \n        # 打印日志\n        if (epoch + 1) % 20 == 0:\n            print(f\"Epoch {epoch+1:3d} | 训练损失: {train_loss_avg:.4f} | 验证损失: {val_loss_avg:.4f} | 验证R²: {val_r2:.4f} | 学习率: {lrs[-1]:.6f}\")\n        elif val_r2 > best_val_r2:\n            best_val_r2 = val_r2\n            print(f\"Epoch {epoch+1:3d} | 最佳R²: {best_val_r2:.4f} | 学习率: {lrs[-1]:.6f}\")\n    \n    # 绘制训练曲线\n    plt.figure(figsize=(15, 5))\n    plt.subplot(1, 3, 1)\n    plt.plot(range(1, epochs+1), train_losses, label='Train Loss')\n    plt.plot(range(1, epochs+1), val_losses, label='Val Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Weighted MSE Loss')\n    plt.title('Training & Validation Loss')\n    plt.legend()\n    plt.grid(alpha=0.3)\n    \n    plt.subplot(1, 3, 2)\n    plt.plot(range(1, epochs+1), val_r2s, label='Val Weighted R²', color='orange')\n    plt.axhline(y=best_val_r2, color='red', linestyle='--', label=f'Best R²: {best_val_r2:.4f}')\n    plt.xlabel('Epoch')\n    plt.ylabel('Weighted R²')\n    plt.title('Validation Weighted R²')\n    plt.legend()\n    plt.grid(alpha=0.3)\n    \n    plt.subplot(1, 3, 3)\n    plt.plot(range(1, epochs+1), lrs, label='Learning Rate', color='green')\n    plt.xlabel('Epoch')\n    plt.ylabel('Learning Rate')\n    plt.title('Learning Rate Schedule')\n    plt.legend()\n    plt.grid(alpha=0.3)\n    \n    plt.tight_layout()\n    plt.show()\n    plt.close()\n    \n    print(f\"\\n训练完成！最佳验证加权R²: {best_val_r2:.4f}\")\n    return model\n\n\n# ======================\n# 9. 多模态测试预测（适配）\n# ======================\ndef predict_multimodal_test(model, test_df, scaler_struct, scaler_visual, scaler_vi, \n                           num_struct_cols, struct_feat_cols, convnext_extractor, target_cols):\n    print(f\"\\n=== 多模态测试集预测 ===\")\n    print(f\"预处理后test_df形状：{test_df.shape}\")\n    \n    unique_image_ids = test_df['image_id'].unique()\n    print(f\"待预测的唯一图片ID数量：{len(unique_image_ids)}\")\n    \n    test_df['target'] = 0.0\n    \n    # 文本嵌入层（从模型中提取）\n    text_emb_layer = model.text_emb_layer\n    text_emb_layer.eval()\n    \n    for img_id in unique_image_ids:\n        id_data = test_df[test_df['image_id'] == img_id].copy()\n        if len(id_data) == 0:\n            print(f\"跳过图片ID {img_id}：无对应数据\")\n            continue\n        \n        # 1. 结构化特征\n        test_struct = {}\n        default_vals = {'Pre_GSHH_NDVI': 0.66, 'Height_Ave_cm': 7.60}\n        for col in num_struct_cols:\n            if col in id_data.columns and not pd.isna(id_data[col].iloc[0]):\n                test_struct[col] = id_data[col].iloc[0]\n            else:\n                test_struct[col] = default_vals.get(col, 0.0)\n        \n        # 标准化\n        if scaler_struct is not None and len(num_struct_cols) > 0:\n            scale_vals = np.array([test_struct[col] for col in num_struct_cols]).reshape(1, -1)\n            scaled_vals = scaler_struct.transform(scale_vals)[0]\n            for i, col in enumerate(num_struct_cols):\n                test_struct[col] = scaled_vals[i]\n        \n        struct_feat = torch.tensor([test_struct[col] for col in struct_feat_cols], \n                                  dtype=torch.float32, device=DEVICE).unsqueeze(0)\n        \n        # 2. 文本特征\n        state_val = id_data.get('State', ['Unknown']).iloc[0]\n        state_val = state_val if state_val in STATE_VOCAB else 'Unknown'\n        species_val = id_data.get('Species', ['Unknown']).iloc[0]\n        species_val = species_val if species_val in SPECIES_VOCAB else 'Unknown'\n        \n        # 3. 视觉+植被指数特征\n        img_path = id_data['full_image_path'].iloc[0]\n        visual_feat = np.zeros(1536, dtype=np.float32)\n        vi_feat = np.zeros(12, dtype=np.float32)\n        \n        try:\n            if os.path.exists(img_path):\n                img_tensor = read_image(img_path)\n                img_tensor = img_tensor.float() / 255.0\n                \n                # 植被指数\n                vi_feat = calculate_vegetation_indices(img_tensor)\n                vi_feat = scaler_vi.transform([vi_feat])[0]\n                \n                # 视觉特征\n                img_processed = CONVNEXT_TRANSFORM(img_tensor).unsqueeze(0).to(DEVICE)\n                with torch.no_grad():\n                    vis_feat = convnext_extractor(img_processed).squeeze().cpu().numpy()\n                visual_feat = scaler_visual.transform([vis_feat])[0]\n            else:\n                print(f\"警告：图片不存在 {img_path}，使用零向量特征\")\n        except Exception as e:\n            print(f\"警告：图片 {img_path} 读取失败：{str(e)[:30]}...，使用零向量特征\")\n        \n        # 转换为张量\n        visual_feat = torch.tensor(visual_feat, dtype=torch.float32, device=DEVICE).unsqueeze(0)\n        vi_feat = torch.tensor(vi_feat, dtype=torch.float32, device=DEVICE).unsqueeze(0)\n        \n        # 4. 多模态预测\n        model.eval()\n        with torch.no_grad():\n            test_pred, attn_weights = model(\n                visual_feat, vi_feat, struct_feat, \n                ([state_val], [species_val])\n            )\n        \n        # 非负约束\n        test_pred = test_pred.cpu().numpy()[0]\n        test_pred = np.where(test_pred < 0, 0, test_pred)\n        \n        # 打印注意力权重（可选）\n        attn = attn_weights.cpu().numpy()[0]\n        print(f\"图片ID {img_id} | 模态注意力权重 [视觉, VI, 结构化, 文本]: {np.round(attn, 3)} | 预测：{np.round(test_pred, 4)}\")\n        \n        # 映射到结果\n        pred_dict = dict(zip(target_cols, test_pred))\n        id_data['target'] = id_data['target_name'].map(pred_dict).fillna(0.0)\n        id_data['target'] = id_data['target'].apply(lambda x: max(x, 0.0))\n        \n        test_df.loc[test_df['image_id'] == img_id, 'target'] = id_data['target'].values\n    \n    # 生成提交文件\n    submission_df = test_df[['sample_id', 'target']].copy()\n    submission_df['target'] = submission_df['target'].fillna(0.0).round(4)\n    \n    submission_path = 'submission.csv'\n    submission_df.to_csv(submission_path, index=False)\n    \n    print(f\"\\n=== 多模态提交文件生成完成 ===\")\n    print(f\"提交文件形状：{submission_df.shape}\")\n    print(f\"提交文件前5行预览：\\n{submission_df.head()}\")\n    print(f\"提交文件路径：{submission_path}\")\n    \n    return submission_path\n\n# ======================\n# 主函数（多模态版本）\n# ======================\ndef main():\n    def generate_backup_submission(test_df):\n        \"\"\"兜底提交文件生成\"\"\"\n        try:\n            backup_sub = test_df[['sample_id']].copy()\n            backup_sub['target'] = 0.0\n            backup_sub.to_csv('submission.csv', index=False)\n            print(f\"✅ 兜底提交文件生成成功（{len(backup_sub)}行）\")\n        except Exception as e:\n            print(f\"❌ 兜底提交文件生成失败：{e}\")\n    \n    try:\n        # 路径配置\n        train_path = '/kaggle/input/csiro-biomass/train.csv'\n        train_img_dir = TRAIN_IMG_DIR\n        test_img_dir = TEST_IMG_DIR\n        test_csv_path = TEST_CSV_PATH\n        \n        # 检查关键文件\n        required_files = [train_path, test_csv_path, train_img_dir]\n        missing_files = [f for f in required_files if not os.path.exists(f)]\n        if missing_files:\n            print(f\"警告：关键文件缺失 {missing_files}\")\n            if os.path.exists(test_csv_path):\n                test_df = pd.read_csv(test_csv_path)\n                generate_backup_submission(test_df)\n            return\n        \n        # 1. 初始化ConvNeXt提取器\n        print(\"=== 1. 初始化ConvNeXt-Large特征提取器 ===\")\n        convnext_extractor = init_convnext_feature_extractor()\n        print(f\"ConvNeXt提取器已加载（设备：{DEVICE}）\")\n        \n        # 2. 训练数据预处理（多模态）\n        print(\"\\n=== 2. 多模态训练数据预处理 ===\")\n        wide_df, target_cols, struct_feat_cols, text_feat_cols, scaler_struct, num_struct_cols = preprocess_structured_data(\n            train_path, train_img_dir\n        )\n        \n        # 3. 提取多模态特征\n        print(\"\\n=== 3. 提取多模态特征（视觉+植被指数） ===\")\n        train_image_paths = wide_df['full_image_path'].tolist()\n        visual_feat, vi_feat, scaler_visual, scaler_vi, missing_imgs = extract_multimodal_features(\n            train_image_paths, convnext_extractor, img_type=\"train\"\n        )\n        \n        # 4. 构建多模态数据集\n        print(\"\\n=== 4. 构建多模态数据集 ===\")\n        train_df, val_df = train_test_split(wide_df, test_size=0.2, random_state=SEED)\n        \n        # 按索引提取特征\n        train_visual = visual_feat[train_df.index]\n        train_vi = vi_feat[train_df.index]\n        val_visual = visual_feat[val_df.index]\n        val_vi = vi_feat[val_df.index]\n        \n        # 构建数据集\n        train_dataset = MultimodalBiomassDataset(\n            train_df, train_visual, train_vi,\n            struct_feat_cols, text_feat_cols, target_cols\n        )\n        val_dataset = MultimodalBiomassDataset(\n            val_df, val_visual, val_vi,\n            struct_feat_cols, text_feat_cols, target_cols\n        )\n        \n        # 关键：DataLoader传入自定义collate_fn\n        train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, collate_fn=multimodal_collate_fn)\n        val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, collate_fn=multimodal_collate_fn)\n        \n        print(f\"训练集：{len(train_dataset)}样本 | 验证集：{len(val_dataset)}样本\")\n        \n        # 5. 初始化多模态模型\n        print(\"\\n=== 5. 初始化多模态回归模型 ===\")\n        # 各模态维度\n        modal_dims = [\n            1536,                # 视觉模态（ConvNeXt）\n            12,                  # 植被指数模态\n            len(struct_feat_cols), # 结构化模态\n            2 * TEXT_EMBED_DIM   # 文本模态（State+Species）\n        ]\n        print(f\"各模态维度：视觉({modal_dims[0]}) | VI({modal_dims[1]}) | 结构化({modal_dims[2]}) | 文本({modal_dims[3]})\")\n        \n        model = MultimodalBiomassRegressor(\n            modal_dims=modal_dims,\n            output_dim=len(target_cols),\n            hidden_dims=[512, 256, 128]\n        ).to(DEVICE)\n        \n        # 优化器和损失函数\n        criterion = WeightedMSELoss(weights=[0.1, 0.1, 0.1, 0.2, 0.5])\n        optimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=1e-5)\n        scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=50, T_mult=2)\n        \n        print(f\"模型设备：{DEVICE} | 总参数量：{sum(p.numel() for p in model.parameters()):,}\")\n        \n        # 6. 训练多模态模型\n        print(\"\\n=== 6. 开始多模态模型训练 ===\")\n        model = train_multimodal_model(model, train_loader, val_loader, criterion, optimizer, scheduler, EPOCHS)\n        \n        # 7. 预处理test.csv\n        print(\"\\n=== 7. test.csv预处理 ===\")\n        test_df = preprocess_test_data(test_csv_path, test_img_dir)\n        \n        # 补充test_df的State/Species（默认值）\n        test_df['State'] = test_df.get('State', 'Tas')\n        test_df['Species'] = test_df.get('Species', 'Ryegrass_Clover')\n        \n        # 8. 多模态预测\n        print(\"\\n=== 8. 多模态测试集预测 ===\")\n        submission_path = predict_multimodal_test(\n            model, test_df, scaler_struct, scaler_visual, scaler_vi,\n            num_struct_cols, struct_feat_cols, convnext_extractor, target_cols\n        )\n        \n        print(f\"\\n=== 多模态流程完成 ===\")\n        print(f\"生成的提交文件：{submission_path}\")\n    \n    except Exception as e:\n        import traceback\n        print(f\"\\n=== 全局异常捕获：{str(e)} ===\")\n        traceback.print_exc()  # 打印详细堆栈信息，便于调试\n        if 'test_df' in locals():\n            generate_backup_submission(test_df)\n        elif os.path.exists(test_csv_path):\n            test_df = pd.read_csv(test_csv_path)\n            generate_backup_submission(test_df)\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-26T11:07:12.037689Z","iopub.execute_input":"2025-12-26T11:07:12.038039Z","iopub.status.idle":"2025-12-26T11:08:51.146694Z","shell.execute_reply.started":"2025-12-26T11:07:12.038012Z","shell.execute_reply":"2025-12-26T11:08:51.146051Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}