{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":277246590,"sourceType":"kernelVersion"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":51.117498,"end_time":"2025-11-12T16:35:18.197901","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-11-12T16:34:27.080403","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from __future__ import annotations\nimport logging\nimport gc\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport cv2\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport timm\nfrom tqdm import tqdm\n\nfrom IPython.display import display\n\n# ===================== 日志配置 =====================\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\nlogger = logging.getLogger(__name__)\nprint('读取分折好的数据')\ndisplay(pd.read_csv('/kaggle/input/0-create-fold/folds_by_Species.csv').head())\n# ===================== 配置类 =====================\nclass CFG:\n    # 路径相关\n    base_path = Path(\"/kaggle/input/csiro-biomass\")\n    train_csv = \"/kaggle/input/0-create-fold/folds_by_random.csv\"\n    train_image_dir = base_path / \"train\"\n    output_dir = Path(\"./models_eva02_single\")\n\n    # 模型与图像设置\n    model_name = \"eva02_base_patch14_448.mim_in22k_ft_in1k\"  # timm 模型名\n    img_size = 448\n    dropout = 0.1\n    num_targets = 5  # 直接预测 5 个 target\n\n    # 训练超参数\n    batch_size = 8\n    num_workers = 2\n    num_epochs = 20\n    lr = 1e-4\n    weight_decay = 1e-5\n\n    # 随机种子\n    seed = 42\n    # 设备\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    # target 列名（宽表中的顺序）\n    target_cols = [\"Dry_Green_g\",\"Dry_Dead_g\",\"Dry_Clover_g\",\"GDM_g\",\"Dry_Total_g\",]\n    loss_weights = [0.1, 0.1, 0.1, 0.2, 0.5]  # 对应 target_cols 的顺序\n    wandb_project = 'CSIRO_mac14'\nCFG.MODEL_ = CFG.model_name.split('.')[0]    \n# 创建输出目录\nCFG.output_dir.mkdir(parents=True, exist_ok=True)\n\n# 设置随机种子\nnp.random.seed(CFG.seed)\ntorch.manual_seed(CFG.seed)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed_all(CFG.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:55:58.463528Z","iopub.execute_input":"2025-11-13T16:55:58.463719Z","iopub.status.idle":"2025-11-13T16:56:12.710547Z","shell.execute_reply.started":"2025-11-13T16:55:58.463702Z","shell.execute_reply":"2025-11-13T16:56:12.709951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===================== 数据增强接口 =====================\ndef get_train_transforms(img_size):\n    \"\"\"训练阶段的数据增强（可自行修改）\"\"\"\n    return A.Compose(\n        [\n            # 随机水平翻转\n            A.HorizontalFlip(p=0.5),\n            # 随机垂直翻转\n            A.VerticalFlip(p=0.2),\n            # 调整到统一尺寸\n            A.Resize(img_size, img_size),\n            # 标准化\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ]\n    )\n\n\ndef get_valid_transforms(img_size):\n    \"\"\"验证阶段的数据预处理（不做随机增强）\"\"\"\n    return A.Compose(\n        [\n            A.Resize(img_size, img_size),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ]\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:56:12.712174Z","iopub.execute_input":"2025-11-13T16:56:12.712608Z","iopub.status.idle":"2025-11-13T16:56:12.717517Z","shell.execute_reply.started":"2025-11-13T16:56:12.712589Z","shell.execute_reply":"2025-11-13T16:56:12.716835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass WeightedMSELoss(nn.Module):\n    \"\"\"对多目标回归按列加权的 MSE loss，适配本题 5 个 target\"\"\"\n    def __init__(self, weights):\n        super().__init__()\n        # weights: list 或 1D tensor，长度等于 num_targets\n        # 注册为 buffer，这样可以自动放到对应 device 上\n        self.register_buffer(\"weights\", torch.tensor(weights).view(1, -1))\n\n    def forward(self, preds, targets):\n        \"\"\"\n        preds, targets: [batch_size, num_targets]，这里是在 log 空间\n        \"\"\"\n        # 计算逐元素平方误差: [B, T]\n        diff2 = (preds - targets) ** 2\n\n        # 按列加权: 每一列乘对应 weight\n        # weights shape 是 [1, T]，可以广播到 [B, T]\n        weighted_diff2 = diff2 * self.weights\n\n        # 这里的平均方式有两种常见写法：\n        # 1）直接对所有元素求 mean（等价于 sum / (B * T)）\n        loss = weighted_diff2.mean()\n\n        # 如果你想让总权重归一化（严格按“总权重除以 sum(weights)”来算），可以改成：\n        # loss = weighted_diff2.sum() / (preds.size(0) * self.weights.sum())\n\n        return loss\n\nclass TrainBiomassDataset(Dataset):\n    \"\"\"\n    训练数据集：\n    - 输入：单流图像\n    - 输出：log1p 变换后的 5 个目标值\n    \"\"\"\n    def __init__(self, df, image_dir, target_cols, transform):\n        \"\"\"\n        df: 宽表，每行一个 image_path + 多个 target 列\n        image_dir: 图像所在文件夹路径\n        target_cols: 目标列名列表\n        transform: Albumentations 数据增强\n        \"\"\"\n        self.df = df.reset_index(drop=True)\n        self.image_dir = image_dir\n        self.image_paths = self.df[\"image_path\"].values\n        self.target_cols = target_cols\n        self.transform = transform\n\n        # 从 df 中提取原始 target 数值，类型为 float32\n        targets = self.df[self.target_cols].values.astype(np.float32)\n\n        # 对 target 做 log1p 变换（log(1+y)），用于稳定训练\n        self.targets_log = np.log1p(targets)\n\n    def __len__(self):\n        \"\"\"返回数据集大小\"\"\"\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        \"\"\"返回单个样本：图像张量 + log1p 后的目标张量\"\"\"\n\n        # 取图像路径\n        img_path = self.image_paths[idx]\n\n        # 只使用文件名拼接到 train 目录（保持与你的推理一致）\n        full_path = self.image_dir / Path(img_path).name\n\n        # 使用 cv2 读取图像（默认 BGR）\n        image = cv2.imread(str(full_path))\n\n        # 如果读取失败，输出警告并使用全 0 图像代替\n        if image is None:\n            logger.warning(f\"Failed to load image: {full_path}, using zeros.\")\n            image = np.zeros((1000, 2000, 3), dtype=np.uint8)\n\n        # BGR → RGB（符合深度学习模型读取格式）\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        # 应用 Albumentations 数据增强\n        augmented = self.transform(image=image)\n        image_tensor = augmented[\"image\"]  # tensor，shape: (3, H, W)\n\n        # 取 log1p 后的目标值\n        target_log = self.targets_log[idx]\n        target_tensor = torch.from_numpy(target_log)\n\n        return image_tensor, target_tensor\nclass SimpleBiomassModel(nn.Module):\n    \"\"\"\n    单流图像回归模型：\n    - 使用 timm backbone 提取图像特征\n    - 使用全连接回归头输出 5 个标签的 log 空间预测\n    \"\"\"\n    def __init__(self, model_name, num_targets, dropout=0.3):\n        super().__init__()\n\n        # 创建 timm backbone，不包含分类头\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=True,  # 使用预训练权重有助于收敛\n            num_classes=0,    # 不要分类头（使用 features 输出）\n            global_pool=\"avg\",  # 使用平均池化得到全局向量\n        )\n\n        # backbone 输出特征维度\n        n_features = self.backbone.num_features\n\n        # 回归头（两层 MLP）\n        self.head = nn.Sequential(\n            nn.Linear(n_features, n_features // 2),\n            nn.ReLU(inplace=True),\n            nn.Dropout(dropout),\n            nn.Linear(n_features // 2, num_targets),\n        )\n\n    def forward(self, x):\n        \"\"\"\n        前向传播：\n        x: 输入图像张量 [B, C, H, W]\n        返回：log 空间的预测值 [B, num_targets]\n        \"\"\"\n        # 特征提取\n        feat = self.backbone(x)\n\n        # 回归输出（log 空间）\n        out = self.head(feat)\n\n        return out\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:56:12.718393Z","iopub.execute_input":"2025-11-13T16:56:12.718717Z","iopub.status.idle":"2025-11-13T16:56:12.743847Z","shell.execute_reply.started":"2025-11-13T16:56:12.718692Z","shell.execute_reply":"2025-11-13T16:56:12.743191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===================== 单个 epoch 训练函数（支持 EMA） =====================\ndef train_one_epoch(model, loader, optimizer, device, ema=None):\n    \"\"\"\n    单个 epoch 的训练循环，返回平均 loss（log 空间，加权 MSE）\n\n    参数:\n    - model: 当前训练的模型\n    - loader: 训练集 DataLoader\n    - optimizer: 优化器\n    - device: 运行设备（CPU/GPU）\n    - ema: 可选的 ModelEMA 实例；如果传入，则在每个 step 后更新 EMA 模型参数\n    \"\"\"\n    model.train()\n    running_loss = 0.0\n\n    # 在 log 空间使用加权 MSE，对 5 个 target 使用不同权重\n    loss_weights = [0.1, 0.1, 0.1, 0.2, 0.5]  # 对应 target_cols 的顺序\n    criterion = WeightedMSELoss(loss_weights).to(device)\n\n    for images, targets_log in tqdm(loader, desc=\"Train\", leave=False):\n        # images: [B, C, H, W]\n        # targets_log: [B, 5]，已经在外面做过 log1p 之类的变换\n        images = images.to(device)\n        targets_log = targets_log.to(device)\n\n        optimizer.zero_grad()\n        preds_log = model(images)          # [B, 5]\n\n        # 计算加权 MSE loss（log 空间）\n        loss = criterion(preds_log, targets_log)\n\n        loss.backward()\n        optimizer.step()\n\n        # 如果传入了 EMA 对象，则在每个 step 后更新 EMA 模型参数\n        if ema is not None:\n            ema.update(model)\n\n        # 累加 batch loss（注意这里用 batch_size 加权平均）\n        running_loss += loss.item() * images.size(0)\n\n    # 返回整个 epoch 的平均 loss（对所有样本做平均）\n    epoch_loss = running_loss / len(loader.dataset)\n    return epoch_loss\n\n\n# ===================== 验证函数 =====================\ndef validate(model, loader, device):\n    \"\"\"\n    验证循环，返回：\n    - avg_loss: 原始空间的加权 MSELoss（自定义监控用）\n    - weighted_r2: 原始空间的加权 R²（严格按官方公式）\n    \"\"\"\n    model.eval()\n\n    # 验证时的 target 权重（与比赛一致）\n    weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])  # 顺序必须与 target_cols 对齐\n\n    running_loss = 0.0\n    all_true = []\n    all_pred = []\n\n    with torch.no_grad():\n        for images, targets_log in tqdm(loader, desc=\"Valid\", leave=False):\n            images = images.to(device)\n            targets_log = targets_log.to(device)\n\n            # 模型输出仍然在 log 空间\n            preds_log = model(images)\n\n            # 反 log 得到原始空间：y = exp(log_y) - 1\n            preds = torch.expm1(preds_log).cpu().numpy()\n            trues = torch.expm1(targets_log).cpu().numpy()\n\n            all_pred.append(preds)\n            all_true.append(trues)\n\n            # ---------- 原空间加权 MSE（监控用）-----------\n            diff2 = (preds - trues) ** 2        # [B, 5]\n            weighted_diff2 = diff2 * weights    # 广播到 [B, 5]\n            batch_loss = weighted_diff2.mean()  # 对所有元素平均\n            running_loss += batch_loss * preds.shape[0]\n\n    # 拼接所有样本\n    y_true = np.concatenate(all_true, axis=0)   # [N, 5]\n    y_pred = np.concatenate(all_pred, axis=0)   # [N, 5]\n\n    # 对所有样本做平均损失\n    avg_loss = running_loss / len(y_true)\n\n    # ---------- 按官方公式计算加权 R² -----------\n    weighted_r2 = weighted_r2_score(y_true, y_pred, weights)\n\n    return avg_loss, weighted_r2\n\n\n# ===================== 加权 R² 计算函数（严格按官方定义） =====================\ndef weighted_r2_score(y_true, y_pred, weights_per_target):\n    \"\"\"\n    y_true, y_pred: [N, T] 的 numpy 数组，T=5\n    weights_per_target: 长度为 T 的 1D 数组，例如 [0.1, 0.1, 0.1, 0.2, 0.5]\n    按照官方公式计算全局加权 R²。\n\n    数学形式：\n        R_w^2 = 1 - sum_j w_j (y_j - ŷ_j)^2 / sum_j w_j (y_j - ȳ_w)^2\n        其中 ȳ_w = sum_j w_j y_j / sum_j w_j\n    \"\"\"\n    # 转成 numpy 数组，确保是 float\n    y_true = np.asarray(y_true, dtype=np.float64)\n    y_pred = np.asarray(y_pred, dtype=np.float64)\n    w_t = np.asarray(weights_per_target, dtype=np.float64)   # [T]\n\n    # 展开成一维：所有 (sample, target) 拼到一起\n    N, T = y_true.shape\n    y_true_flat = y_true.reshape(-1)   # [N*T]\n    y_pred_flat = y_pred.reshape(-1)   # [N*T]\n\n    # 为每个元素生成对应的权重 w_j\n    # 把 [T] 的权重按样本数复制 N 次，再展平 -> [N*T]\n    w_flat = np.broadcast_to(w_t, (N, T)).reshape(-1)\n\n    # 全局加权均值 ȳ_w\n    y_bar_w = np.sum(w_flat * y_true_flat) / (np.sum(w_flat) + 1e-8)\n\n    # 加权残差平方和 SS_res\n    ss_res = np.sum(w_flat * (y_true_flat - y_pred_flat) ** 2)\n\n    # 加权总平方和 SS_tot\n    ss_tot = np.sum(w_flat * (y_true_flat - y_bar_w) ** 2)\n\n    # 加权 R²\n    r2_w = 1.0 - ss_res / (ss_tot + 1e-8)\n    return float(r2_w)\n\n\n# ===================== 手写版本 EMA（适配任意模型） =====================\nfrom copy import deepcopy\n\nclass ModelEMA:\n    \"\"\"\n    简单稳定的 EMA（指数滑动平均），适配任意 PyTorch 模型\n\n    用法示例：\n        ema = ModelEMA(model, decay=0.999)\n        ...\n        # 每个 step 后\n        ema.update(model)\n        ...\n        # 验证时使用 ema.ema_model\n        valid_loss, valid_r2 = validate(ema.ema_model, valid_loader, device)\n    \"\"\"\n    def __init__(self, model, decay=0.999):\n        # 深拷贝一份模型（EMA 模型不会更新梯度）\n        self.ema_model = deepcopy(model)\n        self.ema_model.eval()\n\n        # EMA 衰减因子（越接近 1，更新越平滑）\n        self.decay = decay\n\n        # EMA 参数不参与梯度计算\n        for p in self.ema_model.parameters():\n            p.requires_grad_(False)\n\n    @torch.no_grad()\n    def update(self, model):\n        \"\"\"EMA 参数更新：ema = decay * ema + (1 - decay) * model\"\"\"\n        ema_state = self.ema_model.state_dict()\n        model_state = model.state_dict()\n\n        for key in ema_state.keys():\n            # ema_param = decay * ema_param + (1-decay) * new_param\n            ema_state[key].mul_(self.decay).add_(model_state[key], alpha=1 - self.decay)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:56:12.744612Z","iopub.execute_input":"2025-11-13T16:56:12.744864Z","iopub.status.idle":"2025-11-13T16:56:12.760684Z","shell.execute_reply.started":"2025-11-13T16:56:12.744847Z","shell.execute_reply":"2025-11-13T16:56:12.759928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import wandb\nimport gc\nfrom kaggle_secrets import UserSecretsClient\n\n# ================================\n# 5-fold 训练 + wandb + EarlyStopping（简单版）\n# ================================\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandb\")\nwandb.login(key=secret_value_0)\n\nfor csv_path in [\n    '/kaggle/input/0-create-fold/folds_by_random.csv',\n    '/kaggle/input/0-create-fold/folds_by_Pre_GSHH_NDVI.csv',\n    '/kaggle/input/0-create-fold/folds_by_Sampling_Date.csv',\n    '/kaggle/input/0-create-fold/folds_by_Species.csv',\n]:\n    CFG.train_csv = csv_path\n    Group_mode = csv_path.split('/')[-1].split('.')[0].split('_')[-1]\n\n    train_wide = pd.read_csv(CFG.train_csv)\n    fold_scores = []\n\n    for fold_idx in range(5):\n\n        # ============================\n        # wandb 初始化\n        # ============================\n        run = wandb.init(\n            project=CFG.wandb_project,\n            name=f\"{Group_mode}_{CFG.MODEL_}_fold{fold_idx}_size{CFG.img_size}_bs{CFG.batch_size}_lr{CFG.lr}\",\n            config={\n                \"Group_mode\": Group_mode,\n                \"model_name\": CFG.model_name,\n                \"fold_idx\": fold_idx,\n                \"image_size\": int(CFG.img_size),\n                \"batch_size\": int(CFG.batch_size),\n                \"lr\": float(CFG.lr),\n                \"num_epochs\": int(CFG.num_epochs),\n            },\n            reinit=True,\n        )\n\n        # ============================\n        # 划分训练 / 验证\n        # ============================\n        df_train = train_wide[train_wide[\"fold\"] != fold_idx].reset_index(drop=True)\n        df_valid = train_wide[train_wide[\"fold\"] == fold_idx].reset_index(drop=True)\n\n        train_transform = get_train_transforms(CFG.img_size)\n        valid_transform = get_valid_transforms(CFG.img_size)\n\n        train_dataset = TrainBiomassDataset(\n            df=df_train,\n            image_dir=CFG.train_image_dir,\n            target_cols=CFG.target_cols,\n            transform=train_transform,\n        )\n        valid_dataset = TrainBiomassDataset(\n            df=df_valid,\n            image_dir=CFG.train_image_dir,\n            target_cols=CFG.target_cols,\n            transform=valid_transform,\n        )\n\n        train_loader = DataLoader(\n            train_dataset,\n            batch_size=CFG.batch_size,\n            shuffle=True,\n            num_workers=CFG.num_workers,\n            pin_memory=True,\n        )\n        valid_loader = DataLoader(\n            valid_dataset,\n            batch_size=CFG.batch_size,\n            shuffle=False,\n            num_workers=CFG.num_workers,\n            pin_memory=True,\n        )\n\n        # ============================\n        # 模型与优化器\n        # ============================\n        model = SimpleBiomassModel(\n            model_name=CFG.model_name,\n            num_targets=len(CFG.target_cols),\n            dropout=CFG.dropout,\n        ).to(CFG.device)\n\n        optimizer = torch.optim.AdamW(\n            model.parameters(),\n            lr=CFG.lr,\n            weight_decay=CFG.weight_decay,\n        )\n\n        # ============================\n        # Early Stopping\n        # ============================\n        patience = 5\n        es_counter = 0\n        best_r2 = -1e9\n        best_epoch = 0\n        best_model_path = CFG.output_dir / f\"{Group_mode}_best_model_fold{fold_idx}.pth\"\n\n        # ============================\n        # 训练循环（朴素版）\n        # ============================\n        for epoch in range(1, CFG.num_epochs + 1):\n\n            logger.info(f\"[{Group_mode}] Fold {fold_idx} Epoch {epoch}/{CFG.num_epochs}\")\n\n            # 训练\n            train_loss = train_one_epoch(model, train_loader, optimizer, CFG.device)\n\n            # 验证\n            valid_loss, valid_r2 = validate(model, valid_loader, CFG.device)\n\n            print(\n                f\"[{Group_mode}] Fold {fold_idx} | Epoch {epoch} | \"\n                f\"Train Loss: {train_loss:.4f} | Valid Loss: {valid_loss:.4f} | \"\n                f\"Valid R2: {valid_r2:.4f}\"\n            )\n\n            # wandb 记录\n            wandb.log(\n                {\n                    \"epoch\": epoch,\n                    \"fold\": fold_idx,\n                    \"Group_mode\": Group_mode,\n                    \"train_loss\": train_loss,\n                    \"valid_loss\": valid_loss,\n                    \"valid_r2\": valid_r2,\n                }\n            )\n\n            # ============================\n            # EarlyStopping逻辑\n            # ============================\n            if valid_r2 > best_r2:\n                best_r2 = valid_r2\n                best_epoch = epoch\n                es_counter = 0\n                torch.save(model.state_dict(), best_model_path)\n                wandb.run.summary[\"best_valid_r2\"] = best_r2\n                wandb.run.summary[\"best_epoch\"] = best_epoch\n                print(f\"  ✓ New best model saved: {best_model_path}\")\n            else:\n                es_counter += 1\n                print(f\"  ✗ No improvement ({es_counter}/{patience})\")\n\n            if es_counter >= patience:\n                print(f\"  🔥 Early stopping triggered at epoch {epoch}\")\n                break\n\n        print(f\"[{Group_mode}] Fold {fold_idx} Best R2: {best_r2:.4f} at epoch {best_epoch}\")\n        fold_scores.append(best_r2)\n\n        wandb.finish()\n\n        gc.collect()\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n\n    # ============================\n    # 汇总\n    # ============================\n    print(f\"\\n========== 5-FOLD RESULTS ({Group_mode}) ==========\")\n    for i, score in enumerate(fold_scores):\n        print(f\"Fold {i} R2: {score:.4f}\")\n\n    print(f\"\\n⭐ Average R2 ({Group_mode}): {np.mean(fold_scores):.4f}\\n\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T16:56:12.761362Z","iopub.execute_input":"2025-11-13T16:56:12.761590Z"}},"outputs":[],"execution_count":null}]}