{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":4535,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3327,"modelId":986}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# version_stacking.py\n# 基于 DINOv2 + Stacking Ensemble (LGBM + XGB + Ridge) 的高效生物量预测方案\n# 核心思路：\n# 1. 保持 DINOv2 特征提取不变 (包含 v2 的特征增强: CLS+AVG+MAX, TTA)\n# 2. Level 1: 训练 LightGBM, XGBoost, Ridge 三种差异化模型\n# 3. Level 2: 使用 Ridge 回归进行 Stacking 融合 (抗过拟合)\n\nimport os\nimport sys\n\n# 必须在导入 transformers/tensorflow 之前设置\nos.environ[\"USE_TF\"] = \"0\"\nos.environ[\"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION\"] = \"python\"\n\nimport gc\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom pathlib import Path\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nfrom transformers import AutoModel\n\n# Models\nimport lightgbm as lgb\nimport xgboost as xgb\n# import optuna  # 移除 Optuna\nfrom sklearn.linear_model import Ridge, LinearRegression\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nfrom sklearn.model_selection import KFold, RepeatedKFold\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.base import BaseEstimator, RegressorMixin, clone\n\n# 屏蔽警告\nwarnings.filterwarnings('ignore')\n\n# ====================================================\n# 配置 (Configuration)\n# ====================================================\nclass CFG:\n    DATA_PATH = '/kaggle/input/csiro-biomass'\n    train_path = os.path.join(DATA_PATH, 'train.csv')\n    test_path = os.path.join(DATA_PATH, 'test.csv')\n    \n    # DINOv2 模型路径 (支持 Kaggle 离线路径)\n    dinov2_paths = [\n        './dinov2_large',                                            # 本地路径\n        '/kaggle/input/dinov2/pytorch/large/1',                      # Kaggle 常用路径\n        '/kaggle/input/facebook-dinov2-large'                        # 备选路径\n    ]\n    \n    # 增大图片分辨率至 518 (DINOv2 官方推荐的高分辨率设置)\n    img_size = 518 \n    # 分辨率增大后，显存占用增加，需减小 Batch Size\n    batch_size = 8\n    \n    seed = 42\n    n_fold = 5\n    \n    target_cols = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    target_weights = [0.1, 0.1, 0.1, 0.2, 0.5]\n    \n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# ====================================================\n# 工具函数\n# ====================================================\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n\nseed_everything(CFG.seed)\n\ndef weighted_r2_metric(y_true, y_pred):\n    \"\"\"比赛评估指标: 加权 R2\"\"\"\n    weights = np.array(CFG.target_weights)\n    if y_true.ndim == 1: y_true = y_true.reshape(-1, 1)\n    if y_pred.ndim == 1: y_pred = y_pred.reshape(-1, 1)\n    \n    ss_res = np.sum(weights * np.sum((y_true - y_pred)**2, axis=0))\n    global_mean = np.average(np.mean(y_true, axis=0), weights=weights)\n    ss_tot = np.sum(weights * np.sum((y_true - global_mean)**2, axis=0))\n    return 1 - (ss_res / (ss_tot + 1e-8))\n\ndef post_process_biomass(df_preds):\n    \"\"\"物理约束后处理\"\"\"\n    ordered_cols = [\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\", \"Dry_Total_g\"]\n    if not all(col in df_preds.columns for col in ordered_cols):\n        return df_preds\n    \n    Y = df_preds[ordered_cols].values.T\n    C = np.array([[1, 1, 0, -1, 0], [0, 0, 1, 1, -1]], dtype=float)\n    \n    try:\n        inv_CCt = np.linalg.inv(C @ C.T)\n    except np.linalg.LinAlgError:\n        inv_CCt = np.linalg.pinv(C @ C.T)\n    \n    P = np.eye(5) - C.T @ inv_CCt @ C\n    Y_reconciled = (P @ Y).T.clip(min=0)\n    \n    df_out = df_preds.copy()\n    df_out[ordered_cols] = Y_reconciled\n    \n    # 简单修正\n    df_out['GDM_g'] = df_out['Dry_Green_g'] + df_out['Dry_Clover_g']\n    df_out['Dry_Total_g'] = df_out['GDM_g'] + df_out['Dry_Dead_g']\n    return df_out\n\n# ====================================================\n# 数据加载与处理\n# ====================================================\ndef load_data():\n    train_df = pd.read_csv(CFG.train_path)\n    test_df = pd.read_csv(CFG.test_path)\n    \n    def parse_ids(df):\n        if 'sample_id' in df.columns:\n            df[['image_id', 'target_type']] = df['sample_id'].str.split('__', n=1, expand=True)\n        return df\n\n    train_df = parse_ids(train_df)\n    test_df = parse_ids(test_df)\n    \n    meta_cols = ['image_id', 'image_path']\n    train_pivot = train_df.pivot_table(index=meta_cols, columns='target_name', values='target').reset_index()\n    train_pivot.columns.name = None\n    \n    test_pivot = test_df[meta_cols].drop_duplicates().reset_index(drop=True)\n    \n    return train_pivot, test_pivot\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.image_paths = df['image_path'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.image_paths)\n    \n    def __getitem__(self, idx):\n        path = os.path.join(CFG.DATA_PATH, self.image_paths[idx])\n        try:\n            image = Image.open(path).convert(\"RGB\")\n        except:\n            image = Image.new(\"RGB\", (256, 256))\n            \n        if self.transform:\n            image = self.transform(image)\n        return image\n\n# ====================================================\n# 特征提取 (DINOv2)\n# ====================================================\ndef get_dinov2_model():\n    if AutoModel is not None:\n        for path in CFG.dinov2_paths:\n            if os.path.exists(path):\n                print(f\"Loading DINOv2 (Transformers) from: {path}\")\n                try:\n                    model = AutoModel.from_pretrained(path, local_files_only=True)\n                    return model.to(CFG.device).eval()\n                except Exception as e:\n                    print(f\"Failed to load from {path}: {e}\")\n\n    print(\"Trying to load via torch.hub (facebookresearch/dinov2)...\")\n    try:\n        torch.hub.set_dir('/kaggle/working/torch_hub_cache')\n        model = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitl14', pretrained=True)\n        print(\"Successfully loaded DINOv2 via torch.hub!\")\n        return model.to(CFG.device).eval()\n    except Exception as e:\n        print(f\"torch.hub load failed: {e}\")\n        \n    if AutoModel is not None:\n        print(\"Warning: Local DINOv2 not found. Trying to download...\")\n        try:\n            model = AutoModel.from_pretrained('facebook/dinov2-large')\n            return model.to(CFG.device).eval()\n        except Exception as e:\n            print(f\"Transformers download failed: {e}\")\n\n    raise RuntimeError(\"Could not load DINOv2 model.\")\n\ndef extract_features(df, model):\n    print(f\"Extracting features for {len(df)} images...\")\n    \n    mean = [0.485, 0.456, 0.406]\n    std = [0.229, 0.224, 0.225]\n    \n    transforms_list = [\n        # 使用 Resize 保持宽高比并 Crop，或者直接 Resize 到目标尺寸 (这里采用 Resize+CenterCrop 以匹配预训练习惯)\n        T.Compose([T.Resize(CFG.img_size), T.CenterCrop(CFG.img_size), T.ToTensor(), T.Normalize(mean, std)]),\n        T.Compose([T.Resize(CFG.img_size), T.CenterCrop(CFG.img_size), T.RandomHorizontalFlip(p=1.0), T.ToTensor(), T.Normalize(mean, std)]),\n        T.Compose([T.Resize(CFG.img_size), T.CenterCrop(CFG.img_size), T.RandomVerticalFlip(p=1.0), T.ToTensor(), T.Normalize(mean, std)])\n    ]\n    \n    is_transformer_model = hasattr(model, 'last_hidden_state') or hasattr(model, 'embeddings')\n    feats_tta = []\n    \n    for i, transform in enumerate(transforms_list):\n        ds = BiomassDataset(df, transform=transform)\n        ld = DataLoader(ds, batch_size=CFG.batch_size, shuffle=False, num_workers=2)\n        \n        feats_epoch = []\n        with torch.no_grad():\n            for imgs in tqdm(ld, desc=f\"TTA Round {i+1}\"):\n                imgs = imgs.to(CFG.device)\n                \n                if is_transformer_model:\n                    outputs = model(imgs)\n                    last_hidden = outputs.last_hidden_state\n                    cls_token = last_hidden[:, 0, :]\n                    patch_tokens = last_hidden[:, 1:, :]\n                    avg_pool = patch_tokens.mean(dim=1)\n                    max_pool = patch_tokens.max(dim=1)[0]\n                    emb = torch.cat([cls_token, avg_pool, max_pool], dim=1)\n                else:\n                    outputs = model.forward_features(imgs)\n                    if isinstance(outputs, dict):\n                        cls_token = outputs['x_norm_clstoken']\n                        patch_tokens = outputs['x_norm_patchtokens']\n                        avg_pool = patch_tokens.mean(dim=1)\n                        max_pool = patch_tokens.max(dim=1)[0]\n                        emb = torch.cat([cls_token, avg_pool, max_pool], dim=1)\n                    else:\n                        if outputs.dim() == 3:\n                            cls_token = outputs[:, 0, :]\n                            patch_tokens = outputs[:, 1:, :]\n                            avg_pool = patch_tokens.mean(dim=1)\n                            max_pool = patch_tokens.max(dim=1)[0]\n                            emb = torch.cat([cls_token, avg_pool, max_pool], dim=1)\n                        else:\n                            emb = outputs\n                        \n                feats_epoch.append(emb.cpu().numpy())\n        feats_tta.append(np.concatenate(feats_epoch))\n    \n    final_feats = np.mean(feats_tta, axis=0)\n    feat_cols = [f\"feat_{i}\" for i in range(final_feats.shape[1])]\n    return pd.DataFrame(final_feats, columns=feat_cols)\n\n# ====================================================\n# Stacking Ensemble\n# ====================================================\nclass StackingTrainer:\n    def __init__(self, n_folds=5, seed=42, lgbm_params=None, xgb_params=None):\n        self.n_folds = n_folds\n        self.seed = seed\n        self.models = {}\n        self.lgbm_params = lgbm_params\n        self.xgb_params = xgb_params\n        \n    def get_base_models(self):\n        \"\"\"定义基模型 (Level 1)\"\"\"\n        # 1. LightGBM\n        if self.lgbm_params is None:\n            self.lgbm_params = {\n                'n_estimators': 300,\n                'learning_rate': 0.05,\n                'num_leaves': 31,\n                'random_state': self.seed,\n                'n_jobs': -1,\n                'verbose': -1\n            }\n            if torch.cuda.is_available():\n                 self.lgbm_params.update({'device': 'gpu'})\n                 \n        lgbm = lgb.LGBMRegressor(**self.lgbm_params)\n        \n        # 2. XGBoost (GPU Accelerated)\n        if self.xgb_params is None:\n             self.xgb_params = {\n                'n_estimators': 300,\n                'learning_rate': 0.05,\n                'max_depth': 6,\n                'subsample': 0.8,\n                'colsample_bytree': 0.8,\n                'objective': 'reg:squarederror',\n                'n_jobs': -1,\n                'random_state': self.seed,\n                'tree_method': 'hist',\n                'device': 'cuda' if torch.cuda.is_available() else 'cpu'\n            }\n        xgb_model = xgb.XGBRegressor(**self.xgb_params)\n        \n        # 3. Ridge Regression (线性模型，抗过拟合)\n        ridge = Ridge(alpha=1.0, random_state=self.seed)\n        \n        return {\n            'lgbm': MultiOutputRegressor(lgbm),\n            'xgb': MultiOutputRegressor(xgb_model),\n            'ridge': MultiOutputRegressor(ridge)\n        }\n\n    def train_predict(self, X, y, X_test):\n        \"\"\"训练基模型并生成 Out-of-Fold 预测和测试集预测\"\"\"\n        kf = KFold(n_splits=self.n_folds, shuffle=True, random_state=self.seed)\n        \n        base_models = self.get_base_models()\n        self.models = base_models # Store for reference\n        \n        oof_preds = {name: np.zeros(y.shape) for name in base_models}\n        test_preds = {name: np.zeros((X_test.shape[0], y.shape[1])) for name in base_models}\n        \n        print(\"Training Base Models...\")\n        \n        for fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):\n            print(f\"  Fold {fold+1}/{self.n_folds}\")\n            X_tr, X_val = X[train_idx], X[val_idx]\n            y_tr, y_val = y[train_idx], y[val_idx]\n            \n            for name, model in base_models.items():\n                # Clone model for each fold to avoid state leakage\n                model_fold = clone(model)\n                model_fold.fit(X_tr, y_tr)\n                \n                # Predict Val\n                val_pred = model_fold.predict(X_val)\n                val_pred = np.clip(val_pred, 0, None) # Clip negative values\n                oof_preds[name][val_idx] = val_pred\n                \n                # Predict Test\n                test_pred = model_fold.predict(X_test)\n                test_pred = np.clip(test_pred, 0, None)\n                test_preds[name] += test_pred / self.n_folds\n                \n        return oof_preds, test_preds\n\n    def fit_stacking(self, oof_preds, y_true, test_preds_dict):\n        \"\"\"训练元模型 (Level 2) - 使用简单的加权或 Ridge\"\"\"\n        print(\"\\nTraining Meta Model (Stacking)...\")\n        \n        # 准备 Level 2 数据: 将不同模型的预测结果拼接\n        # Shape: (N_samples, N_models * N_targets)\n        def prepare_level2_data(preds_dict):\n            # 堆叠策略：对每个目标变量单独堆叠\n            # 但为了简单，我们这里先采用简单的平均融合作为 Baseline，或者训练一个 Meta Ridge\n            # 这里的实现：对每个 Target，取所有基模型对该 Target 的预测作为特征\n            \n            # 简单起见，我们对每个 Target 单独训练 Meta Model\n            return preds_dict\n            \n        final_oof = np.zeros_like(y_true)\n        final_test_pred = np.zeros_like(list(test_preds_dict.values())[0])\n        \n        meta_scores = []\n        \n        # 对 5 个目标变量分别训练 Meta Model\n        for i in range(5): # 5 targets\n            # 构建该 Target 的特征矩阵: (N, 3) -> [LGBM_pred, XGB_pred, Ridge_pred]\n            X_meta_train = np.column_stack([\n                oof_preds['lgbm'][:, i],\n                oof_preds['xgb'][:, i],\n                oof_preds['ridge'][:, i]\n            ])\n            \n            X_meta_test = np.column_stack([\n                test_preds_dict['lgbm'][:, i],\n                test_preds_dict['xgb'][:, i],\n                test_preds_dict['ridge'][:, i]\n            ])\n            \n            y_meta_train = y_true[:, i]\n            \n            # Meta Model: Ridge (Positive constraint by post-clip)\n            meta_model = Ridge(alpha=10.0, random_state=self.seed)\n            meta_model.fit(X_meta_train, y_meta_train)\n            \n            # Predict\n            final_oof[:, i] = meta_model.predict(X_meta_train)\n            final_test_pred[:, i] = meta_model.predict(X_meta_test)\n            \n            # Log weights (coef) to see contribution\n            print(f\"  Target {i} Weights -> LGBM: {meta_model.coef_[0]:.3f}, XGB: {meta_model.coef_[1]:.3f}, Ridge: {meta_model.coef_[2]:.3f}\")\n            \n        final_oof = np.clip(final_oof, 0, None)\n        final_test_pred = np.clip(final_test_pred, 0, None)\n        \n        return final_oof, final_test_pred\n\n# ====================================================\n# 主流程\n# ====================================================\ndef main():\n    print(\"Initializing Stacking Pipeline (LGBM + XGB + Ridge)...\")\n    \n    # 1. 数据准备\n    train_df, test_df = load_data()\n    print(f\"Train shape: {train_df.shape}, Test shape: {test_df.shape}\")\n    \n    # 2. 特征提取\n    model = get_dinov2_model()\n    X_train = extract_features(train_df, model)\n    X_test = extract_features(test_df, model)\n    \n    del model\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    # 3. 预处理\n    print(\"Preprocessing features...\")\n    # 使用 RobustScaler 抗异常值\n    scaler = RobustScaler()\n    X_train_scaled = scaler.fit_transform(X_train)\n    X_test_scaled = scaler.transform(X_test)\n    \n    Y_train = train_df[CFG.target_cols].values\n    # Log 变换 (用于基模型训练)\n    Y_train_log = np.log1p(Y_train)\n    \n    # 4. Stacking 训练\n    trainer = StackingTrainer(n_folds=CFG.n_fold, seed=CFG.seed)\n    \n    # Step 1: Base Models (Level 1)\n    # 注意：基模型在 Log 空间训练\n    oof_preds_log, test_preds_log = trainer.train_predict(X_train_scaled, Y_train_log, X_test_scaled)\n    \n    # Step 2: Meta Model (Level 2)\n    # Meta Model 也在 Log 空间训练，或者还原后训练？通常在 Log 空间继续融合效果更稳\n    final_oof_log, final_test_pred_log = trainer.fit_stacking(oof_preds_log, Y_train_log, test_preds_log)\n    \n    # 还原到原始空间\n    final_oof = np.expm1(final_oof_log)\n    final_test_pred = np.expm1(final_test_pred_log)\n    \n    # 评估\n    score = weighted_r2_metric(Y_train, final_oof)\n    print(f\"\\nStacking OOF Score: {score:.4f}\")\n    \n    # 5. 后处理与提交\n    print(\"Generating submission...\")\n    df_sub = test_df.copy()\n    df_sub[CFG.target_cols] = final_test_pred\n    df_sub = post_process_biomass(df_sub)\n    \n    sub_rows = []\n    for i, row in df_sub.iterrows():\n        img_id = row['image_id']\n        for col in CFG.target_cols:\n            sub_rows.append({\n                'sample_id': f\"{img_id}__{col}\",\n                'target': row[col]\n            })\n            \n    submission = pd.DataFrame(sub_rows)\n    submission.to_csv('submission.csv', index=False)\n    print(\"Saved to submission.csv\")\n\nif __name__ == '__main__':\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-06T08:05:17.788276Z","iopub.execute_input":"2026-01-06T08:05:17.789120Z","iopub.status.idle":"2026-01-06T08:19:35.587180Z","shell.execute_reply.started":"2026-01-06T08:05:17.789081Z","shell.execute_reply":"2026-01-06T08:19:35.586191Z"}},"outputs":[],"execution_count":null}]}