{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14060333,"sourceType":"datasetVersion","datasetId":8883320},{"sourceId":14464889,"sourceType":"datasetVersion","datasetId":9011999}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --no-deps /kaggle/input/open-clip-dependencies-2/timm-1.0.22-py3-none-any.whl\n!pip install --no-deps /kaggle/input/open-clip-dependencies-2/protobuf-5.28.3-cp38-abi3-manylinux2014_x86_64.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-21T09:43:48.957181Z","iopub.execute_input":"2026-01-21T09:43:48.957441Z","iopub.status.idle":"2026-01-21T09:43:54.331961Z","shell.execute_reply.started":"2026-01-21T09:43:48.957411Z","shell.execute_reply":"2026-01-21T09:43:54.331125Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 📦\nimport os\nimport gc\nimport cv2\nimport h5py\nimport json\nimport time\nimport shutil\nimport socket\nimport psutil\nimport subprocess\nfrom pathlib import Path\nfrom datetime import datetime, timedelta\nfrom transformers import AutoImageProcessor, AutoModel\nimport math\nimport torch.nn.functional as F\nimport numpy as np\nimport pandas as pd\nimport random\n\nfrom tqdm import tqdm\nfrom tabulate import tabulate\n\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.metrics import r2_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.optim.lr_scheduler import _LRScheduler\nfrom torch.amp import autocast, GradScaler       \n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\nimport random\nfrom PIL import Image, ImageDraw, ImageFilter\n\nimport timm\nfrom torchvision import models, transforms\nfrom torchvision.models import get_model_weights\n\n\n# 🌱 Path Initialization\nif socket.gethostname() == 'hao-2':\n    dir = Path('D:/DATA_hao/Kaggle_/csiro-biomass/')\n    DIRS = {\n        \"dir\"  : dir,\n        \"train\": Path(dir, \"train\"),\n        \"test\" : Path(dir, \"test\"),\n        \"model\": Path(dir, \"DualStream_multihead\"),\n        \"data\" : Path(dir),\n    }\nelif socket.gethostname() == 'simon-MS-7D94':\n    dir = Path('/home/simon/simondisk2/simon/CSIRO/csiro-biomass')\n    DIRS = {\n        \"dir\"  : dir,\n        \"train\": Path(dir, \"train\"),\n        \"test\" : Path(dir, \"test\"),\n        \"model\": Path(\"/home/simon/simondisk2/simon/CSIRO/Train_75\", \"DualStream_multihead\"),\n        \"data\" : Path(dir),\n    }\nelif socket.gethostname() == 'user-PowerEdge-XE9680':\n    dir = Path('/data4/huangweigang/gh/csiro-biomass')\n    DIRS = {\n        \"dir\"  : dir,\n        \"train\": Path(dir, \"train\"),\n        \"test\" : Path(dir, \"test\"),\n        \"model\": Path(dir, \"DualStream_multihead\"),\n        \"data\" : Path(dir),\n    }\nelif socket.gethostname() == 'I24fece71c700101943':\n    dir = Path('/hy-tmp/CSIRO/data/csiro-biomass')\n    DIRS = {\n        \"dir\"  : dir,\n        \"train\": Path(dir, \"train\"),\n        \"test\" : Path(dir, \"test\"),\n        \"model\": Path(dir, \"biomasshead\"),\n        \"data\" : Path(dir),\n    }\nelse:\n    dir = Path('/kaggle/input/csiro-biomass')\n    DIRS = {\n        \"dir\"  : dir,\n        \"train\": Path(dir, \"train\"),\n        \"test\" : Path(dir, \"test\"),\n        \"model\": Path('/kaggle/input/single-1209-swa-models'),\n        \"data\" : Path(\"/kaggle/working/\"),\n    }","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-01-21T09:43:54.334050Z","iopub.execute_input":"2026-01-21T09:43:54.334313Z","iopub.status.idle":"2026-01-21T09:44:55.919429Z","shell.execute_reply.started":"2026-01-21T09:43:54.334283Z","shell.execute_reply":"2026-01-21T09:44:55.918859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_df_info(df, name: str):\n    \"\"\"\n    Print the shape and column names of a single DataFrame.\n\n    Args:\n        df   : pandas.DataFrame\n        name : Display name (string)\n    \"\"\"\n    print(f\"📊 {name:<16} shape: {str(df.shape):<16}  列名: {df.columns.tolist()}\")\n\n\ndef move_column_first(df, col_name):\n    if col_name not in df.columns:\n        raise ValueError(f\"列 '{col_name}' 不存在于 DataFrame 中。\")\n\n    cols = [col_name] + [c for c in df.columns if c != col_name]\n    return df[cols]\n\n\ndef select_free_gpu(threshold_mem_MB = 500, threshold_util = 20):\n    \"\"\"\n    Automatically select an available GPU (works in both .py and Jupyter environments).\n    \"\"\"\n    def get_gpu_info():\n        \"\"\"Retrieve GPU information using nvidia-smi.\"\"\"\n        query = (\n            \"index,name,memory.used,memory.total,utilization.gpu,temperature.gpu,power.draw\"\n        )\n        result = subprocess.run(\n            [\"nvidia-smi\", f\"--query-gpu={query}\", \"--format=csv,noheader,nounits\"],\n            capture_output=True, text=True\n        )\n\n        gpus = []\n        for line in result.stdout.strip().split(\"\\n\"):\n            idx, name, mem_used, mem_total, util, temp, power = [x.strip() for x in line.split(\",\")]\n            gpus.append({\n                \"index\": int(idx),\n                \"name\": name,\n                \"mem_used_MB\": int(mem_used),\n                \"mem_total_MB\": int(mem_total),\n                \"util_%\": int(util),\n                \"temp_C\": int(temp),\n                \"power_W\": float(power),\n            })\n        return gpus\n\n    gpus = get_gpu_info()\n\n    free_gpus = [\n        g for g in gpus\n        if g[\"mem_used_MB\"] < threshold_mem_MB and g[\"util_%\"] <= threshold_util\n    ]\n\n    if not free_gpus:\n        gpus.sort(key=lambda x: x[\"mem_used_MB\"])\n        selected = gpus[0]\n        reason = \"(No fully idle GPU found — selected the one with lowest memory usage)\"\n    else:\n        selected = free_gpus[0]\n        reason = \"(Idle GPU detected)\"\n\n    print(tabulate(\n        [[g[\"index\"], g[\"name\"], f\"{g['mem_used_MB']}/{g['mem_total_MB']} MB\",\n          f\"{g['util_%']}%\", f\"{g['temp_C']}°C\", f\"{g['power_W']}W\"]\n         for g in gpus],\n        headers=[\"GPU\", \"Name\", \"Memory\", \"Util\", \"Temp\", \"Power\"],\n        tablefmt=\"grid\"\n    ))\n\n    idx = selected[\"index\"]\n    device_name = f\"cuda:{idx}\"\n\n    try:\n        from IPython import get_ipython\n        in_notebook = get_ipython() is not None\n    except Exception:\n        in_notebook = False\n\n    if not in_notebook:\n        os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(idx)\n        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        print(f\"\\n✅ Selected GPU {idx} {reason}\")\n        print(f\"Current device: {device} (logical GPU {idx})\\n\")\n    else:\n        device = torch.device(device_name if torch.cuda.is_available() else \"cpu\")\n        print(f\"\\n⚠️ Detected Jupyter environment — not modifying CUDA_VISIBLE_DEVICES.\")\n        print(f\"✅ Using device: {device_name} {reason}\\n\")\n\n    return idx, device\n\n\ndef setup_seed(seed, deterministic=True):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = deterministic\n    torch.backends.cudnn.benchmark = False\n    \n# 打印清单\ndef config_to_str(config, indent = 0):\n    \"\"\"递归生成配置字符串\"\"\"\n    prefix = \"     \" * indent\n    lines = []\n    for key, value in config.items():\n        if isinstance(value, dict):\n            lines.append(f\"{prefix}🔹 {key}:\")\n            lines.append(config_to_str(value, indent + 1))  # 递归拼接子字典\n        else:\n            lines.append(f\"{prefix}- {key:<20}: {value}\")\n    return \"\\n\".join(lines)\n\n\n\n# Model \nclass MySingleStreamModel(nn.Module):\n    def __init__(self, \n                backbone_name=\"convnext_tiny\", \n                pretrained=True, \n                config = None):\n        \"\"\"\n        Args:\n            backbone_name: timm model name (e.g., convnext_tiny, resnet50)\n            pretrained: whether to load ImageNet pretrained weights\n            freeze_ratio: ratio of backbone layers to freeze (0~1)\n            weights_dict: per-target weights (dict) for WeightedSmoothL1Loss\n        \"\"\"\n        super().__init__()\n        print(\"Current backbone:\", backbone_name)\n\n        img_size = (config[\"img_size\"], config.get(\"img_width\", config[\"img_size\"]))\n        # 1️⃣ Backbone\n        if socket.gethostname() == \"user-PowerEdge-XE9680\":\n            model_path = f\"/data4/huangweigang/gh/timm_model/{backbone_name}.pth\"\n            self.backbone = timm.create_model(backbone_name, pretrained=False, num_classes=0)\n            try:\n                state_dict = torch.load(model_path, map_location='cpu', weights_only=True)\n            except TypeError:\n                state_dict = torch.load(model_path, map_location='cpu')\n            self.backbone.load_state_dict(state_dict, strict=False)\n        else:\n            self.backbone = timm.create_model(backbone_name, pretrained=pretrained, num_classes=0)\n\n\n\n        in_dim = self.backbone.num_features\n\n        params = list(self.backbone.parameters())\n        freeze_until = int(len(params) * config[\"freeze_ratio\"])\n        for i, p in enumerate(params):\n            p.requires_grad = i >= freeze_until     # freeze front part, train later part\n\n        self.feature_dim = in_dim\n        print(\"feature dim = \", self.feature_dim)\n\n        def make_head():\n            return nn.Sequential(\n                nn.Linear(self.feature_dim, 512),\n                nn.ReLU(),\n                nn.Dropout(0.3),\n                nn.Linear(512, 128),\n                nn.ReLU(),\n                nn.Linear(128, 1)\n            )\n\n        self.head_green  = make_head() \n        self.head_clover = make_head()  \n        self.head_dead   = make_head()   \n        self.head_gdm    = make_head()\n        self.head_total  = make_head()\n        self.softplus = nn.Softplus(beta=1.0)\n\n        self.weights = config[\"weights\"]\n\n    def forward(self, img):\n        # Extract features\n        feat = self.backbone(img)\n\n        # Predict three primary targets\n        G = self.softplus(self.head_green(feat))\n        C = self.softplus(self.head_clover(feat))\n        D = self.softplus(self.head_dead(feat))\n        GDM   = self.softplus(self.head_gdm(feat))\n        Total = self.softplus(self.head_total(feat))\n\n        preds = torch.cat([G, C, D, GDM, Total], dim=1)  # [B, 5]\n        return preds\n\n\n    def compute_loss(self, preds, targets):\n        \"\"\"\n        Loss aligned with Kaggle's official Global Weighted R² metric:\n        loss = ss_res / ss_tot\n        (smaller is better — same direction as the competition metric)\n        \"\"\"\n        preds = preds.view(-1, 5)\n        targets = targets.view(-1, 5)\n\n        weights = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5], device=preds.device)\n\n        y_true_flat = targets.view(-1)\n        y_pred_flat = preds.view(-1)\n\n        w_flat = torch.cat([\n            torch.full_like(targets[:, i], weights[i], device=preds.device)\n            for i in range(5)\n        ])\n        y_mean = torch.sum(w_flat * y_true_flat) / torch.sum(w_flat)\n\n        ss_res = torch.sum(w_flat * (y_true_flat - y_pred_flat) ** 2)\n        ss_tot = torch.sum(w_flat * (y_true_flat - y_mean) ** 2)\n\n        loss = ss_res / ss_tot\n\n        return loss\n\n\nclass SingleStreamDataset(Dataset):\n    def __init__(self, df, image_dir, config, pre_transform=None, transform=None):\n        \"\"\"\n        Args:\n            df: DataFrame containing an 'image_path' column.\n            image_dir: Directory where images are stored.\n            target_cols: Target columns (used for training datasets).\n            transform: Albumentations transform pipeline.\n        \"\"\"\n        self.df = df.reset_index(drop=True)\n        self.image_dir = image_dir\n        self.target_cols = config[\"target_cols\"]\n        self.pre_transform = pre_transform\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = Path(self.image_dir, str(row[\"image_path\"]))\n\n        if not img_path.exists():\n            print(f\"⚠️ Image not found: {img_path}\")\n            image = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        else:\n            try:\n                image = Image.open(img_path).convert(\"RGB\")\n            except Exception as e:\n                print(f\"⚠️ Failed to read image: {img_path} ({e})\")\n                image = np.zeros((1000, 2000, 3), dtype=np.uint8)\n    \n\n        image = np.array(image)\n        if self.pre_transform:\n            image = self.pre_transform(image=image)[\"image\"]\n            \n        if self.transform:\n            image = self.transform(image=image)[\"image\"]\n\n\n\n        if self.target_cols is not None:\n            targets = torch.tensor(\n                row[self.target_cols].astype(float).values,\n                dtype=torch.float32\n            )\n            return image, targets\n        else:\n            return image\n\n\n\ndef get_pre_transforms():\n    return A.Compose([\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.OneOf([\n            A.RandomBrightnessContrast(\n                brightness_limit=(0.8, 1.2), \n                contrast_limit=(0.8, 1.2),   \n                p=0.3\n            ),\n            A.ColorJitter(p=0.7)\n        ], p=0.4),\n    ])\n\n\ndef _get_resize_dims(config):\n    height = config[\"img_size\"]\n    width = int(config.get(\"img_width\", config[\"img_size\"]))\n    return height, width\n\n\ndef get_train_transforms(config):\n    height, width = _get_resize_dims(config)\n    return A.Compose([\n        A.Resize(height, width),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n\n\ndef get_valid_transforms(config):\n    height, width = _get_resize_dims(config)\n    return A.Compose([\n        A.Resize(height, width),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n\n\ndef get_tta_transforms(config):\n    height, width = _get_resize_dims(config)\n    return {\n        \"base\": A.Compose([\n            A.Resize(height, width),\n            A.Normalize(mean=[0.485, 0.456, 0.406],\n                        std=[0.229, 0.224, 0.225]),\n            ToTensorV2()\n        ]),\n        \"hflip\": A.Compose([\n            A.HorizontalFlip(p=1.0)\n        ]),\n        # \"vflip\": A.Compose([\n        #     A.VerticalFlip(p=1.0)\n        # ])\n    }\n\n\ndef compute_cv_score(all_preds, all_targets):\n    preds = np.concatenate(all_preds, axis=0)\n    targets = np.concatenate(all_targets, axis=0)\n\n    target_cols = [\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\", \"Dry_Total_g\"]\n    weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n  \n    y_true_flat = np.concatenate([targets[i, :] for i in range(targets.shape[0])])\n    y_pred_flat = np.concatenate([preds[i, :] for i in range(preds.shape[0])])\n    w_flat = np.concatenate([np.full_like(targets[:, i], weights[i]) for i in range(5)])\n\n    y_mean = np.sum(w_flat * y_true_flat) / np.sum(w_flat)\n\n    ss_res = np.sum(w_flat * (y_true_flat - y_pred_flat) ** 2)\n    ss_tot = np.sum(w_flat * (y_true_flat - y_mean) ** 2)\n\n    r2_global = 1 - ss_res / ss_tot\n    return r2_global\n\n\nclass WarmupCosineStepLR(_LRScheduler):\n    def __init__(\n        self,\n        optimizer,\n        total_steps=50,\n        n_constant=12,\n        n_cosine_1=12,\n        n_constant_2=13,\n        n_cosine_2=13,\n        min_lr=1e-6,\n        last_epoch=-1,\n    ):\n        self.total_steps = total_steps\n        self.n_constant = n_constant\n        self.n_cosine_1 = n_cosine_1\n        self.n_constant_2 = n_constant_2\n        self.n_cosine_2 = n_cosine_2\n        self.min_lr = min_lr\n        super().__init__(optimizer, last_epoch)\n\n    def get_lr(self):\n        step = self.last_epoch + 1\n        if step <= self.n_constant:                       # 1. 线性 warmup\n            return [base_lr * step / self.n_constant for base_lr in self.base_lrs]\n        elif step <= self.n_constant + self.n_cosine_1:   # 2. 第一次 cosine\n            s = step - self.n_constant\n            return [0.5 * bl + (bl - 0.5 * bl) * (1 + math.cos(math.pi * s / self.n_cosine_1)) / 2\n                    for bl in self.base_lrs]\n        elif step <= self.n_constant + self.n_cosine_1 + self.n_constant_2:  # 3. 常数\n            return [0.5 * bl for bl in self.base_lrs]\n        else:                                             # 4. 第二次 cosine\n            s = step - (self.n_constant + self.n_cosine_1 + self.n_constant_2)\n            return [self.min_lr + (0.5 * bl - self.min_lr) * (1 + math.cos(math.pi * s / self.n_cosine_2)) / 2\n                    for bl in self.base_lrs]\n\n\ndef mixup_batch(images, targets, alpha=0.4):\n    if alpha <= 0:\n        return images, targets, targets, 1.0\n\n    lam = np.random.beta(alpha, alpha)\n    indices = torch.randperm(images.size(0), device=images.device)\n    shuffled_images = images[indices]\n    shuffled_targets = targets[indices]\n\n    mixed_images = lam * images + (1 - lam) * shuffled_images\n    mixed_images = mixed_images.contiguous(memory_format=torch.channels_last)\n    return mixed_images, targets, shuffled_targets, lam\n\n\ndef cutmix_batch(images, targets, alpha=1.0):\n    if alpha <= 0:\n        return images, targets, targets, 1.0\n\n    lam = np.random.beta(alpha, alpha)\n    indices = torch.randperm(images.size(0), device=images.device)\n    shuffled_images = images[indices]\n    shuffled_targets = targets[indices]\n\n    _, _, H, W = images.size()\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    x1 = np.clip(cx - cut_w // 2, 0, W)\n    x2 = np.clip(cx + cut_w // 2, 0, W)\n    y1 = np.clip(cy - cut_h // 2, 0, H)\n    y2 = np.clip(cy + cut_h // 2, 0, H)\n\n    images[:, :, y1:y2, x1:x2] = shuffled_images[:, :, y1:y2, x1:x2]\n\n    lam_adjusted = 1 - ((x2 - x1) * (y2 - y1) / (W * H))\n    images = images.contiguous(memory_format=torch.channels_last)\n    return images, targets, shuffled_targets, lam_adjusted\n\n\ndef train_one_epoch(model, dataloader, optimizer, device, scaler, config):\n    model.train()\n    running_loss = []\n\n    start_epoch = time.time()\n    prev_end = start_epoch  \n\n    for step, (images, targets) in enumerate(dataloader):\n        t_load = time.time()  \n        data_load_time = t_load - prev_end\n\n        t0 = time.time()\n        images = images.to(device, non_blocking=True).contiguous(memory_format=torch.channels_last)\n        targets = targets.to(device, non_blocking=True)\n        t1 = time.time()\n\n        # ====== Forward + Backward ======\n        optimizer.zero_grad(set_to_none=True)\n\n        use_mixup = config[\"mixup\"] and np.random.rand() < 0.5\n        use_cutmix = (not use_mixup) and config[\"cutmix\"] and np.random.rand() < 0.5\n\n        if use_mixup:\n            mixed_images, targets_a, targets_b, lam = mixup_batch(\n                images, targets, config.get(\"mixup_alpha\", 0.4)\n            )\n            with torch.amp.autocast(device_type='cuda'):\n                preds = model(mixed_images)\n                loss = lam * model.compute_loss(preds, targets_a) + (1 - lam) * model.compute_loss(preds, targets_b)\n        elif use_cutmix:\n            mixed_images, targets_a, targets_b, lam = cutmix_batch(\n                images, targets, config.get(\"cutmix_alpha\", 1.0)\n            )\n            with torch.amp.autocast(device_type='cuda'):\n                preds = model(mixed_images)\n                loss = lam * model.compute_loss(preds, targets_a) + (1 - lam) * model.compute_loss(preds, targets_b)\n        else:\n            with torch.amp.autocast(device_type='cuda'):\n                preds = model(images)\n                loss = model.compute_loss(preds, targets)\n                \n        t2 = time.time()\n\n        scaler.scale(loss).backward()\n        scaler.unscale_(optimizer)\n        scaler.step(optimizer)\n        scaler.update()\n        t3 = time.time()\n\n        running_loss.append(loss.item())\n        prev_end = t3  \n        if step % 20  == 0:\n            print(\n                f\"[TRAIN] Step {step:4d} | \"\n                f\"data load: {data_load_time*1000:8.1f} ms | \"\n                f\"to(device): {(t1-t0)*1000:.1f} ms | \"\n                f\"forward+loss: {(t2-t1)*1000:7.1f} ms | \"\n                f\"backward+opt: {(t3-t2)*1000:.1f} ms | \"\n                f\"total: {(t3-t_load)*1000:7.1f} ms\"\n            )\n\n\n\n    end_epoch = time.time()\n    epoch_time = end_epoch - start_epoch\n    avg_batch_time = epoch_time / len(dataloader)\n\n    print(f\"[TRAIN] Epoch total time: {epoch_time:.2f}s | \"\n          f\"{len(dataloader)} batches | {avg_batch_time:.3f}s/batch\")\n    \n\n    return float(np.mean(running_loss))\n\n\ndef validate_one_epoch(model, dataloader, device):\n    model.eval()\n    val_losses, all_preds, all_targets = [], [], []\n\n    start_epoch = time.time()\n    prev_end = start_epoch \n\n    with torch.no_grad():\n        for step, (images, targets) in enumerate(dataloader):\n            t_load = time.time()\n            data_load_time = t_load - prev_end\n\n            t0 = time.time()\n            images = images.to(device, non_blocking=True).contiguous(memory_format=torch.channels_last)\n            targets = targets.to(device, non_blocking=True)\n            t1 = time.time()\n\n            preds = model(images)\n            val_loss = model.compute_loss(preds, targets).item()\n            t2 = time.time()\n\n            val_losses.append(val_loss)\n            all_preds.append(preds.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n\n            prev_end = t2 \n\n            if step % 6  == 0:\n                print(\n                    f\"[VAL  ] Step {step:4d} | \"\n                    f\"data load: {data_load_time*1000:8.1f} ms | \"\n                    f\"to(device): {(t1 - t0)*1000:.1f} ms | \"\n                    f\"forward+loss: {(t2 - t1)*1000:7.1f} ms | \"\n                    f\"total: {(t2 - t_load)*1000:7.1f} ms\"\n                )\n            \n\n    end_epoch = time.time()\n    epoch_time = end_epoch - start_epoch\n    avg_val_loss = float(np.mean(val_losses))\n    r2_global = compute_cv_score(all_preds, all_targets)\n\n    print(\n        f\"[VAL  ] Epoch total time: {epoch_time:.2f}s | \"\n        f\"{len(dataloader)} batches | {epoch_time / len(dataloader):.3f}s/batch\"\n    )\n\n    return avg_val_loss, r2_global\n\n\ndef get_fold_loaders(df, fold, groups, config, device):\n    \"\"\"为单个 fold 构建训练和验证 DataLoader（各自独立 transform）\"\"\"\n    train_df = df[df[\"fold\"] != fold].reset_index(drop=True)\n    valid_df = df[df[\"fold\"] == fold].reset_index(drop=True)\n\n    train_dataset = SingleStreamDataset(\n        train_df,\n        DIRS[\"dir\"], \n        config, \n        pre_transform=get_pre_transforms(), \n        transform=get_train_transforms(config)\n    )\n    valid_dataset = SingleStreamDataset(\n        valid_df, \n        DIRS[\"dir\"], \n        config, \n        pre_transform=None, \n        transform=get_valid_transforms(config)\n    )\n\n    num_workers = config[\"num_workers\"]\n    prefetch_factor = config[\"prefetch_factor\"]\n\n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=config[\"batch_size\"],\n        shuffle=True,\n        num_workers=num_workers,           # ✅ 启用多核加载\n        pin_memory=True,                   # ✅ 加速 CPU→GPU 拷贝\n        prefetch_factor=prefetch_factor,   # ✅ 每个 worker 预加载 3 个 batch\n        persistent_workers=True            # ✅ 保持 worker 常驻\n    )\n    valid_loader = DataLoader(\n        valid_dataset,\n        batch_size=config[\"batch_size\"],\n        shuffle=False,\n        num_workers=max(1, num_workers // 2),\n        pin_memory=True,\n        prefetch_factor=prefetch_factor,\n        persistent_workers=True\n    )\n\n    return train_loader, valid_loader\n\n\ndef should_stop(cv_scores, window=20, threshold=0.001):\n    \"\"\"Check if recent CV score fluctuations are stable enough to trigger early stopping.\"\"\"\n\n    if len(cv_scores) < window :  \n        return False\n    window_scores = cv_scores[-window:]\n    diff = max(window_scores) - min(window_scores)\n    if diff < threshold:\n        print(\"diff =\", diff, f\"< {threshold}\")\n    return diff < threshold\n\n\ndef fetch_scheduler(optimizer, config):\n    if config[\"scheduler_type\"] == \"warmup_cosine\":\n        print(\"use custom warmup_cosine scheduler!!\")\n        scheduler = WarmupCosineStepLR(optimizer,\n                                       total_steps=int(config[\"epochs\"]),  # 总共训练的epoch数\n                                       n_constant=int(config[\"n_constant\"]),  # warmup阶段的epoch数\n                                       n_cosine_1=int(config[\"n_cosine_1\"]),  # 第一个Cosine退火阶段的epoch数\n                                       n_constant_2=int(config[\"n_constant_2\"]),  # Constant阶段的epoch数\n                                       n_cosine_2=int(config[\"n_cosine_2\"]),  # 第二个Cosine退火阶段的epoch数\n                                       min_lr=float(config[\"min_lr\"]),  # 最小学习率\n                                       )\n    else:\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=config[\"epochs\"], eta_min=config[\"lr\"]/100)\n\n    return scheduler\n\n\ndef train_one_fold(fold, df, groups, save_dir, config, device):\n    \"\"\"Train a single fold and return performance metrics for that fold.\"\"\"\n    if config[\"mixup\"]:\n        print(\"use mixup!!!\")\n        \n    train_loader, valid_loader = get_fold_loaders(df, fold, groups, config, device)\n\n    model = MySingleStreamModel(config[\"backbone_name\"], pretrained=True, config=config)\n    model = model.to(device).to(memory_format=torch.channels_last)\n\n    optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=config[\"lr\"], weight_decay=2e-5)\n    scheduler = fetch_scheduler(optimizer, config)\n    scaler = torch.amp.GradScaler(device=\"cuda\")\n\n    train_losses, val_losses, cv_scores, LR_records = [], [], [], []\n    epoch_times = [] \n    all_progress = config[\"epochs\"] * config[\"n_splits\"]\n\n    top_cv_models = []  # (cv_score, epoch, model_path)\n    top_loss_models = []  # (loss, epoch, model_path)\n    \n    best_cv = -float(\"inf\") \n    best_loss = 100\n\n    for epoch in range(config[\"epochs\"]):\n        epoch_start = time.time()\n\n        train_loss = train_one_epoch(model, train_loader, optimizer, device, scaler, config)\n        val_loss, cv = validate_one_epoch(model, valid_loader, device)\n        scheduler.step()\n\n        train_losses.append(train_loss)\n        val_losses.append(val_loss)\n        cv_scores.append(cv)\n        LR_records.append(scheduler.get_last_lr()[0])\n\n        epoch_time = time.time() - epoch_start\n        if epoch > 0:\n            epoch_times.append(epoch_time)\n            if len(epoch_times) > 10:\n                epoch_times.pop(0)\n        if len(epoch_times) > 0:\n            avg_epoch_time = np.mean(epoch_times)\n        else:\n            avg_epoch_time = epoch_time\n\n        progress = fold * config[\"epochs\"] + (epoch + 1)\n        remaining_epochs = all_progress - progress\n\n        eta_seconds = avg_epoch_time * remaining_epochs\n        if not np.isnan(eta_seconds) and not np.isinf(eta_seconds):\n            eta_time = datetime.now() + timedelta(seconds=float(eta_seconds))\n            eta_time = eta_time.replace(microsecond=0)\n            days_diff = (eta_time.date() - datetime.now().date()).days\n            eta_str = (\n                f\"T+{days_diff} \" + eta_time.strftime(\"%H:%M:%S\")\n                if days_diff > 0 else eta_time.strftime(\"%H:%M:%S\")\n            )\n        else:\n            eta_str = \"--:--:--\"\n\n        now_str = datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n        print(\n            f\"[{now_str}]🧩[{progress/all_progress*100:6.2f}%] \"\n            f\"Fold {fold+1}/{config['n_splits']} | \"\n            f\"Epoch {epoch+1:03d}/{config['epochs']} | \"\n            f\"Train={train_loss:.4f} | \"\n            f\"Val={val_loss:.4f} | \"\n            f\"CV={cv:.4f} | \"\n            f\"LR={scheduler.get_last_lr()[0]:.6f} | \"\n            f\"{epoch_time:6.2f}s/it | \"\n            f\"ETA≈{eta_str}\"\n        )\n\n        min_epoch = config[\"min_epoch\"] \n\n        if epoch >= min_epoch:\n            should_save_cv = len(top_cv_models) < 3 or cv > min(top_cv_models, key=lambda x: x[0])[0]\n            if should_save_cv:\n                cv_model_path = save_dir / f\"model_cv_epoch{epoch+1}_fold{fold+1}.pt\"\n                torch.save(model.state_dict(), cv_model_path)\n                \n                top_cv_models.append((cv, epoch + 1, str(cv_model_path)))\n                top_cv_models.sort(key=lambda x: x[0], reverse=True)\n                if len(top_cv_models) > 3:\n                    removed_cv = top_cv_models.pop()\n                    try:\n                        os.remove(removed_cv[2])\n                        print(f\"🗑️ 删除CV  模型: {os.path.basename(removed_cv[2])}\")\n                    except:\n                        pass\n                print(f\"🌟 更新CV   Top{len(top_cv_models)}模型！Fold {fold+1} | Epoch {epoch+1} | CV  ={cv:.4f}\")\n\n        should_save_loss = len(top_loss_models) < 3 or val_loss < max(top_loss_models, key=lambda x: x[0])[0]\n        if should_save_loss:\n            loss_model_path = save_dir / f\"model_loss_epoch{epoch+1}_fold{fold+1}.pt\"\n            torch.save(model.state_dict(), loss_model_path)\n            top_loss_models.append((val_loss, epoch + 1, str(loss_model_path)))\n            top_loss_models.sort(key=lambda x: x[0])\n            if len(top_loss_models) > 3:\n                removed_loss = top_loss_models.pop()\n                try:\n                    os.remove(removed_loss[2])\n                    print(f\"🗑️ 删除Loss模型: {os.path.basename(removed_loss[2])}\")\n                except:\n                    pass\n            \n            print(f\"🌟 更新Loss Top{len(top_loss_models)}模型！Fold {fold+1} | Epoch {epoch+1} | Loss={val_loss:.4f}\")\n\n    \n        top_models_info = {\n            \"fold\": fold,\n            \"cv_top3\": [\n                {\"rank\": i+1, \"cv\": float(score), \"epoch\": ep, \"path\": path}\n                for i, (score, ep, path) in enumerate(top_cv_models)\n            ],\n            \"loss_top3\": [\n                {\"rank\": i+1, \"loss\": float(score), \"epoch\": ep, \"path\": path}\n                for i, (score, ep, path) in enumerate(top_loss_models)\n            ]\n        }\n\n        with open(save_dir / f\"model_top3_fold{fold+1}.json\", \"w\") as f:\n            json.dump(top_models_info, f, indent=4)\n\n        if should_stop(cv_scores, threshold=config[\"cv_stability_stop_threshold\"]):\n            print(f\"🛑 Early Stop at Epoch {epoch+1}\")\n            break\n\n    del train_loader, valid_loader, model, optimizer, scheduler, scaler\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\n    return train_losses, val_losses, cv_scores, LR_records\n\n\ndef save_metrics_to_excel(all_metrics, save_dir):\n    \"\"\"Combine metrics from all folds and save them into an Excel file.\"\"\"\n    max_epochs = max(len(m[0]) for m in all_metrics)\n    df_out = pd.DataFrame({\"Epoch\": range(1, max_epochs + 1)})\n\n    for i, (train, val, cv, lr) in enumerate(all_metrics, 1):\n        df_out[f\"Train_Fold{i}\"] = train + [None] * (max_epochs - len(train))\n        df_out[f\"Val_Fold{i}\"]   = val   + [None] * (max_epochs - len(val))\n        df_out[f\"CV_Fold{i}\"]    = cv    + [None] * (max_epochs - len(cv))\n        df_out[f\"LR_Fold{i}\"]    = lr    + [None] * (max_epochs - len(lr))\n\n    out_path = Path(save_dir) / \"fold_metrics.xlsx\"\n    df_out.to_excel(out_path, index=False)\n    print(f\"✅ Training log saved to: {out_path}\")\n\n\ndef load_and_prepare_train_df():\n    df_file_path = Path(DIRS[\"dir\"]) / \"train_with_folds_singleflow.csv\"\n    df = pd.read_csv(df_file_path)\n\n    return df\n\n\n\ndef allocate_gpu_memory(memory_size_in_gb, device=None):\n      \n    # 计算张量的大小，假设每个元素是 float32（4 字节）\n    tensor_size = memory_size_in_gb * 1024 * 1024 * 1024 // 4  # 每个 float32 元素占用 4 字节\n\n    # 在 GPU 上创建一个全是零的大张量\n    tensor = torch.zeros(tensor_size, dtype=torch.float32, device=device)\n\n    # 打印当前 GPU 内存使用情况\n    if device.type == 'cuda':\n        print(f\"已分配 {memory_size_in_gb}GB 的内存，当前 GPU 内存使用情况：\")\n        print(torch.cuda.memory_allocated(device) / 1024 / 1024 / 1024, \"GB\")\n    \n    return tensor\n\n\n\ndef train_with_groupkfold(df_train, save_dir, config, device):\n    \n\n    df = df_train.copy()\n    \n    \n    if socket.gethostname() in [\"user-PowerEdge-XE9680\"]:\n        tensor = allocate_gpu_memory(65, device)\n\n\n    unique_groups = np.random.permutation(df[\"Sampling_Date\"].unique())\n    group_map = {g: i for i, g in enumerate(unique_groups)}\n    df[\"GroupID\"] = df[\"Sampling_Date\"].map(group_map)\n\n    all_metrics = []\n    for fold in range(config[\"n_splits\"]):\n        metrics = train_one_fold(fold, df, df[\"GroupID\"], save_dir, config, device)\n        all_metrics.append(metrics)\n        gc.collect()\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n\n    save_metrics_to_excel(all_metrics, save_dir)\n    print(\"🎯 All folds training completed.\")\n\n\nconfig = {\n    # ⚙️ basic train params\n    \"seed\"                           : 43,  \n    \"epochs\"                         : 200,\n    \"freeze_ratio\"                   : 0.8,\n    # \"freeze_start_layer\"             : 9, \n    \"num_workers\"                    : 4,\n    \"prefetch_factor\"                : 3,\n\n    \"batch_size\"                     : 4,\n    \n    \"lr\"                             : 1e-4,\n    \"n_splits\"                       : 5,\n    \"img_size\"                       : 1024,\n    \"img_ratio\"                      : 2.0,  # width = img_size * img_ratio\n    \"backbone_name\"                  : \"vit_large_patch16_dinov3_qkvb.lvd1689m\", #\"vit_base_patch16_dinov3.lvd1689m\",\n    \"cv_stability_stop_threshold\"    : 0.005,\n    \"min_epoch\"                      : 0,\n    \n    \"mixup\"                          : False,\n    \"cutmix\"                         : False,\n    \"mixup_alpha\"                    : 0.4,\n    \"cutmix_alpha\"                   : 1.0,\n    \n    \"scheduler_type\"                 : \"cosine\",\n    \"n_constant\"                     : 5,\n    \"n_cosine_1\"                     : 25,\n    \"n_constant_2\"                   : 5,\n    \"n_cosine_2\"                     : 80,\n    \"min_lr\"                         : 5e-6,\n    \n    \"weights\": {\n        \"Dry_Green_g\" : 0.1,\n        \"Dry_Clover_g\": 0.1,\n        \"Dry_Dead_g\"  : 0.1,\n        \"GDM_g\"       : 0.2,\n        \"Dry_Total_g\" : 0.5\n    },\n\n\n    \"target_cols\": [\n        \"Dry_Green_g\",\n        \"Dry_Clover_g\",\n        \"Dry_Dead_g\",\n        \"GDM_g\",\n        \"Dry_Total_g\"\n    ]\n}\n\nconfig[\"img_width\"] = int(config[\"img_size\"] * config.get(\"img_ratio\", 1.0))\n\n\nisTRAIN     = True\nisPREDICT   = True\nDEBUG       = False\n\n\nif (\n    __name__ == \"__main__\"\n    and isTRAIN\n    and socket.gethostname() in [\"user-PowerEdge-XE9680\", \"hao-2\", \"I24fece71c700101943\",\"simon-MS-7D94\"]\n):\n    torch.multiprocessing.freeze_support()       \n    setup_seed(config[\"seed\"])        \n    idx, device = select_free_gpu()\n\n    time_str = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n    model_root = DIRS[\"model\"]\n    history_DIR = Path(model_root, time_str)\n\n    os.makedirs(model_root, exist_ok=True)\n    os.makedirs(history_DIR, exist_ok=True)\n\n    if DEBUG:\n        config[\"epochs\"] = 1\n        print(f\"⚠️ DEBUG mode enabled — running only {config['epochs']} epochs\\n\")\n\n    config[\"time_str\"] = time_str\n    config_path = history_DIR / \"config.json\"\n    with open(config_path, \"w\", encoding=\"utf-8\") as f:\n        json.dump(config, f, indent=4, ensure_ascii=False)\n    print(f\"✅ Config saved to: {config_path}\")\n    print(\"\\n\")\n    print(config_to_str(config))\n    print(\"\\n\")\n\n    try:\n        source_path = Path(__file__).resolve()\n        target_path = history_DIR / source_path.name\n        shutil.copy2(source_path, target_path)\n        print(f\"✅ Source code {source_path.name} copied to: {target_path}\")\n    except Exception as e:\n        print(f\"⚠️ Failed to copy source code: {e}\")\n\n    print(\"\\n🚀🚀🚀 Starting training process... 🚀🚀🚀\\n\")\n\n    df_train = load_and_prepare_train_df()\n    if DEBUG:\n        df_train = df_train.iloc[:7].copy()\n        print(f\"⚠️ DEBUG mode enabled — Using first {len(df_train)} training data \")\n    \n    train_with_groupkfold(\n        df_train             = df_train,\n        save_dir             = history_DIR,\n        config               = config,\n        device               = device\n    )\n\n    print(f\"\\n✅ Training completed! Results saved in: {history_DIR}\")\n    print(\"✅\"*65)\n\n\n\n# Infer\ndef load_and_prepare_test_df():\n    df_file_path = Path(DIRS[\"dir\"]) / \"test.csv\"\n    df = pd.read_csv(df_file_path)\n\n    df[\"ID\"] = df[\"sample_id\"].str.split(\"__\").str[0]\n    df = move_column_first(df, \"ID\")\n    df[\"target\"] = 0  \n\n    df_targets = (\n        df.pivot_table(index=\"ID\", columns=\"target_name\", values=\"target\", aggfunc=\"first\")\n        .reset_index()\n    )\n    df_targets.columns.name = None\n\n    df_meta = df[[\"ID\", \"image_path\"]].drop_duplicates(subset=\"ID\")\n    df_test = pd.merge(df_meta, df_targets, on=\"ID\", how=\"left\")\n    show_df_info(df_test, \"df_test\")\n\n    print(f\"✅ Test set loaded: {df_test.shape}\")\n    return df_test\n\n\ndef load_model_for_inference(model_path, device, config):\n    model = MySingleStreamModel(config[\"backbone_name\"], pretrained=False, config=config) # , pretrained=False\n    model = model.to(device).to(memory_format=torch.channels_last)\n\n    try:\n        state_dict = torch.load(model_path, map_location=device, weights_only=True)\n    except TypeError:\n        state_dict = torch.load(model_path, map_location=device)\n\n    model.load_state_dict(state_dict, strict=False)\n    model.eval()\n    return model\n\n\ndef predict_with_model(model, dataloader, device):\n    preds_list = []\n    model.eval()\n\n    with torch.no_grad():\n        for step, (images, targets) in enumerate(dataloader):\n            images = images.to(device, non_blocking=True).contiguous(memory_format=torch.channels_last)\n            targets = targets.to(device, non_blocking=True)\n\n            with torch.amp.autocast(device_type='cuda'):\n                preds = model(images)\n            \n            preds_list.append(preds.cpu().numpy())\n    return np.concatenate(preds_list, axis=0)\n\n\ndef predict_ensemble(df_test, pre_transform, transform, model_dir, device, config):\n    model_paths = sorted(Path(model_dir).glob(\"*.pt\"))\n    assert len(model_paths) > 0, f\"❌ No model files found in: {model_dir}\"\n\n    print(f\"✅ Detected {len(model_paths)} 个模型:\")\n    for p in model_paths:\n        print(\"   -\", p.name)\n\n    test_dataset = SingleStreamDataset(\n        df_test,\n        DIRS[\"dir\"],\n        config,\n        pre_transform=pre_transform,\n        transform=transform,\n    )\n\n    test_loader = DataLoader(\n        test_dataset,\n        batch_size=config[\"batch_size\"],\n        shuffle=False,\n        num_workers=4,\n        pin_memory=True,\n    )\n\n    fold_preds = []\n    for fold, model_path in enumerate(model_paths):\n        print(f\"Fold {fold+1}/{len(model_paths)} 推理: {model_path.name}\")\n        model = load_model_for_inference(model_path, device, config)\n        fold_pred = predict_with_model(model, test_loader, device)\n        fold_preds.append(fold_pred)\n\n    preds_mean = np.mean(fold_preds, axis=0)\n    df_pred = pd.DataFrame(preds_mean, columns=config[\"target_cols\"])\n    \n    df_pred[\"ID\"] = df_test[\"ID\"]\n    df_pred = df_pred[[\"ID\"] + config[\"target_cols\"]]\n    show_df_info(df_pred, \"df_pred\")\n\n    return df_pred\n\n\ndef generate_submission(df_pred_final):\n    print(\"\\n\\n\\nGenerating Kaggle submission file ------------------------------------\")\n\n    print(\"📦 [Step 1] Original prediction DataFrame info:\")\n    print(df_pred_final.head(5))\n    print(f\"🔹 Data shape: {df_pred_final.shape}\")\n\n    ordered_target_cols = [\n        \"Dry_Green_g\",  # 1️⃣\n        \"Dry_Clover_g\", # 2️⃣\n        \"Dry_Dead_g\",   # 3️⃣\n        \"GDM_g\",        # 4️⃣\n        \"Dry_Total_g\"   # 5️⃣\n    ]\n    print(f\"\\n📋 [Step 2] Target column order: {ordered_target_cols}\")\n\n\n    print(\"\\n🧮 [Step 2.2] Scaling Dry_Clover_g by factor 0.8\")\n    if \"Dry_Clover_g\" in df_pred_final.columns:\n        df_pred_final[\"Dry_Clover_g\"] = df_pred_final[\"Dry_Clover_g\"] * 0.8\n    else:\n        print(\"   ⚠️ Warning: Column Dry_Clover_g not found in DataFrame\")\n\n\n\n    \n    print(\"\\n🔒 [Step 2.5] Clipping negative values to 0\")\n    for col in ordered_target_cols:\n        if col in df_pred_final.columns:\n            negative_count = (df_pred_final[col] < 0).sum()\n            if negative_count > 0:\n                print(f\"   - {col}: {negative_count} negative values clipped to 0\")\n            df_pred_final[col] = df_pred_final[col].clip(lower=0)\n        else:\n            print(f\"   ⚠️ Warning: Column {col} not found in DataFrame\")\n\n    print(\"\\n📊 [Step 3] Preview after clipping:\")\n    print(df_pred_final[ordered_target_cols].head(5))\n\n    df_submit = (\n        df_pred_final.melt(\n            id_vars=\"ID\",\n            value_vars=ordered_target_cols,\n            var_name=\"target_name\",\n            value_name=\"target\"\n        )\n    )\n    print(\"\\n📊 [Step 4] Preview of DataFrame after melt:\")\n    print(df_submit.head(10))\n    print(f\"🔹 Shape after melt: {df_submit.shape}\")\n\n    df_submit[\"sample_id\"] = df_submit[\"ID\"] + \"__\" + df_submit[\"target_name\"]\n\n    print(\"\\n🔧 [Step 5] Preview after adding sample_id column:\")\n    print(df_submit[[\"ID\", \"target_name\", \"sample_id\"]].head(10))\n\n    df_submit = df_submit[[\"sample_id\", \"target\"]]\n    df_submit = df_submit.sort_values(\"sample_id\").reset_index(drop=True)\n    print(\"\\n📋 [Step 6] Preview after reordering & sorting by sample_id:\")\n    print(df_submit.head(10))\n    print(f\"🔹 Final output shape: {df_submit.shape}\")\n\n    negative_in_final = (df_submit[\"target\"] < 0).sum()\n    if negative_in_final > 0:\n        print(f\"⚠️  Warning: Still found {negative_in_final} negative values in final submission!\")\n    else:\n        print(\"✅ All values are non-negative in final submission\")\n\n    df_submit.to_csv(\"submission.csv\", index=False)\n    print(\"\\n✅ Submission file generated: submission.csv\")\n\n\ndef run_tta_prediction(df_test, model_dir, device, config):\n    tta_transforms = get_tta_transforms(config)\n\n    tta_names = list(tta_transforms.keys())\n    print(f\"\\n✅ Detected {len(tta_names)} TTA modes: {tta_names}\\n\")\n\n    all_preds = []\n    for name, _ in tta_transforms.items():\n        print(f\"\\n🚀 TTA mode: {name}\")\n        if name == \"base\":\n            print(\"base no pre_transform\")\n            pre_transform = None\n            transform = tta_transforms[name]\n        else:\n            pre_transform = tta_transforms[name]\n            transform = tta_transforms[\"base\"]\n        df_pred = predict_ensemble(df_test, pre_transform, transform, model_dir, device, config)\n        all_preds.append(df_pred[config[\"target_cols\"]].values)\n\n        # ✅ Show intermediate results\n        print(f\"Preview of predictions for TTA mode [{name}]:\")\n        print(df_pred.head())\n\n        print(f\"Number of collected TTA results: {len(all_preds)}\")\n        print(f\"Current cumulative shape: {np.array(all_preds).shape}\")\n        print(\"\\n\\n\")\n\n    \n    print(\"\\nAggregating all TTA predictions:\")\n    print(f\"Total {len(all_preds)} sets of predictions collected.\")\n    for i, arr in enumerate(all_preds):\n        print(f\"  └─ Prediction set {i+1}: \\n{arr}\")\n\n    mean_preds = np.mean(all_preds, axis=0)\n    print(f\"Mean computation completed. Shape: {mean_preds.shape}, values:\")\n    print(mean_preds)\n\n    df_pred_final = df_pred.copy()\n    df_pred_final[config[\"target_cols\"]] = mean_preds\n\n    print(\"\\n✅ TTA aggregation completed. Preview of final predictions (df_pred_final):\")\n    print(df_pred_final.head())\n\n    return df_pred_final\n\n\nif __name__ == \"__main__\" and isPREDICT:\n    print(\"\\n🧠 Starting prediction pipeline...\")\n    idx, device = select_free_gpu()\n    print(idx,device)\n\n    if socket.gethostname() in [\"user-PowerEdge-XE9680\", \"hao-2\",\"simon-MS-7D94\"]:\n        if DEBUG:\n            df_test = load_and_prepare_train_df().iloc[:7]\n        else:\n            df_test = load_and_prepare_train_df()\n    else:\n        df_test = load_and_prepare_test_df()\n\n    print(f\"\\nCurrent dataset shape: {df_test.shape}\\n\")\n\n    if socket.gethostname() == \"hao-2\":\n        model_dir = Path(DIRS[\"model\"], \"2025-11-06 17-05-25\")\n        if isTRAIN:\n            model_dir = Path(DIRS[\"model\"], time_str)\n    elif socket.gethostname() == \"user-PowerEdge-XE9680\":\n        model_dir = Path(\n            \"/data4/huangweigang/gh/csiro-biomass/Model_History\",\n            \"dualstream-multihead_v14_A100_v2_maxepoch\"\n        )\n        if isTRAIN:\n            model_dir = Path(DIRS[\"model\"], time_str)\n    \n    elif socket.gethostname() == \"simon-MS-7D94\":\n        model_dir = Path(\"/home/simon/simondisk2/simon/CSIRO/Train_base_0/DualStream_multihead\", \"2025-11-12 17-32-38\")\n        if isTRAIN:\n            model_dir = Path(DIRS[\"model\"], time_str)\n    else:\n        model_dir = DIRS[\"model\"]\n\n    print(f\"Model directory loaded: {model_dir}\")\n\n    df_pred_final = run_tta_prediction(df_test, model_dir, device, config)\n\n    if socket.gethostname() in [\"user-PowerEdge-XE9680\", \"hao-2\",\"simon-MS-7D94\"]:\n        out_xlsx = Path(model_dir) / \"df_pred_final.xlsx\"\n        df_pred_final.to_excel(out_xlsx, index=False)\n        print(f\"✅ Prediction results saved to: {out_xlsx}\")\n\n    generate_submission(df_pred_final)\n    print(\"🎯 Prediction pipeline completed.\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-01-21T09:45:04.390514Z","iopub.execute_input":"2026-01-21T09:45:04.391217Z","iopub.status.idle":"2026-01-21T09:47:00.697621Z","shell.execute_reply.started":"2026-01-21T09:45:04.391184Z","shell.execute_reply":"2026-01-21T09:47:00.696841Z"}},"outputs":[],"execution_count":null}]}