{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# DINOv3を使うためにtimmを最新版にアップデート\n!pip install timm -U --no-deps\n\n# 確認\nimport timm\nprint(f\"Timm version: {timm.__version__}\")  # 1.0.22以上が必要","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-05T00:29:41.917425Z","iopub.execute_input":"2026-01-05T00:29:41.917639Z","iopub.status.idle":"2026-01-05T00:29:55.559925Z","shell.execute_reply.started":"2026-01-05T00:29:41.917617Z","shell.execute_reply":"2026-01-05T00:29:55.559258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nexp04: Improved Biomass Regression (based on exp01 structure)\n=============================================================\nBase: exp01 (left-right split, DINOv3)\nImprovements:\n1. Dead loss weight: 1.5 → 2.5\n2. Zero-classification auxiliary head\n3. Enhanced augmentation for NSW conditions\n\"\"\"\n\nimport os\nimport sys\nimport random\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom PIL import Image\nfrom datetime import datetime\nfrom dataclasses import dataclass, field\nfrom typing import Tuple, List, Dict\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nfrom sklearn.model_selection import StratifiedKFold\nimport timm\nfrom torchvision import transforms\nfrom tqdm import tqdm\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n# ============================================\n# Configuration (based on exp01 + improvements)\n# ============================================\n@dataclass\nclass Config:\n    # Paths (Kaggle)\n    data_root: str = \"/kaggle/input/csiro-biomass\"\n    train_path: str = \"/kaggle/input/csiro-biomass/train.csv\"\n    output_dir: str = \"/kaggle/working\"\n    \n    # Model - FIXED: Use same img_size as exp01\n    model_name: str = \"vit_base_patch16_dinov3.lvd1689m\"\n    img_size: int = 256  # FIXED: Was 512, exp01 uses 256\n    \n    # Training - FIXED: Match exp01 settings\n    n_folds: int = 5\n    epochs: int = 40  # FIXED: Was 25, exp01 uses 40\n    batch_size: int = 8  # FIXED: Was 16, exp01 uses 8\n    num_workers: int = 4\n    \n    # Optimizer - FIXED: Match exp01 LR\n    learning_rate: float = 5e-4  # FIXED: Was 1e-4, exp01 uses 5e-4\n    weight_decay: float = 0.01\n    \n    # Scheduler\n    warmup_epochs: int = 3\n    \n    # Image preprocessing (from exp01)\n    crop_bottom_ratio: float = 0.1\n    remove_timestamp: bool = True\n    \n    # IMPROVED: Loss weights - Dead increased from 1.5 to 2.0 (more conservative)\n    loss_weights: List[float] = field(default_factory=lambda: [1.0, 2.0, 1.0])  # [Green, Dead, Clover]\n    \n    # Zero classification settings - DISABLED for testing\n    use_zero_cls: bool = True  # Disabled to test regression-only first\n    zero_cls_weight: float = 0.1  # Reduced from 0.3\n    \n    # Other\n    seed: int = 42\n    device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    use_amp: bool = torch.cuda.is_available()  # Only use AMP if CUDA is available\n    gradient_clip: float = 1.0\n    dropout: float = 0.2\n\n\nconfig = Config()\n\n\n# ============================================\n# Model Availability Check\n# ============================================\ndef get_best_available_model():\n    models = [\n        (\"vit_base_patch16_dinov3.lvd1689m\", 512),\n        (\"vit_base_patch14_dinov2.lvd142m\", 518),\n        (\"convnext_base.fb_in22k_ft_in1k\", 224),\n    ]\n    for name, size in models:\n        try:\n            _ = timm.create_model(name, pretrained=False, num_classes=0)\n            print(f\"✅ Model: {name}\")\n            return name, size\n        except:\n            continue\n    raise RuntimeError(\"No model found\")\n\n\ndef set_seed(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\n# ============================================\n# Image Preprocessing (from exp01)\n# ============================================\ndef clean_image(img_array):\n    \"\"\"Clean image by removing artifacts\"\"\"\n    h, w = img_array.shape[:2]\n    \n    if config.crop_bottom_ratio > 0:\n        new_h = int(h * (1 - config.crop_bottom_ratio))\n        img_array = img_array[:new_h, :]\n    \n    if config.remove_timestamp:\n        hsv = cv2.cvtColor(img_array, cv2.COLOR_RGB2HSV)\n        lower_orange = np.array([5, 150, 150])\n        upper_orange = np.array([25, 255, 255])\n        mask = cv2.inRange(hsv, lower_orange, upper_orange)\n        mask = cv2.dilate(mask, np.ones((3, 3), np.uint8), iterations=2)\n        \n        if np.sum(mask) > 0:\n            img_bgr = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)\n            img_bgr = cv2.inpaint(img_bgr, mask, 3, cv2.INPAINT_TELEA)\n            img_array = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n    \n    return img_array\n\n\n# ============================================\n# Dataset (from exp01 with zero labels)\n# ============================================\nclass BiomassDataset(Dataset):\n    def __init__(self, df, root_dir, transform=None, is_train=True):\n        self.df = df.reset_index(drop=True)\n        self.root_dir = root_dir\n        self.transform = transform\n        self.is_train = is_train\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        img_path = os.path.join(self.root_dir, row[\"image_path\"])\n        img = Image.open(img_path).convert(\"RGB\")\n        img_array = np.array(img)\n        \n        img_array = clean_image(img_array)\n        \n        # Left-right split (from exp01)\n        h, w = img_array.shape[:2]\n        mid = w // 2\n        img_left = Image.fromarray(img_array[:, :mid])\n        img_right = Image.fromarray(img_array[:, mid:])\n        \n        if self.transform:\n            seed = np.random.randint(2147483647)\n            \n            random.seed(seed)\n            torch.manual_seed(seed)\n            img_left = self.transform(img_left)\n            \n            random.seed(seed)\n            torch.manual_seed(seed)\n            img_right = self.transform(img_right)\n        \n        # Targets: [Green, Dead, Clover] (same order as exp01)\n        targets = torch.tensor([\n            row[\"Dry_Green_g\"],\n            row[\"Dry_Dead_g\"],\n            row[\"Dry_Clover_g\"],\n        ], dtype=torch.float32)\n        \n        # NEW: Zero classification labels\n        zero_labels = torch.tensor([\n            1.0 if row[\"Dry_Green_g\"] > 0 else 0.0,\n            1.0 if row[\"Dry_Dead_g\"] > 0 else 0.0,\n            1.0 if row[\"Dry_Clover_g\"] > 0 else 0.0,\n        ], dtype=torch.float32)\n        \n        return img_left, img_right, targets, zero_labels\n\n\n# ============================================\n# IMPROVED: Enhanced Transforms for NSW\n# ============================================\ndef get_transforms(is_train=True):\n    if is_train:\n        return transforms.Compose([\n            transforms.Resize((config.img_size, config.img_size)),\n            transforms.RandomHorizontalFlip(p=0.5),\n            transforms.RandomVerticalFlip(p=0.5),\n            transforms.RandomApply([\n                transforms.RandomRotation([90, 90])\n            ], p=0.5),\n            # ENHANCED: Stronger ColorJitter for NSW drought conditions\n            transforms.ColorJitter(\n                brightness=0.25,   # Increased from 0.15\n                contrast=0.25,     # Increased from 0.15\n                saturation=0.25,   # Increased from 0.15\n                hue=0.08           # Increased from 0.05\n            ),\n            # NEW: RandomGrayscale for dry/desaturated scenes\n            transforms.RandomGrayscale(p=0.1),\n            transforms.ToTensor(),\n            transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225]\n            )\n        ])\n    else:\n        return transforms.Compose([\n            transforms.Resize((config.img_size, config.img_size)),\n            transforms.ToTensor(),\n            transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225]\n            )\n        ])\n\n\n# ============================================\n# Model (from exp01 + zero classification head)\n# ============================================\nclass BiomassRegressor(nn.Module):\n    def __init__(self, model_name, pretrained=True, freeze_backbone=True, dropout=0.2):\n        super().__init__()\n        \n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg'\n        )\n        \n        if freeze_backbone:\n            for param in self.backbone.parameters():\n                param.requires_grad = False\n            print(f\"Backbone frozen.\")\n        \n        feat_dim = self.backbone.num_features * 2  # Left + Right\n        print(f\"Feature dimension: {feat_dim}\")\n        \n        # Regression heads (same as exp01)\n        def make_head(in_dim):\n            return nn.Sequential(\n                nn.Linear(in_dim, 256),\n                nn.ReLU(inplace=True),\n                nn.Dropout(dropout),\n                nn.Linear(256, 64),\n                nn.ReLU(inplace=True),\n                nn.Linear(64, 1)\n            )\n        \n        self.head_green = make_head(feat_dim)\n        self.head_dead = make_head(feat_dim)\n        self.head_clover = make_head(feat_dim)\n        \n        # NEW: Zero classification head (no Sigmoid - use BCEWithLogitsLoss)\n        self.zero_cls_head = nn.Sequential(\n            nn.Linear(feat_dim, 128),\n            nn.ReLU(inplace=True),\n            nn.Dropout(dropout),\n            nn.Linear(128, 3)\n        )\n        \n        self.softplus = nn.Softplus(beta=1.0)\n    \n    def forward(self, x_left, x_right, return_zero_cls=True):\n        feat_left = self.backbone(x_left)\n        feat_right = self.backbone(x_right)\n        feat = torch.cat([feat_left, feat_right], dim=1)\n        \n        green = self.softplus(self.head_green(feat))\n        dead = self.softplus(self.head_dead(feat))\n        clover = self.softplus(self.head_clover(feat))\n        \n        preds = torch.cat([green, dead, clover], dim=1)\n        \n        if return_zero_cls:\n            zero_cls = self.zero_cls_head(feat)\n            return preds, zero_cls\n        \n        return preds\n\n\n# ============================================\n# IMPROVED: Loss Function with Zero Classification\n# ============================================\nclass ImprovedLoss(nn.Module):\n    def __init__(self, weights, zero_cls_weight=0.3, beta=1.0):\n        super().__init__()\n        self.register_buffer('weights', torch.tensor(weights))\n        self.zero_cls_weight = zero_cls_weight\n        self.beta = beta\n        self.bce = nn.BCEWithLogitsLoss()  # Safe with autocast\n    \n    def forward(self, pred, target, zero_cls_pred=None, zero_cls_true=None):\n        # Weighted SmoothL1\n        diff = torch.abs(pred - target)\n        smooth_l1 = torch.where(\n            diff < self.beta,\n            0.5 * diff ** 2 / self.beta,\n            diff - 0.5 * self.beta\n        )\n        weighted_loss = smooth_l1 * self.weights\n        reg_loss = weighted_loss.mean()\n        \n        # Zero classification loss\n        cls_loss = 0.0\n        if zero_cls_pred is not None and zero_cls_true is not None:\n            cls_loss = self.bce(zero_cls_pred, zero_cls_true)\n            total_loss = reg_loss + self.zero_cls_weight * cls_loss\n            return total_loss, reg_loss.item(), cls_loss.item()\n        \n        return reg_loss, reg_loss.item(), 0.0\n\n\n# ============================================\n# Metrics (from exp01)\n# ============================================\ndef weighted_r2_score(y_true, y_pred, weights=None):\n    if weights is None:\n        weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n    \n    r2_scores = []\n    for i in range(5):\n        y_t = y_true[:, i]\n        y_p = y_pred[:, i]\n        ss_res = np.sum((y_t - y_p) ** 2)\n        ss_tot = np.sum((y_t - np.mean(y_t)) ** 2)\n        r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n        r2_scores.append(r2)\n    \n    r2_scores = np.array(r2_scores)\n    weighted_r2 = np.sum(r2_scores * weights) / np.sum(weights)\n    \n    return weighted_r2, r2_scores\n\n\ndef derive_all_targets(preds_3):\n    \"\"\"Convert [Green, Dead, Clover] to [Green, Clover, Dead, GDM, Total]\"\"\"\n    green = preds_3[:, 0]\n    dead = preds_3[:, 1]\n    clover = preds_3[:, 2]\n    \n    gdm = green + clover\n    total = green + dead + clover\n    \n    return np.column_stack([green, clover, dead, gdm, total])\n\n\ndef calc_metric(outputs, targets):\n    y_pred = derive_all_targets(outputs)\n    y_true = derive_all_targets(targets)\n    return weighted_r2_score(y_true, y_pred)\n\n\n# ============================================\n# Scheduler\n# ============================================\nclass WarmupCosineScheduler:\n    def __init__(self, optimizer, warmup_epochs, total_epochs, base_lr, min_lr=1e-6):\n        self.optimizer = optimizer\n        self.warmup_epochs = warmup_epochs\n        self.total_epochs = total_epochs\n        self.base_lr = base_lr\n        self.min_lr = min_lr\n        self.current_epoch = 0\n    \n    def step(self):\n        self.current_epoch += 1\n        if self.current_epoch <= self.warmup_epochs:\n            lr = self.base_lr * (self.current_epoch / self.warmup_epochs)\n        else:\n            progress = (self.current_epoch - self.warmup_epochs) / (self.total_epochs - self.warmup_epochs)\n            lr = self.min_lr + 0.5 * (self.base_lr - self.min_lr) * (1 + np.cos(np.pi * progress))\n        \n        for param_group in self.optimizer.param_groups:\n            param_group['lr'] = lr\n        return lr\n\n\n# ============================================\n# Training Loop\n# ============================================\ndef train_one_epoch(model, loader, criterion, optimizer, scaler, device):\n    model.train()\n    total_reg_loss = 0\n    total_cls_loss = 0\n    all_preds = []\n    all_targets = []\n    \n    for img_left, img_right, targets, zero_labels in tqdm(loader, desc=\"Training\"):\n        img_left = img_left.to(device)\n        img_right = img_right.to(device)\n        targets = targets.to(device)\n        zero_labels = zero_labels.to(device)\n        \n        optimizer.zero_grad()\n        \n        with autocast(enabled=config.use_amp):\n            if config.use_zero_cls:\n                preds, zero_cls = model(img_left, img_right, return_zero_cls=True)\n                loss, reg_loss, cls_loss = criterion(preds, targets, zero_cls, zero_labels)\n            else:\n                preds = model(img_left, img_right, return_zero_cls=False)\n                loss, reg_loss, cls_loss = criterion(preds, targets, None, None)\n        \n        scaler.scale(loss).backward()\n        scaler.unscale_(optimizer)\n        torch.nn.utils.clip_grad_norm_(model.parameters(), config.gradient_clip)\n        scaler.step(optimizer)\n        scaler.update()\n        \n        total_reg_loss += reg_loss\n        total_cls_loss += cls_loss\n        \n        all_preds.append(preds.detach().cpu().numpy())\n        all_targets.append(targets.cpu().numpy())\n    \n    all_preds = np.concatenate(all_preds)\n    all_targets = np.concatenate(all_targets)\n    \n    weighted_r2, _ = calc_metric(all_preds, all_targets)\n    \n    return total_reg_loss / len(loader), total_cls_loss / len(loader), weighted_r2\n\n\ndef validate(model, loader, criterion, device):\n    model.eval()\n    total_reg_loss = 0\n    total_cls_loss = 0\n    all_preds = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for img_left, img_right, targets, zero_labels in tqdm(loader, desc=\"Validating\"):\n            img_left = img_left.to(device)\n            img_right = img_right.to(device)\n            targets = targets.to(device)\n            zero_labels = zero_labels.to(device)\n            \n            with autocast(enabled=config.use_amp):\n                if config.use_zero_cls:\n                    preds, zero_cls = model(img_left, img_right, return_zero_cls=True)\n                    loss, reg_loss, cls_loss = criterion(preds, targets, zero_cls, zero_labels)\n                else:\n                    preds = model(img_left, img_right, return_zero_cls=False)\n                    loss, reg_loss, cls_loss = criterion(preds, targets, None, None)\n            \n            total_reg_loss += reg_loss\n            total_cls_loss += cls_loss\n            \n            all_preds.append(preds.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n    \n    all_preds = np.concatenate(all_preds)\n    all_targets = np.concatenate(all_targets)\n    \n    weighted_r2, r2_scores = calc_metric(all_preds, all_targets)\n    \n    return total_reg_loss / len(loader), total_cls_loss / len(loader), weighted_r2, r2_scores\n\n\ndef train_fold(fold, train_df, val_df):\n    print(f\"\\n{'='*60}\")\n    print(f\"Fold {fold + 1}/{config.n_folds}\")\n    print(f\"{'='*60}\")\n    print(f\"Train: {len(train_df)}, Val: {len(val_df)}\")\n    \n    device = torch.device(config.device)\n    \n    model = BiomassRegressor(\n        config.model_name,\n        pretrained=True,\n        freeze_backbone=True,\n        dropout=config.dropout\n    ).to(device)\n    \n    train_transform = get_transforms(is_train=True)\n    val_transform = get_transforms(is_train=False)\n    \n    train_dataset = BiomassDataset(train_df, config.data_root, train_transform, is_train=True)\n    val_dataset = BiomassDataset(val_df, config.data_root, val_transform, is_train=False)\n    \n    train_loader = DataLoader(train_dataset, batch_size=config.batch_size,\n                              shuffle=True, num_workers=config.num_workers, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=config.batch_size,\n                            shuffle=False, num_workers=config.num_workers, pin_memory=True)\n    \n    criterion = ImprovedLoss(config.loss_weights, config.zero_cls_weight).to(device)\n    \n    trainable_params = []\n    trainable_params.extend(model.head_green.parameters())\n    trainable_params.extend(model.head_dead.parameters())\n    trainable_params.extend(model.head_clover.parameters())\n    trainable_params.extend(model.zero_cls_head.parameters())\n    \n    optimizer = torch.optim.AdamW(trainable_params, lr=config.learning_rate, weight_decay=config.weight_decay)\n    scheduler = WarmupCosineScheduler(optimizer, config.warmup_epochs, config.epochs, config.learning_rate)\n    scaler = GradScaler(enabled=config.use_amp)\n    \n    best_r2 = -float('inf')\n    best_state = None\n    \n    for epoch in range(config.epochs):\n        lr = scheduler.step()\n        train_reg, train_cls, train_r2 = train_one_epoch(model, train_loader, criterion, optimizer, scaler, device)\n        val_reg, val_cls, val_r2, r2_scores = validate(model, val_loader, criterion, device)\n        \n        # Detailed output\n        print(f\"\\nEpoch {epoch+1:2d}/{config.epochs} | LR: {lr:.2e}\")\n        print(f\"  Train Loss: {train_reg:.4f} (cls: {train_cls:.4f}) | Train R²: {train_r2:.4f}\")\n        print(f\"  Val Loss: {val_reg:.4f} (cls: {val_cls:.4f}) | Val Weighted R²: {val_r2:.4f}\")\n        print(f\"  R² per target: Green={r2_scores[0]:.3f}, Clover={r2_scores[1]:.3f}, \"\n              f\"Dead={r2_scores[2]:.3f}, GDM={r2_scores[3]:.3f}, Total={r2_scores[4]:.3f}\")\n        \n        if val_r2 > best_r2:\n            best_r2 = val_r2\n            # Deep copy to avoid shallow copy bug\n            best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}\n            print(f\"  ✅ New best model saved! R²: {best_r2:.4f}\")\n    \n    # Save\n    path = os.path.join(config.output_dir, f\"exp04_fold{fold}.pth\")\n    torch.save(best_state, path)\n    print(f\"\\nSaved: {path}\")\n    \n    return best_r2\n\n\ndef main():\n    set_seed(config.seed)\n    \n    print(\"=\"*60)\n    print(\"exp04: Improved Biomass Regression\")\n    print(\"=\"*60)\n    print(\"Based on exp01 structure with improvements:\")\n    print(f\"  1. Dead weight: 1.5 → {config.loss_weights[1]}\")\n    print(f\"  2. Zero classification: {config.use_zero_cls}\")\n    print(f\"  3. Enhanced augmentation\")\n    print(\"=\"*60)\n    \n    config.model_name, config.img_size = get_best_available_model()\n    \n    print(\"\\nLoading data...\")\n    train_df = pd.read_csv(config.train_path)\n    \n    df = train_df.pivot(index=\"image_path\", columns=\"target_name\", values=\"target\").reset_index()\n    df.columns.name = None\n    \n    state_map = train_df.drop_duplicates(\"image_path\")[[\"image_path\", \"State\"]]\n    df = df.merge(state_map, on=\"image_path\")\n    \n    print(f\"Images: {len(df)}\")\n    \n    skf = StratifiedKFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n    fold_r2s = []\n    for fold, (train_idx, val_idx) in enumerate(skf.split(df, df['State'])):\n        train_fold_df = df.iloc[train_idx]\n        val_fold_df = df.iloc[val_idx]\n        r2 = train_fold(fold, train_fold_df, val_fold_df)\n        fold_r2s.append(r2)\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"COMPLETE\")\n    print(\"=\"*60)\n    print(f\"CV Weighted R²: {np.mean(fold_r2s):.4f} ± {np.std(fold_r2s):.4f}\")\n\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-05T00:49:25.350208Z","iopub.execute_input":"2026-01-05T00:49:25.350789Z","iopub.status.idle":"2026-01-05T01:09:36.261018Z","shell.execute_reply.started":"2026-01-05T00:49:25.350761Z","shell.execute_reply":"2026-01-05T01:09:36.259968Z"}},"outputs":[],"execution_count":null}]}