{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31240,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# DINOv3を使うためにtimmを最新版にアップデート\n!pip install timm -U --no-deps\n\n# 確認\nimport timm\nprint(f\"Timm version: {timm.__version__}\")  # 1.0.22以上が必要","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-09T13:50:31.811635Z","iopub.execute_input":"2026-01-09T13:50:31.811893Z","iopub.status.idle":"2026-01-09T13:50:45.944833Z","shell.execute_reply.started":"2026-01-09T13:50:31.811869Z","shell.execute_reply":"2026-01-09T13:50:45.944203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nexp07: EDA-based Optimal CV Strategy + Simple Model\n====================================================\nKey changes from exp04:\n1. StratifiedGroupKFold (State stratify + Sampling_Date group)\n2. 4-fold instead of 5-fold (better stratification from EDA)\n3. seed=33 (optimal from seed search)\n4. Simple model (no gates, no 2-stage)\n5. img_size = 512\n\nBased on comprehensive EDA analysis:\n- Date-State matrix shows most dates have only 1 state\n- 4-fold with seed=33 gives missing_states=1 (vs 2 for 5-fold)\n- State stratification ensures WA (9%) represented in each fold\n\"\"\"\n\nimport os\nimport sys\nimport random\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom PIL import Image\nfrom datetime import datetime\nfrom dataclasses import dataclass, field\nfrom typing import Tuple, List, Dict\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nfrom sklearn.model_selection import StratifiedGroupKFold\nimport timm\nfrom torchvision import transforms\nfrom tqdm import tqdm\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n# ============================================\n# Configuration\n# ============================================\n@dataclass\nclass Config:\n    # Paths (Kaggle)\n    data_root: str = \"/kaggle/input/csiro-biomass\"\n    train_path: str = \"/kaggle/input/csiro-biomass/train.csv\"\n    output_dir: str = \"/kaggle/working\"\n    \n    # Model - Use 512 for better features\n    model_name: str = \"vit_base_patch16_dinov3.lvd1689m\"\n    img_size: int = 512\n    \n    # Training - 4-fold is optimal based on EDA (better stratification)\n    n_folds: int = 4\n    epochs: int = 40\n    batch_size: int = 4  # Smaller batch for 512 images\n    num_workers: int = 4\n    \n    # Optimizer\n    learning_rate: float = 5e-4\n    weight_decay: float = 0.01\n    \n    # Scheduler\n    warmup_epochs: int = 3\n    \n    # Image preprocessing\n    crop_bottom_ratio: float = 0.1\n    remove_timestamp: bool = True\n    \n    # Loss weights [Green, Dead, Clover]\n    loss_weights: List[float] = field(default_factory=lambda: [1.0, 2.0, 1.0])\n    \n    # Other - seed=33 is optimal from EDA seed search\n    seed: int = 33\n    device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    use_amp: bool = torch.cuda.is_available()\n    gradient_clip: float = 1.0\n    dropout: float = 0.2\n\n\nconfig = Config()\n\n\n# ============================================\n# Utilities\n# ============================================\ndef get_best_available_model():\n    models = [\n        (\"vit_base_patch16_dinov3.lvd1689m\", 512),\n        (\"vit_base_patch14_dinov2.lvd142m\", 518),\n        (\"convnext_base.fb_in22k_ft_in1k\", 224),\n    ]\n    for name, size in models:\n        try:\n            _ = timm.create_model(name, pretrained=False, num_classes=0)\n            print(f\"✅ Model: {name}\")\n            return name, size\n        except:\n            continue\n    raise RuntimeError(\"No model found\")\n\n\ndef set_seed(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\n# ============================================\n# Image Preprocessing\n# ============================================\ndef clean_image(img_array):\n    h, w = img_array.shape[:2]\n    \n    if config.crop_bottom_ratio > 0:\n        new_h = int(h * (1 - config.crop_bottom_ratio))\n        img_array = img_array[:new_h, :]\n    \n    if config.remove_timestamp:\n        hsv = cv2.cvtColor(img_array, cv2.COLOR_RGB2HSV)\n        lower_orange = np.array([5, 150, 150])\n        upper_orange = np.array([25, 255, 255])\n        mask = cv2.inRange(hsv, lower_orange, upper_orange)\n        mask = cv2.dilate(mask, np.ones((3, 3), np.uint8), iterations=2)\n        \n        if np.sum(mask) > 0:\n            img_bgr = cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)\n            img_bgr = cv2.inpaint(img_bgr, mask, 3, cv2.INPAINT_TELEA)\n            img_array = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n    \n    return img_array\n\n\n# ============================================\n# Dataset\n# ============================================\nclass BiomassDataset(Dataset):\n    def __init__(self, df, root_dir, transform=None, is_train=True):\n        self.df = df.reset_index(drop=True)\n        self.root_dir = root_dir\n        self.transform = transform\n        self.is_train = is_train\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        img_path = os.path.join(self.root_dir, row[\"image_path\"])\n        img = Image.open(img_path).convert(\"RGB\")\n        img_array = np.array(img)\n        \n        img_array = clean_image(img_array)\n        \n        # Left-right split\n        h, w = img_array.shape[:2]\n        mid = w // 2\n        img_left = Image.fromarray(img_array[:, :mid])\n        img_right = Image.fromarray(img_array[:, mid:])\n        \n        if self.transform:\n            seed = np.random.randint(2147483647)\n            \n            random.seed(seed)\n            torch.manual_seed(seed)\n            img_left = self.transform(img_left)\n            \n            random.seed(seed)\n            torch.manual_seed(seed)\n            img_right = self.transform(img_right)\n        \n        # Targets: [Green, Dead, Clover]\n        targets = torch.tensor([\n            row[\"Dry_Green_g\"],\n            row[\"Dry_Dead_g\"],\n            row[\"Dry_Clover_g\"],\n        ], dtype=torch.float32)\n        \n        return img_left, img_right, targets\n\n\n# ============================================\n# Transforms\n# ============================================\ndef get_transforms(is_train=True):\n    if is_train:\n        return transforms.Compose([\n            transforms.Resize((config.img_size, config.img_size)),\n            transforms.RandomHorizontalFlip(p=0.5),\n            transforms.RandomVerticalFlip(p=0.5),\n            transforms.RandomApply([\n                transforms.RandomRotation([90, 90])\n            ], p=0.5),\n            transforms.ColorJitter(\n                brightness=0.25,\n                contrast=0.25,\n                saturation=0.25,\n                hue=0.08\n            ),\n            transforms.RandomGrayscale(p=0.1),\n            transforms.ToTensor(),\n            transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225]\n            )\n        ])\n    else:\n        return transforms.Compose([\n            transforms.Resize((config.img_size, config.img_size)),\n            transforms.ToTensor(),\n            transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225]\n            )\n        ])\n\n\n# ============================================\n# Simple Model (no gates)\n# ============================================\nclass SimpleBiomassRegressor(nn.Module):\n    \"\"\"\n    Simple regressor without gates or 2-stage prediction.\n    Based on exp01 structure.\n    \"\"\"\n    \n    def __init__(self, model_name, pretrained=True, freeze_backbone=True, dropout=0.2):\n        super().__init__()\n        \n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg'\n        )\n        \n        if freeze_backbone:\n            for param in self.backbone.parameters():\n                param.requires_grad = False\n            print(f\"Backbone frozen.\")\n        \n        feat_dim = self.backbone.num_features * 2  # Left + Right\n        print(f\"Feature dimension: {feat_dim}\")\n        \n        def make_head(in_dim):\n            return nn.Sequential(\n                nn.Linear(in_dim, 256),\n                nn.ReLU(inplace=True),\n                nn.Dropout(dropout),\n                nn.Linear(256, 64),\n                nn.ReLU(inplace=True),\n                nn.Linear(64, 1)\n            )\n        \n        self.head_green = make_head(feat_dim)\n        self.head_dead = make_head(feat_dim)\n        self.head_clover = make_head(feat_dim)\n        \n        self.softplus = nn.Softplus(beta=1.0)\n    \n    def forward(self, x_left, x_right):\n        feat_left = self.backbone(x_left)\n        feat_right = self.backbone(x_right)\n        feat = torch.cat([feat_left, feat_right], dim=1)\n        \n        green = self.softplus(self.head_green(feat))\n        dead = self.softplus(self.head_dead(feat))\n        clover = self.softplus(self.head_clover(feat))\n        \n        return torch.cat([green, dead, clover], dim=1)\n\n\n# ============================================\n# Loss\n# ============================================\nclass WeightedSmoothL1Loss(nn.Module):\n    def __init__(self, target_weights, beta=1.0):\n        super().__init__()\n        self.register_buffer('target_weights', torch.tensor(target_weights))\n        self.beta = beta\n    \n    def forward(self, pred, target):\n        diff = torch.abs(pred - target)\n        smooth_l1 = torch.where(\n            diff < self.beta,\n            0.5 * diff ** 2 / self.beta,\n            diff - 0.5 * self.beta\n        )\n        weighted_loss = smooth_l1 * self.target_weights\n        return weighted_loss.mean()\n\n\n# ============================================\n# Metrics\n# ============================================\ndef weighted_r2_score(y_true, y_pred, weights=None):\n    if weights is None:\n        weights = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n    \n    r2_scores = []\n    for i in range(5):\n        y_t = y_true[:, i]\n        y_p = y_pred[:, i]\n        ss_res = np.sum((y_t - y_p) ** 2)\n        ss_tot = np.sum((y_t - np.mean(y_t)) ** 2)\n        r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n        r2_scores.append(r2)\n    \n    r2_scores = np.array(r2_scores)\n    return np.sum(r2_scores * weights) / np.sum(weights), r2_scores\n\n\ndef derive_all_targets(preds_3):\n    green = preds_3[:, 0]\n    dead = preds_3[:, 1]\n    clover = preds_3[:, 2]\n    gdm = green + clover\n    total = green + dead + clover\n    return np.column_stack([green, clover, dead, gdm, total])\n\n\ndef calc_metric(outputs, targets):\n    y_pred = derive_all_targets(outputs)\n    y_true = derive_all_targets(targets)\n    return weighted_r2_score(y_true, y_pred)\n\n\n# ============================================\n# Scheduler\n# ============================================\nclass WarmupCosineScheduler:\n    def __init__(self, optimizer, warmup_epochs, total_epochs, base_lr, min_lr=1e-6):\n        self.optimizer = optimizer\n        self.warmup_epochs = warmup_epochs\n        self.total_epochs = total_epochs\n        self.base_lr = base_lr\n        self.min_lr = min_lr\n        self.current_epoch = 0\n    \n    def step(self):\n        self.current_epoch += 1\n        if self.current_epoch <= self.warmup_epochs:\n            lr = self.base_lr * (self.current_epoch / self.warmup_epochs)\n        else:\n            progress = (self.current_epoch - self.warmup_epochs) / (self.total_epochs - self.warmup_epochs)\n            lr = self.min_lr + 0.5 * (self.base_lr - self.min_lr) * (1 + np.cos(np.pi * progress))\n        \n        for param_group in self.optimizer.param_groups:\n            param_group['lr'] = lr\n        return lr\n\n\n# ============================================\n# Training Loop\n# ============================================\ndef train_one_epoch(model, loader, criterion, optimizer, scaler, device):\n    model.train()\n    total_loss = 0\n    all_preds = []\n    all_targets = []\n    \n    for img_left, img_right, targets in tqdm(loader, desc=\"Training\"):\n        img_left = img_left.to(device)\n        img_right = img_right.to(device)\n        targets = targets.to(device)\n        \n        optimizer.zero_grad()\n        \n        with autocast(enabled=config.use_amp):\n            preds = model(img_left, img_right)\n            loss = criterion(preds, targets)\n        \n        scaler.scale(loss).backward()\n        scaler.unscale_(optimizer)\n        torch.nn.utils.clip_grad_norm_(model.parameters(), config.gradient_clip)\n        scaler.step(optimizer)\n        scaler.update()\n        \n        total_loss += loss.item()\n        all_preds.append(preds.detach().cpu().numpy())\n        all_targets.append(targets.cpu().numpy())\n    \n    all_preds = np.concatenate(all_preds)\n    all_targets = np.concatenate(all_targets)\n    weighted_r2, _ = calc_metric(all_preds, all_targets)\n    \n    return total_loss / len(loader), weighted_r2\n\n\ndef validate(model, loader, criterion, device):\n    model.eval()\n    total_loss = 0\n    all_preds = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for img_left, img_right, targets in tqdm(loader, desc=\"Validating\"):\n            img_left = img_left.to(device)\n            img_right = img_right.to(device)\n            targets = targets.to(device)\n            \n            with autocast(enabled=config.use_amp):\n                preds = model(img_left, img_right)\n                loss = criterion(preds, targets)\n            \n            total_loss += loss.item()\n            all_preds.append(preds.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n    \n    all_preds = np.concatenate(all_preds)\n    all_targets = np.concatenate(all_targets)\n    weighted_r2, r2_scores = calc_metric(all_preds, all_targets)\n    \n    return total_loss / len(loader), weighted_r2, r2_scores\n\n\ndef train_fold(fold, train_df, val_df):\n    print(f\"\\n{'='*60}\")\n    print(f\"Fold {fold + 1}/{config.n_folds}\")\n    print(f\"{'='*60}\")\n    print(f\"Train: {len(train_df)}, Val: {len(val_df)}\")\n    \n    device = torch.device(config.device)\n    \n    model = SimpleBiomassRegressor(\n        config.model_name,\n        pretrained=True,\n        freeze_backbone=True,\n        dropout=config.dropout\n    ).to(device)\n    \n    train_transform = get_transforms(is_train=True)\n    val_transform = get_transforms(is_train=False)\n    \n    train_dataset = BiomassDataset(train_df, config.data_root, train_transform, is_train=True)\n    val_dataset = BiomassDataset(val_df, config.data_root, val_transform, is_train=False)\n    \n    train_loader = DataLoader(train_dataset, batch_size=config.batch_size,\n                              shuffle=True, num_workers=config.num_workers, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=config.batch_size,\n                            shuffle=False, num_workers=config.num_workers, pin_memory=True)\n    \n    criterion = WeightedSmoothL1Loss(target_weights=config.loss_weights).to(device)\n    \n    # Collect trainable parameters\n    trainable_params = []\n    trainable_params.extend(model.head_green.parameters())\n    trainable_params.extend(model.head_dead.parameters())\n    trainable_params.extend(model.head_clover.parameters())\n    \n    optimizer = torch.optim.AdamW(trainable_params, lr=config.learning_rate, weight_decay=config.weight_decay)\n    scheduler = WarmupCosineScheduler(optimizer, config.warmup_epochs, config.epochs, config.learning_rate)\n    scaler = GradScaler(enabled=config.use_amp)\n    \n    best_r2 = -float('inf')\n    best_state = None\n    patience = 10\n    patience_counter = 0\n    \n    for epoch in range(config.epochs):\n        lr = scheduler.step()\n        train_loss, train_r2 = train_one_epoch(model, train_loader, criterion, optimizer, scaler, device)\n        val_loss, val_r2, r2_scores = validate(model, val_loader, criterion, device)\n        \n        print(f\"\\nEpoch {epoch+1:2d}/{config.epochs} | LR: {lr:.2e}\")\n        print(f\"  Train Loss: {train_loss:.4f} | Train R²: {train_r2:.4f}\")\n        print(f\"  Val Loss: {val_loss:.4f} | Val Weighted R²: {val_r2:.4f}\")\n        print(f\"  R² per target: Green={r2_scores[0]:.3f}, Clover={r2_scores[1]:.3f}, \"\n              f\"Dead={r2_scores[2]:.3f}, GDM={r2_scores[3]:.3f}, Total={r2_scores[4]:.3f}\")\n        \n        if val_r2 > best_r2:\n            best_r2 = val_r2\n            best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}\n            print(f\"  ✅ New best model saved! R²: {best_r2:.4f}\")\n            patience_counter = 0\n        else:\n            patience_counter += 1\n            if patience_counter >= patience:\n                print(f\"  🛑 Early stopping at epoch {epoch + 1}\")\n                break\n    \n    # Save\n    path = os.path.join(config.output_dir, f\"exp07_fold{fold}.pth\")\n    torch.save(best_state, path)\n    print(f\"\\nSaved: {path}\")\n    \n    return best_r2\n\n\ndef main():\n    set_seed(config.seed)\n    \n    print(\"=\"*60)\n    print(\"exp07: EDA-based Optimal CV Strategy + Simple Model\")\n    print(\"=\"*60)\n    print(\"Key Changes:\")\n    print(\"  1. StratifiedGroupKFold (State stratify + Sampling_Date group)\")\n    print(\"  2. 4-fold with seed=33 (optimal from EDA)\")\n    print(\"  3. Simple model (no gates)\")\n    print(f\"  4. img_size = {config.img_size}\")\n    print(\"=\"*60)\n    \n    config.model_name, config.img_size = get_best_available_model()\n    \n    print(\"\\nLoading data...\")\n    train_df = pd.read_csv(config.train_path)\n    \n    # Create aggregated dataframe\n    df = train_df.pivot(index=\"image_path\", columns=\"target_name\", values=\"target\").reset_index()\n    df.columns.name = None\n    \n    # Merge metadata\n    meta_cols = [\"image_path\", \"State\", \"Sampling_Date\"]\n    meta_df = train_df.drop_duplicates(\"image_path\")[meta_cols]\n    df = df.merge(meta_df, on=\"image_path\")\n    \n    print(f\"Images: {len(df)}\")\n    print(f\"State distribution:\\n{df['State'].value_counts()}\")\n    print(f\"Unique Sampling Dates: {df['Sampling_Date'].nunique()}\")\n    \n    # ===== FINAL CV STRATEGY (EDA-based Optimal) =====\n    # Stratify on State (ensures WA is represented in each fold)\n    df['stratify_col'] = df['State']\n    \n    # Group by Sampling_Date (prevents temporal data leakage)\n    df['group_key'] = df['Sampling_Date']\n    \n    print(f\"\\nStratify (State) distribution:\\n{df['stratify_col'].value_counts()}\")\n    print(f\"Unique groups (Sampling_Date): {df['group_key'].nunique()}\")\n    \n    # StratifiedGroupKFold with optimal settings from EDA:\n    # - 4-fold (config.n_folds)\n    # - seed=33 (config.seed) \n    sgkf = StratifiedGroupKFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n    fold_r2s = []\n    \n    for fold, (train_idx, val_idx) in enumerate(sgkf.split(df, df['stratify_col'], df['group_key'])):\n        train_fold_df = df.iloc[train_idx]\n        val_fold_df = df.iloc[val_idx]\n        \n        # Print fold info\n        print(f\"\\n--- Fold {fold+1} Info ---\")\n        print(f\"Train states: {train_fold_df['State'].value_counts().to_dict()}\")\n        print(f\"Val states: {val_fold_df['State'].value_counts().to_dict()}\")\n        print(f\"Train dates: {train_fold_df['Sampling_Date'].nunique()}\")\n        print(f\"Val dates: {val_fold_df['Sampling_Date'].nunique()}\")\n        \n        r2 = train_fold(fold, train_fold_df, val_fold_df)\n        fold_r2s.append(r2)\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"COMPLETE\")\n    print(\"=\"*60)\n    print(f\"CV Weighted R²: {np.mean(fold_r2s):.4f} ± {np.std(fold_r2s):.4f}\")\n    print(f\"Per-fold R²: {[f'{r:.4f}' for r in fold_r2s]}\")\n\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-09T13:55:36.106266Z","iopub.execute_input":"2026-01-09T13:55:36.106624Z"}},"outputs":[],"execution_count":null}]}