{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# DINOv3を使うためにtimmを最新版にアップデート\n!pip install timm -U --no-deps\n\n# 確認\nimport timm\nprint(f\"Timm version: {timm.__version__}\")  # 1.0.22以上が必要","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-04T12:36:45.711668Z","iopub.execute_input":"2026-01-04T12:36:45.711903Z","iopub.status.idle":"2026-01-04T12:36:59.243881Z","shell.execute_reply.started":"2026-01-04T12:36:45.711878Z","shell.execute_reply":"2026-01-04T12:36:59.243220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nexp03: State Classification Model (IMPROVED)\n=============================================\nPredicts Australian state (NSW, Vic, Tas, WA) from pasture images.\nImprovements:\n1. Focal Loss for class imbalance\n2. Label Smoothing (0.1)\n3. Stronger augmentation\n4. Deeper classifier head\n5. More epochs (30)\n\"\"\"\n\nimport os\nimport sys\nimport random\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom datetime import datetime\nfrom dataclasses import dataclass\nfrom typing import Tuple, List, Dict, Optional\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nimport timm\nfrom torchvision import transforms\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n# ============================================================\n# Configuration (IMPROVED)\n# ============================================================\n\n@dataclass\nclass Config:\n    \"\"\"Training configuration\"\"\"\n    # Paths (Kaggle environment)\n    data_root: str = \"/kaggle/input/csiro-biomass\"\n    train_path: str = \"/kaggle/input/csiro-biomass/train.csv\"\n    output_dir: str = \"/kaggle/working\"\n    \n    # Model\n    model_name: str = \"vit_base_patch16_dinov3.lvd1689m\"\n    img_size: int = 512\n    num_classes: int = 4\n    \n    # Training - IMPROVED\n    n_folds: int = 5\n    epochs: int = 30  # Increased from 20\n    batch_size: int = 32\n    num_workers: int = 4\n    \n    # Optimizer\n    learning_rate: float = 2e-4  # Increased from 1e-4\n    weight_decay: float = 0.01\n    \n    # Scheduler\n    warmup_epochs: int = 3\n    \n    # Loss - NEW\n    label_smoothing: float = 0.1  # NEW: Reduce overconfidence\n    focal_gamma: float = 2.0  # NEW: Focus on hard examples\n    \n    # Other\n    seed: int = 42\n    device: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    use_amp: bool = True\n    gradient_clip: float = 1.0\n    dropout: float = 0.4  # Increased from 0.3\n    \n    # State mapping\n    state_to_idx: Dict[str, int] = None\n    idx_to_state: Dict[int, str] = None\n    \n    def __post_init__(self):\n        self.state_to_idx = {\"NSW\": 0, \"Vic\": 1, \"Tas\": 2, \"WA\": 3}\n        self.idx_to_state = {v: k for k, v in self.state_to_idx.items()}\n\n\nconfig = Config()\n\n\n# ============================================================\n# Model Availability Check\n# ============================================================\n\ndef get_best_available_model() -> Tuple[str, int]:\n    models_to_try = [\n        (\"vit_base_patch16_dinov3.lvd1689m\", 512),\n        (\"vit_base_patch14_dinov2.lvd142m\", 518),\n        (\"convnext_base.fb_in22k_ft_in1k\", 224),\n    ]\n    \n    for model_name, img_size in models_to_try:\n        try:\n            _ = timm.create_model(model_name, pretrained=False, num_classes=0)\n            print(f\"✅ Using model: {model_name}\")\n            return model_name, img_size\n        except Exception:\n            continue\n    \n    raise RuntimeError(\"No suitable model found!\")\n\n\ndef set_seed(seed: int):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\n# ============================================================\n# Dataset with Enhanced Augmentation\n# ============================================================\n\nclass StateClassificationDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, transform=None, is_train: bool = True):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n        self.is_train = is_train\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def preprocess_image(self, img: Image.Image) -> Image.Image:\n        width, height = img.size\n        crop_height = int(height * 0.9)\n        img = img.crop((0, 0, width, crop_height))\n        return img\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(config.data_root, row['image_path'])\n        \n        img = Image.open(img_path).convert('RGB')\n        img = self.preprocess_image(img)\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        state = row['State']\n        label = config.state_to_idx[state]\n        \n        return img, label\n\n\ndef get_transforms(is_train: bool = True):\n    \"\"\"IMPROVED: Stronger augmentation\"\"\"\n    if is_train:\n        return transforms.Compose([\n            transforms.Resize((config.img_size, config.img_size)),\n            transforms.RandomHorizontalFlip(p=0.5),\n            transforms.RandomVerticalFlip(p=0.3),\n            transforms.RandomRotation(15),\n            # ENHANCED: Stronger color jitter\n            transforms.ColorJitter(\n                brightness=0.3,\n                contrast=0.3,\n                saturation=0.3,\n                hue=0.1\n            ),\n            # NEW: RandomGrayscale\n            transforms.RandomGrayscale(p=0.1),\n            # NEW: RandomErasing\n            transforms.ToTensor(),\n            transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225]\n            ),\n            transforms.RandomErasing(p=0.1, scale=(0.02, 0.1)),\n        ])\n    else:\n        return transforms.Compose([\n            transforms.Resize((config.img_size, config.img_size)),\n            transforms.ToTensor(),\n            transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225]\n            )\n        ])\n\n\n# ============================================================\n# IMPROVED: Model with Deeper Classifier\n# ============================================================\n\nclass StateClassifier(nn.Module):\n    def __init__(self, model_name: str, num_classes: int = 4, pretrained: bool = True):\n        super().__init__()\n        \n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg'\n        )\n        \n        # Freeze backbone\n        for param in self.backbone.parameters():\n            param.requires_grad = False\n        \n        with torch.no_grad():\n            dummy = torch.randn(1, 3, config.img_size, config.img_size)\n            feat_dim = self.backbone(dummy).shape[-1]\n        \n        # IMPROVED: Deeper classifier head\n        self.classifier = nn.Sequential(\n            nn.Linear(feat_dim, 512),\n            nn.ReLU(),\n            nn.Dropout(config.dropout),\n            nn.Linear(512, 128),\n            nn.ReLU(),\n            nn.Dropout(config.dropout),\n            nn.Linear(128, num_classes)\n        )\n        \n        print(f\"Model initialized: backbone={model_name}, feat_dim={feat_dim}\")\n    \n    def forward(self, x):\n        with torch.no_grad():\n            features = self.backbone(x)\n        \n        logits = self.classifier(features)\n        return logits\n    \n    def predict_proba(self, x):\n        logits = self.forward(x)\n        return F.softmax(logits, dim=-1)\n\n\n# ============================================================\n# NEW: Focal Loss with Label Smoothing\n# ============================================================\n\nclass FocalLoss(nn.Module):\n    \"\"\"\n    Focal Loss for addressing class imbalance\n    FL = -alpha * (1 - p)^gamma * log(p)\n    \"\"\"\n    def __init__(self, alpha=None, gamma=2.0, label_smoothing=0.0, reduction='mean'):\n        super().__init__()\n        self.alpha = alpha  # Class weights\n        self.gamma = gamma\n        self.label_smoothing = label_smoothing\n        self.reduction = reduction\n    \n    def forward(self, inputs, targets):\n        # Apply label smoothing\n        num_classes = inputs.size(-1)\n        if self.label_smoothing > 0:\n            with torch.no_grad():\n                smoothed_targets = torch.zeros_like(inputs)\n                smoothed_targets.fill_(self.label_smoothing / (num_classes - 1))\n                smoothed_targets.scatter_(1, targets.unsqueeze(1), 1.0 - self.label_smoothing)\n        \n        # Compute cross entropy with log_softmax\n        log_probs = F.log_softmax(inputs, dim=-1)\n        probs = torch.exp(log_probs)\n        \n        if self.label_smoothing > 0:\n            ce_loss = -torch.sum(smoothed_targets * log_probs, dim=-1)\n            pt = torch.sum(smoothed_targets * probs, dim=-1)\n        else:\n            ce_loss = F.nll_loss(log_probs, targets, reduction='none')\n            pt = probs.gather(1, targets.unsqueeze(1)).squeeze(1)\n        \n        # Focal weight\n        focal_weight = (1 - pt) ** self.gamma\n        focal_loss = focal_weight * ce_loss\n        \n        # Apply class weights\n        if self.alpha is not None:\n            alpha_weight = self.alpha.gather(0, targets)\n            focal_loss = alpha_weight * focal_loss\n        \n        if self.reduction == 'mean':\n            return focal_loss.mean()\n        elif self.reduction == 'sum':\n            return focal_loss.sum()\n        return focal_loss\n\n\n# ============================================================\n# Training Utilities\n# ============================================================\n\nclass WarmupCosineScheduler:\n    def __init__(self, optimizer, warmup_epochs: int, total_epochs: int, \n                 base_lr: float, min_lr: float = 1e-6):\n        self.optimizer = optimizer\n        self.warmup_epochs = warmup_epochs\n        self.total_epochs = total_epochs\n        self.base_lr = base_lr\n        self.min_lr = min_lr\n        self.current_epoch = 0\n    \n    def step(self):\n        self.current_epoch += 1\n        if self.current_epoch <= self.warmup_epochs:\n            lr = self.base_lr * (self.current_epoch / self.warmup_epochs)\n        else:\n            progress = (self.current_epoch - self.warmup_epochs) / (self.total_epochs - self.warmup_epochs)\n            lr = self.min_lr + 0.5 * (self.base_lr - self.min_lr) * (1 + np.cos(np.pi * progress))\n        \n        for param_group in self.optimizer.param_groups:\n            param_group['lr'] = lr\n        \n        return lr\n\n\ndef compute_class_weights(df: pd.DataFrame) -> torch.Tensor:\n    state_counts = df['State'].value_counts()\n    total = len(df)\n    weights = []\n    \n    for state in ['NSW', 'Vic', 'Tas', 'WA']:\n        if state in state_counts:\n            # Balanced class weighting with cap for WA\n            weight = total / (len(state_counts) * state_counts[state])\n            weight = min(weight, 2.0)  # Cap at 2.0 to prevent over-prediction\n        else:\n            weight = 1.0\n        weights.append(weight)\n    \n    return torch.tensor(weights, dtype=torch.float32)\n\n\n# ============================================================\n# Training and Validation Functions\n# ============================================================\n\ndef train_one_epoch(model, train_loader, criterion, optimizer, scaler, device):\n    model.train()\n    \n    total_loss = 0\n    all_preds = []\n    all_labels = []\n    \n    for images, labels in train_loader:\n        images = images.to(device)\n        labels = labels.to(device)\n        \n        optimizer.zero_grad()\n        \n        with autocast(enabled=config.use_amp):\n            logits = model(images)\n            loss = criterion(logits, labels)\n        \n        scaler.scale(loss).backward()\n        scaler.unscale_(optimizer)\n        torch.nn.utils.clip_grad_norm_(model.parameters(), config.gradient_clip)\n        scaler.step(optimizer)\n        scaler.update()\n        \n        total_loss += loss.item()\n        \n        preds = logits.argmax(dim=1).cpu().numpy()\n        all_preds.extend(preds)\n        all_labels.extend(labels.cpu().numpy())\n    \n    accuracy = accuracy_score(all_labels, all_preds)\n    return total_loss / len(train_loader), accuracy\n\n\ndef validate(model, val_loader, criterion, device):\n    model.eval()\n    \n    total_loss = 0\n    all_preds = []\n    all_labels = []\n    all_probs = []\n    \n    with torch.no_grad():\n        for images, labels in val_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            with autocast(enabled=config.use_amp):\n                logits = model(images)\n                loss = criterion(logits, labels)\n            \n            total_loss += loss.item()\n            \n            probs = F.softmax(logits, dim=1)\n            preds = logits.argmax(dim=1)\n            \n            all_preds.extend(preds.cpu().numpy())\n            all_labels.extend(labels.cpu().numpy())\n            all_probs.extend(probs.cpu().numpy())\n    \n    accuracy = accuracy_score(all_labels, all_preds)\n    return total_loss / len(val_loader), accuracy, np.array(all_preds), np.array(all_labels)\n\n\ndef train_fold(fold: int, train_df: pd.DataFrame, val_df: pd.DataFrame, \n               class_weights: torch.Tensor):\n    print(f\"\\n{'='*60}\")\n    print(f\"Training Fold {fold + 1}/{config.n_folds}\")\n    print(f\"{'='*60}\")\n    print(f\"Train: {len(train_df)}, Val: {len(val_df)}\")\n    \n    device = torch.device(config.device)\n    \n    model = StateClassifier(config.model_name, config.num_classes).to(device)\n    \n    train_transform = get_transforms(is_train=True)\n    val_transform = get_transforms(is_train=False)\n    \n    train_dataset = StateClassificationDataset(train_df, train_transform, is_train=True)\n    val_dataset = StateClassificationDataset(val_df, val_transform, is_train=False)\n    \n    train_loader = DataLoader(train_dataset, batch_size=config.batch_size,\n                              shuffle=True, num_workers=config.num_workers, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=config.batch_size,\n                            shuffle=False, num_workers=config.num_workers, pin_memory=True)\n    \n    # IMPROVED: Focal Loss with Label Smoothing\n    criterion = FocalLoss(\n        alpha=class_weights.to(device),\n        gamma=config.focal_gamma,\n        label_smoothing=config.label_smoothing\n    )\n    \n    optimizer = torch.optim.AdamW(\n        model.classifier.parameters(),\n        lr=config.learning_rate,\n        weight_decay=config.weight_decay\n    )\n    \n    scheduler = WarmupCosineScheduler(\n        optimizer, config.warmup_epochs, config.epochs, config.learning_rate\n    )\n    \n    scaler = GradScaler(enabled=config.use_amp)\n    \n    best_acc = 0\n    best_model_state = None\n    patience = 0\n    max_patience = 10\n    \n    for epoch in range(config.epochs):\n        lr = scheduler.step()\n        \n        train_loss, train_acc = train_one_epoch(\n            model, train_loader, criterion, optimizer, scaler, device\n        )\n        \n        val_loss, val_acc, preds, labels = validate(model, val_loader, criterion, device)\n        \n        print(f\"Epoch {epoch+1:2d}/{config.epochs} | \"\n              f\"LR: {lr:.2e} | \"\n              f\"Train Loss: {train_loss:.4f}, Acc: {train_acc:.4f} | \"\n              f\"Val Loss: {val_loss:.4f}, Acc: {val_acc:.4f}\")\n        \n        if val_acc > best_acc:\n            best_acc = val_acc\n            best_model_state = model.state_dict().copy()\n            patience = 0\n            print(f\"  → New best accuracy: {best_acc:.4f}\")\n        else:\n            patience += 1\n        \n        # Early stopping\n        if patience >= max_patience:\n            print(f\"Early stopping at epoch {epoch + 1}\")\n            break\n    \n    model_path = os.path.join(config.output_dir, f\"state_classifier_fold{fold}.pth\")\n    torch.save(best_model_state, model_path)\n    print(f\"Saved: {model_path}\")\n    \n    model.load_state_dict(best_model_state)\n    _, final_acc, final_preds, final_labels = validate(model, val_loader, criterion, device)\n    \n    print(f\"\\nFold {fold + 1} Classification Report:\")\n    print(classification_report(final_labels, final_preds, \n                                target_names=['NSW', 'Vic', 'Tas', 'WA']))\n    \n    return best_acc, final_preds, final_labels\n\n\ndef main():\n    set_seed(config.seed)\n    \n    print(\"=\"*60)\n    print(\"exp03: State Classification Model (IMPROVED)\")\n    print(\"=\"*60)\n    print(\"Improvements:\")\n    print(f\"  1. Focal Loss (gamma={config.focal_gamma})\")\n    print(f\"  2. Label Smoothing: {config.label_smoothing}\")\n    print(f\"  3. Enhanced augmentation + RandomErasing\")\n    print(f\"  4. Deeper classifier (768→512→128→4)\")\n    print(f\"  5. Epochs: {config.epochs}, LR: {config.learning_rate}\")\n    print(\"=\"*60)\n    \n    config.model_name, config.img_size = get_best_available_model()\n    \n    print(\"\\nLoading training data...\")\n    train_df = pd.read_csv(config.train_path)\n    \n    df = train_df.drop_duplicates('image_path')[['image_path', 'State']].reset_index(drop=True)\n    \n    print(f\"Total images: {len(df)}\")\n    print(f\"State distribution:\\n{df['State'].value_counts()}\")\n    \n    class_weights = compute_class_weights(df)\n    print(f\"\\nClass weights: {dict(zip(['NSW', 'Vic', 'Tas', 'WA'], class_weights.numpy()))}\")\n    \n    device = torch.device(config.device)\n    print(f\"Device: {device}\")\n    \n    skf = StratifiedKFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n    fold_accuracies = []\n    all_preds = []\n    all_labels = []\n    \n    for fold, (train_idx, val_idx) in enumerate(skf.split(df, df['State'])):\n        train_df_fold = df.iloc[train_idx]\n        val_df_fold = df.iloc[val_idx]\n        \n        acc, preds, labels = train_fold(fold, train_df_fold, val_df_fold, class_weights)\n        fold_accuracies.append(acc)\n        all_preds.extend(preds)\n        all_labels.extend(labels)\n    \n    print(\"\\n\" + \"=\"*60)\n    print(\"TRAINING COMPLETE\")\n    print(\"=\"*60)\n    \n    mean_acc = np.mean(fold_accuracies)\n    std_acc = np.std(fold_accuracies)\n    print(f\"\\nCV Accuracy: {mean_acc:.4f} ± {std_acc:.4f}\")\n    \n    print(\"\\nOverall Confusion Matrix:\")\n    cm = confusion_matrix(all_labels, all_preds)\n    print(pd.DataFrame(cm, \n                       index=['NSW', 'Vic', 'Tas', 'WA'],\n                       columns=['Pred_NSW', 'Pred_Vic', 'Pred_Tas', 'Pred_WA']))\n    \n    print(\"\\nOverall Classification Report:\")\n    print(classification_report(all_labels, all_preds, target_names=['NSW', 'Vic', 'Tas', 'WA']))\n\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-04T12:36:59.245212Z","iopub.execute_input":"2026-01-04T12:36:59.245449Z"}},"outputs":[],"execution_count":null}]}