{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport argparse\nimport random\nfrom pathlib import Path\nimport math\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:29:40.187101Z","iopub.execute_input":"2025-11-12T08:29:40.187461Z","iopub.status.idle":"2025-11-12T08:29:40.194208Z","shell.execute_reply.started":"2025-11-12T08:29:40.187438Z","shell.execute_reply":"2025-11-12T08:29:40.192754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\n\ndef ensure_dir(d):\n    os.makedirs(d, exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:29:42.476534Z","iopub.execute_input":"2025-11-12T08:29:42.477485Z","iopub.status.idle":"2025-11-12T08:29:42.483203Z","shell.execute_reply.started":"2025-11-12T08:29:42.477447Z","shell.execute_reply":"2025-11-12T08:29:42.482018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TARGETS = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\n\ndef load_and_pivot_train(train_csv):\n    \"\"\"Load train CSV and pivot to one row per image with 5 target columns.\n    Accepts two common formats:\n      (A) long format: each row is (sample_id, image_path, target_name, value)\n      (B) wide format: each row is (image, Dry_Green_g, Dry_Dead_g, ...)\n    Returns DataFrame with columns: image_path + TARGETS\n    \"\"\"\n    df = pd.read_csv(train_csv)\n    # If dataframe already contains all target columns, return subset\n    if all(t in df.columns for t in TARGETS):\n        # ensure image_path column exists\n        if \"image_path\" not in df.columns:\n            raise ValueError(\"train.csv contains target columns but no 'image_path' column found\")\n        # keep only necessary cols\n        return df[[\"image_path\"] + TARGETS].copy()\n\n    # otherwise expect long format\n    long_cols = set(df.columns)\n    needed = {\"sample_id\", \"image_path\", \"target_name\"} | {\"value\"}\n    if {\"sample_id\", \"image_path\", \"target_name\"}.issubset(long_cols):\n        if \"value\" in long_cols:\n            val_col = \"value\"\n        elif \"target\" in long_cols:\n            val_col = \"target\"\n        else:\n            # try to guess the numeric column\n            numeric_cols = [c for c in df.columns if df[c].dtype != object and c not in {\"sample_id\"}]\n            if len(numeric_cols) == 1:\n                val_col = numeric_cols[0]\n            else:\n                raise ValueError(\"Could not find numeric value column in train.csv\")\n\n        # pivot\n        pivot = df.pivot_table(index=\"image_path\", columns=\"target_name\", values=val_col, aggfunc=\"first\")\n        pivot = pivot.reset_index()\n        # ensure all targets exist, fill missing with NaN\n        for t in TARGETS:\n            if t not in pivot.columns:\n                pivot[t] = np.nan\n        return pivot[[\"image_path\"] + TARGETS]\n    else:\n        raise ValueError(\"Unrecognized train.csv format. Expected either wide-format with target columns or long-format with target_name column.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:29:44.159835Z","iopub.execute_input":"2025-11-12T08:29:44.160199Z","iopub.status.idle":"2025-11-12T08:29:44.171599Z","shell.execute_reply.started":"2025-11-12T08:29:44.160176Z","shell.execute_reply":"2025-11-12T08:29:44.170477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PastureDataset(Dataset):\n    def __init__(self, df, img_root, transforms=None, is_train=True):\n        self.df = df.reset_index(drop=True)\n        self.img_root = Path(img_root)\n        self.transforms = transforms\n        self.is_train = is_train\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = self.img_root / row['image_path']\n        # some CSVs contain full paths; handle either\n        if not img_path.exists():\n            # try raw path\n            img_path = Path(row['image_path'])\n        img = Image.open(img_path).convert('RGB')\n        if self.transforms:\n            img = self.transforms(img)\n        if self.is_train:\n            targets = torch.tensor([row[t] if not pd.isna(row[t]) else 0.0 for t in TARGETS], dtype=torch.float32)\n            return img, targets\n        else:\n            return img, row['image_path']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:29:47.157059Z","iopub.execute_input":"2025-11-12T08:29:47.157443Z","iopub.status.idle":"2025-11-12T08:29:47.166703Z","shell.execute_reply.started":"2025-11-12T08:29:47.157419Z","shell.execute_reply":"2025-11-12T08:29:47.165129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_transforms(img_size=224, is_train=True):\n    if is_train:\n        return transforms.Compose([\n            transforms.Resize((img_size, img_size)),\n            transforms.RandomHorizontalFlip(p=0.5),\n            transforms.RandomVerticalFlip(p=0.2),\n            transforms.RandomRotation(10),\n            transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.02),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        ])\n    else:\n        return transforms.Compose([\n            transforms.Resize((img_size, img_size)),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        ])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:29:48.951996Z","iopub.execute_input":"2025-11-12T08:29:48.952986Z","iopub.status.idle":"2025-11-12T08:29:48.960703Z","shell.execute_reply.started":"2025-11-12T08:29:48.952949Z","shell.execute_reply":"2025-11-12T08:29:48.958909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_model(model_name='resnet50', pretrained=True, n_outputs=5):\n    if model_name == 'resnet50':\n        model = models.resnet50(pretrained=pretrained)\n        in_feats = model.fc.in_features\n        model.fc = nn.Linear(in_feats, n_outputs)\n    elif model_name == 'resnet18':\n        model = models.resnet18(pretrained=pretrained)\n        in_feats = model.fc.in_features\n        model.fc = nn.Linear(in_feats, n_outputs)\n    else:\n        raise ValueError('Unsupported model_name')\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:29:51.224178Z","iopub.execute_input":"2025-11-12T08:29:51.225115Z","iopub.status.idle":"2025-11-12T08:29:51.231400Z","shell.execute_reply.started":"2025-11-12T08:29:51.225086Z","shell.execute_reply":"2025-11-12T08:29:51.230044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, loader, optimizer, device, loss_fn, scaler=None):\n    model.train()\n    running_loss = 0.0\n    n = 0\n    for imgs, targets in tqdm(loader, desc='train', leave=False):\n        imgs = imgs.to(device)\n        targets = targets.to(device)\n        optimizer.zero_grad()\n        if scaler is not None:\n            with torch.cuda.amp.autocast():\n                preds = model(imgs)\n                loss = loss_fn(preds, targets)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n        else:\n            preds = model(imgs)\n            loss = loss_fn(preds, targets)\n            loss.backward()\n            optimizer.step()\n        running_loss += loss.item() * imgs.size(0)\n        n += imgs.size(0)\n    return running_loss / max(n, 1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate(model, loader, device, loss_fn):\n    model.eval()\n    running_loss = 0.0\n    preds_all = []\n    targs_all = []\n    with torch.no_grad():\n        for imgs, targets in tqdm(loader, desc='val', leave=False):\n            imgs = imgs.to(device)\n            targets = targets.to(device)\n            preds = model(imgs)\n            loss = loss_fn(preds, targets)\n            running_loss += loss.item() * imgs.size(0)\n            preds_all.append(preds.detach().cpu().numpy())\n            targs_all.append(targets.detach().cpu().numpy())\n    if len(preds_all) > 0:\n        preds_all = np.vstack(preds_all)\n        targs_all = np.vstack(targs_all)\n        mae = mean_absolute_error(targs_all, preds_all)\n    else:\n        mae = float('nan')\n    return running_loss / max(len(loader.dataset), 1), mae\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:29:55.541325Z","iopub.execute_input":"2025-11-12T08:29:55.541638Z","iopub.status.idle":"2025-11-12T08:29:55.549957Z","shell.execute_reply.started":"2025-11-12T08:29:55.541617Z","shell.execute_reply":"2025-11-12T08:29:55.548809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef predict_on_images(model, loader, device):\n    model.eval()\n    results = {}\n    with torch.no_grad():\n        for imgs, img_paths in tqdm(loader, desc='predict'):\n            imgs = imgs.to(device)\n            preds = model(imgs).detach().cpu().numpy()\n            for p, img_path in zip(preds, img_paths):\n                results[img_path] = p.tolist()\n    return results\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:29:58.308386Z","iopub.execute_input":"2025-11-12T08:29:58.308754Z","iopub.status.idle":"2025-11-12T08:29:58.315534Z","shell.execute_reply.started":"2025-11-12T08:29:58.308730Z","shell.execute_reply":"2025-11-12T08:29:58.314477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train(args):\n    seed_everything(args.seed)\n    df = load_and_pivot_train(args.train_csv)\n    print(f\"Loaded training data: {len(df)} images\")\n\n    # Simple split\n    train_df, val_df = train_test_split(df, test_size=args.val_size, random_state=args.seed)\n    print(f\"Train {len(train_df)} | Val {len(val_df)}\")\n\n    train_ds = PastureDataset(train_df, img_root=args.img_root, transforms=get_transforms(args.img_size, True), is_train=True)\n    val_ds = PastureDataset(val_df, img_root=args.img_root, transforms=get_transforms(args.img_size, False), is_train=True)\n\n    train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, pin_memory=True)\n    val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True)\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = build_model(model_name=args.model, pretrained=True, n_outputs=len(TARGETS))\n    model = model.to(device)\n\n    loss_fn = nn.MSELoss()\n    optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr)\n\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3)\n    scaler = torch.cuda.amp.GradScaler() if args.use_amp and torch.cuda.is_available() else None\n\n    best_mae = float('inf')\n    ensure_dir(args.output_dir)\n    current_patience = 0\n\n    for epoch in range(1, args.epochs + 1):\n        print(f\"Epoch {epoch}/{args.epochs}\")\n        train_loss = train_one_epoch(model, train_loader, optimizer, device, loss_fn, scaler)\n        val_loss, val_mae = validate(model, val_loader, device, loss_fn)\n        print(f\"Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val MAE: {val_mae:.4f}\")\n\n        scheduler.step(val_loss)\n\n        if val_mae < best_mae:\n            current_patience = 0\n            best_mae = val_mae\n            best_path = os.path.join(args.output_dir, 'best.pth')\n            torch.save({'model_state_dict': model.state_dict(), 'args': vars(args)}, best_path)\n            print(f\"Saved best model to {best_path}\")\n        else:\n            current_patience+=1\n        if current_patience > args.patience:\n            print('Early Stopped at epoch ' + str(epoch))\n            break\n\n    print(\"Training finished.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Args:\n    train_csv = '/kaggle/input/csiro-biomass/train.csv'\n    test_csv = '/kaggle/input/csiro-biomass/test.csv'\n    img_root = '/kaggle/input/csiro-biomass/'\n    output_result = '/kaggle/working/submission.csv'\n    output_dir = '/kaggle/working/'\n    checkpoint = '/kaggle/working/best.pth'\n    model = 'resnet50'\n    img_size = 224\n    batch_size = 16\n    epochs = 1000\n    patience = 50\n    lr = 1e-4\n    val_size = 0.2\n    seed = 42\n    num_workers = 4\n    use_amp = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:30:03.537262Z","iopub.execute_input":"2025-11-12T08:30:03.537583Z","iopub.status.idle":"2025-11-12T08:30:03.544062Z","shell.execute_reply.started":"2025-11-12T08:30:03.537563Z","shell.execute_reply":"2025-11-12T08:30:03.542313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"args = Args()\ntrain(args)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_inference(args):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = build_model(model_name=args.model, pretrained=False, n_outputs=len(TARGETS))\n    ck = torch.load(args.checkpoint, map_location='cpu')\n    model.load_state_dict(ck['model_state_dict'])\n    model = model.to(device)\n\n    # prepare test images (unique images)\n    test_df = pd.read_csv(args.test_csv_path)\n    # test.csv expected columns: sample_id, image_path, target_name\n    # get unique images in original order\n    unique_imgs = test_df['image_path'].drop_duplicates().reset_index(drop=True)\n    imgs_df = pd.DataFrame({'image_path': unique_imgs})\n\n    test_ds = PastureDataset(imgs_df, img_root=args.img_root, transforms=get_transforms(args.img_size, False), is_train=False)\n    test_loader = DataLoader(test_ds, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True)\n\n    preds_dict = predict_on_images(model, test_loader, device)\n\n    # Expand preds into rows matching test.csv\n    rows = []\n    for _, row in test_df.iterrows():\n        img = row['image_path']\n        sample_id = row['sample_id']\n        target_name = row['target_name']\n        if img in preds_dict:\n            pred_vector = preds_dict[img]\n            try:\n                t_idx = TARGETS.index(target_name)\n                pred_val = float(pred_vector[t_idx])\n            except Exception:\n                pred_val = float('nan')\n        else:\n            pred_val = float('nan')\n        rows.append({'sample_id': sample_id, 'target': pred_val})\n\n    submission = pd.DataFrame(rows)\n    submission.to_csv(args.output_result_path, index=False)\n    print(f\"Saved predictions to {args.output_result_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:30:07.950692Z","iopub.execute_input":"2025-11-12T08:30:07.951121Z","iopub.status.idle":"2025-11-12T08:30:07.962061Z","shell.execute_reply.started":"2025-11-12T08:30:07.951094Z","shell.execute_reply":"2025-11-12T08:30:07.960558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_csv_path = '/kaggle/input/csiro-biomass/test.csv'\noutput_result_path = '/kaggle/working/submission.csv'\nargs.test_csv_path = test_csv_path\nargs.output_result_path = output_result_path\nrun_inference(args)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-12T08:31:06.022821Z","iopub.execute_input":"2025-11-12T08:31:06.023268Z","iopub.status.idle":"2025-11-12T08:31:07.408015Z","shell.execute_reply.started":"2025-11-12T08:31:06.023229Z","shell.execute_reply":"2025-11-12T08:31:07.406583Z"}},"outputs":[],"execution_count":null}]}