{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install transformers","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:32.942117Z","iopub.execute_input":"2025-10-30T12:10:32.942326Z","iopub.status.idle":"2025-10-30T12:10:36.371120Z","shell.execute_reply.started":"2025-10-30T12:10:32.942309Z","shell.execute_reply":"2025-10-30T12:10:36.370367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torch.amp import autocast, GradScaler\nfrom torchvision import transforms\nfrom PIL import Image\nimport os\nimport pandas as pd\nimport numpy as np\nfrom transformers import AutoModel\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:36.372132Z","iopub.execute_input":"2025-10-30T12:10:36.372464Z","iopub.status.idle":"2025-10-30T12:10:41.577932Z","shell.execute_reply.started":"2025-10-30T12:10:36.372430Z","shell.execute_reply":"2025-10-30T12:10:41.577353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.578562Z","iopub.execute_input":"2025-10-30T12:10:41.578897Z","iopub.status.idle":"2025-10-30T12:10:41.591012Z","shell.execute_reply.started":"2025-10-30T12:10:41.578878Z","shell.execute_reply":"2025-10-30T12:10:41.589967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.592936Z","iopub.execute_input":"2025-10-30T12:10:41.593559Z","iopub.status.idle":"2025-10-30T12:10:41.615727Z","shell.execute_reply.started":"2025-10-30T12:10:41.593540Z","shell.execute_reply":"2025-10-30T12:10:41.614978Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"base_id\"] = df[\"sample_id\"].str.split(\"__\").str[0]\n\ndf_pivot = df.pivot_table(\n    index=[\"base_id\", \"image_path\", \"Sampling_Date\", \"State\", \"Species\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\"],\n    columns=\"target_name\",\n    values=\"target\"\n).reset_index()\n\ndf_pivot.columns.name = None\n\ndf_pivot.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.616381Z","iopub.execute_input":"2025-10-30T12:10:41.616577Z","iopub.status.idle":"2025-10-30T12:10:41.652083Z","shell.execute_reply.started":"2025-10-30T12:10:41.616562Z","shell.execute_reply":"2025-10-30T12:10:41.651394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_pivot.to_numpy()[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.652836Z","iopub.execute_input":"2025-10-30T12:10:41.653180Z","iopub.status.idle":"2025-10-30T12:10:41.658015Z","shell.execute_reply.started":"2025-10-30T12:10:41.653153Z","shell.execute_reply":"2025-10-30T12:10:41.657476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CSIRO(Dataset):\n    def __init__(self, df, transform, base_path=\"/kaggle/input/csiro-biomass\"):\n        self.df = df\n        self.transform = transform\n        self.base_path = base_path\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self,idx):\n        row = self.df[idx]\n        image_path = os.path.join(self.base_path, row[1])\n        img = Image.open(image_path)\n        tensor_img = self.transform(img)\n        \n        dry_clover = row[-5]\n        dry_dead = row[-4]\n        dry_green = row[-3]\n        dry_total = row[-2]\n        gdm = row[-1]\n        \n        label = torch.tensor([\n            dry_clover,\n            dry_dead,\n            dry_green,\n            dry_total,\n            gdm\n        ]).log1p()\n\n        return tensor_img, label.to(dtype=torch.float32)\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.658694Z","iopub.execute_input":"2025-10-30T12:10:41.659062Z","iopub.status.idle":"2025-10-30T12:10:41.673910Z","shell.execute_reply.started":"2025-10-30T12:10:41.659042Z","shell.execute_reply":"2025-10-30T12:10:41.673402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.Resize((256, 256)),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomVerticalFlip(p=0.5),\n    transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.05),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406],\n                [0.229, 0.224, 0.225])\n])\n\ntest_transform = transforms.Compose([\n    transforms.Resize((256, 256)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406],\n                [0.229, 0.224, 0.225])\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.674630Z","iopub.execute_input":"2025-10-30T12:10:41.674836Z","iopub.status.idle":"2025-10-30T12:10:41.697476Z","shell.execute_reply.started":"2025-10-30T12:10:41.674811Z","shell.execute_reply":"2025-10-30T12:10:41.696906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, test_df = train_test_split(df_pivot, test_size=0.05, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.698174Z","iopub.execute_input":"2025-10-30T12:10:41.698475Z","iopub.status.idle":"2025-10-30T12:10:41.716480Z","shell.execute_reply.started":"2025-10-30T12:10:41.698449Z","shell.execute_reply":"2025-10-30T12:10:41.715930Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_df.to_numpy()), len(test_df.to_numpy())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.717138Z","iopub.execute_input":"2025-10-30T12:10:41.717487Z","iopub.status.idle":"2025-10-30T12:10:41.734183Z","shell.execute_reply.started":"2025-10-30T12:10:41.717464Z","shell.execute_reply":"2025-10-30T12:10:41.733608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = CSIRO(df=train_df.to_numpy(), transform=train_transform)\ntest_dataset = CSIRO(df=test_df.to_numpy(), transform=test_transform)\nlen(train_dataset), len(test_dataset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.734815Z","iopub.execute_input":"2025-10-30T12:10:41.735262Z","iopub.status.idle":"2025-10-30T12:10:41.751078Z","shell.execute_reply.started":"2025-10-30T12:10:41.735224Z","shell.execute_reply":"2025-10-30T12:10:41.750208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_workers = 4\nbatch_size = 4\nprefetch_factor = 8","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.751796Z","iopub.execute_input":"2025-10-30T12:10:41.752027Z","iopub.status.idle":"2025-10-30T12:10:41.766437Z","shell.execute_reply.started":"2025-10-30T12:10:41.752002Z","shell.execute_reply":"2025-10-30T12:10:41.765695Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataloader = DataLoader(dataset=train_dataset, \n                              shuffle=True,\n                              batch_size=batch_size, \n                              prefetch_factor=prefetch_factor, \n                              num_workers=num_workers)\n\ntest_dataloader = DataLoader(dataset=test_dataset, \n                              shuffle=True,\n                              batch_size=batch_size, \n                              prefetch_factor=prefetch_factor, \n                              num_workers=num_workers)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.768435Z","iopub.execute_input":"2025-10-30T12:10:41.768633Z","iopub.status.idle":"2025-10-30T12:10:41.784583Z","shell.execute_reply.started":"2025-10-30T12:10:41.768618Z","shell.execute_reply":"2025-10-30T12:10:41.784053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_r2_score(y_true, y_pred):\n    \n    # Convert from log1p space back to original space\n    y_true_orig = torch.expm1(y_true)\n    y_pred_orig = torch.expm1(y_pred)\n    \n    # Define weights for each dimension\n    weights = torch.tensor([\n        0.1,  # Dry_Clover_g (index 0)\n        0.1,  # Dry_Dead_g (index 1)\n        0.1,  # Dry_Green_g (index 2)\n        0.5,  # Dry_Total_g (index 3)\n        0.2   # GDM_g (index 4)\n    ])\n    \n    # Dimension names for reference\n    dim_names = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n    \n    # Calculate R² for each dimension\n    r2_scores = []\n    individual_r2 = {}\n    \n    for i in range(5):\n        y_true_dim = y_true_orig[:, i]\n        y_pred_dim = y_pred_orig[:, i]\n        \n        # Calculate SS_res (Residual Sum of Squares)\n        ss_res = torch.sum((y_true_dim - y_pred_dim) ** 2)\n        \n        # Calculate SS_tot (Total Sum of Squares)\n        y_mean = torch.mean(y_true_dim)\n        ss_tot = torch.sum((y_true_dim - y_mean) ** 2)\n        \n        # Calculate R²\n        r2 = 1 - (ss_res / ss_tot)\n        r2_scores.append(r2)\n        individual_r2[dim_names[i]] = r2.item()\n    \n    # Convert to tensor\n    r2_scores = torch.stack(r2_scores)\n    \n    # Calculate weighted final score\n    final_score = torch.sum(weights * r2_scores)\n    \n    return final_score.item(), individual_r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.785157Z","iopub.execute_input":"2025-10-30T12:10:41.785359Z","iopub.status.idle":"2025-10-30T12:10:41.806623Z","shell.execute_reply.started":"2025-10-30T12:10:41.785345Z","shell.execute_reply":"2025-10-30T12:10:41.805785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = GradScaler('cuda')\n\ndef train_step(model, optimizer, dataloader, loss_fn, epoch, device=\"cuda\"):\n    model.train()\n    train_loss = 0\n    all_pred = []\n    all_true = []\n\n    progress_bar = tqdm(dataloader, desc=f\"Epoch {epoch}\", dynamic_ncols=True)\n    \n    for batch_idx, (img, label) in enumerate(progress_bar):\n        img, label = img.to(device), label.to(device)\n\n        with autocast('cuda'):\n            y_pred = model(img)\n\n            all_pred.append(y_pred.detach().cpu())\n            all_true.append(label.detach().cpu())\n\n            loss = loss_fn(y_pred, label)\n            batch_loss = loss.item()\n            train_loss += batch_loss\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n\n        avg_loss = train_loss / (batch_idx + 1)\n        progress_bar.set_postfix(\n            batch_loss=f\"{batch_loss:.4f}\",\n            avg_loss=f\"{avg_loss:.4f}\",\n        )\n\n        del img, label, loss, y_pred\n\n    all_pred = torch.cat(all_pred, dim=0)\n    all_true = torch.cat(all_true, dim=0)\n\n    r2_score, individual_r2 = calculate_r2_score(all_true, all_pred)\n\n    avg_loss = train_loss / len(dataloader)\n\n    print(f\"Epoch {epoch} | Train Loss: {avg_loss:.4f} | Train R2: {r2_score:.2f}\")\n    return avg_loss, r2_score\n\n\n\ndef test_step(model, dataloader, loss_fn, epoch, scheduler, device=\"cuda\"):\n    model.eval()\n    test_loss = 0\n    all_pred = []\n    all_true = []\n\n    progress_bar = tqdm(dataloader, desc=f\"Epoch {epoch} (Eval)\", dynamic_ncols=True)\n\n    with torch.no_grad():\n        for batch_idx, (img, label) in enumerate(progress_bar):\n            img, label = img.to(device), label.to(device)\n\n            with autocast('cuda'):\n                y_pred = model(img)\n\n                all_pred.append(y_pred.detach().cpu())\n                all_true.append(label.detach().cpu())\n\n                loss = loss_fn(y_pred, label)\n                batch_loss = loss.item()\n                test_loss += batch_loss\n\n            avg_loss = train_loss / (batch_idx + 1)\n            progress_bar.set_postfix(\n                batch_loss=f\"{batch_loss:.4f}\",\n                avg_loss=f\"{avg_loss:.4f}\",\n            )\n\n            del img, label, loss, y_pred\n\n    all_pred = torch.cat(all_pred, dim=0)\n    all_true = torch.cat(all_true, dim=0)\n\n    r2_score, individual_r2 = calculate_r2_score(all_true, all_pred)\n\n    avg_loss = test_loss / len(dataloader)\n\n    if isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau):\n        scheduler.step(avg_loss)\n\n    print(f\"Epoch {epoch} | Test Loss: {avg_loss:.4f} | Test R2: {r2_score:.2f}\")\n    return avg_loss, r2_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.807359Z","iopub.execute_input":"2025-10-30T12:10:41.807600Z","iopub.status.idle":"2025-10-30T12:10:41.875929Z","shell.execute_reply.started":"2025-10-30T12:10:41.807578Z","shell.execute_reply":"2025-10-30T12:10:41.875341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CSIRO_Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.vit = AutoModel.from_pretrained(\"apple/mobilevit-x-small\")\n        self.fc = nn.Sequential(\n            nn.Linear(384, 128),\n            nn.LayerNorm(128),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n\n            nn.Linear(128, 5),\n        )\n\n    def forward(self, img):\n        encoded_output = self.vit(img).pooler_output\n        logits = self.fc(encoded_output)\n        return logits","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.876747Z","iopub.execute_input":"2025-10-30T12:10:41.876951Z","iopub.status.idle":"2025-10-30T12:10:41.897836Z","shell.execute_reply.started":"2025-10-30T12:10:41.876934Z","shell.execute_reply":"2025-10-30T12:10:41.897280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seeds = [n for n in range(1,4)]\nbest_models = []\n\nfor seed in seeds:\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    device = \"cuda\"\n    \n    model = CSIRO_Model().to(device)\n    \n    loss_fn = nn.HuberLoss()\n    optimizer = torch.optim.AdamW(params=model.parameters(),\n                                lr=3e-4,\n                                 weight_decay=1e-3)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer,\n        mode='min',\n        factor=0.5,\n        patience=2,\n        min_lr=1e-6,\n    )\n    \n    num_epochs = 20\n    best = 0.30\n    \n    start = 1\n    epochs = num_epochs + 1\n\n    print(f\"Training model no.:{seed}\")\n    \n    for epoch in range(start, epochs):\n        train_loss, train_r2 = train_step(\n            model=model,\n            optimizer=optimizer,\n            loss_fn=loss_fn,\n            epoch=epoch,\n            dataloader=train_dataloader\n        )\n    \n        test_loss, test_r2 = test_step(\n            model=model,\n            loss_fn=loss_fn,\n            epoch=epoch,\n            dataloader=test_dataloader,\n            scheduler=scheduler\n        )\n        \n        if test_r2 > best:\n            checkpoint = {\n                \"model_state_dict\" : model.state_dict(),\n                \"train_r2\" : train_r2,\n                \"test_r2\" : test_r2,\n                \"train_loss\" : train_loss,\n                \"test_loss\" : test_loss,\n                \"epoch\" : epoch,\n                \"optimizer\" : optimizer.state_dict(),\n                \"scheduler\" : scheduler.state_dict()\n            }\n            torch.save(obj=checkpoint, f=f\"CSIRO_{epoch}_epochs_{test_r2:.2f}_r2_{seed}.pth\")\n            best_model = f\"CSIRO_{epoch}_epochs_{test_r2:.2f}_r2_{seed}.pth\"\n            best = test_r2\n            del checkpoint\n    best_models.append(best_model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:10:41.898394Z","iopub.execute_input":"2025-10-30T12:10:41.898553Z","iopub.status.idle":"2025-10-30T12:16:16.206338Z","shell.execute_reply.started":"2025-10-30T12:10:41.898540Z","shell.execute_reply":"2025-10-30T12:16:16.205328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(best_models)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:16:16.207626Z","iopub.execute_input":"2025-10-30T12:16:16.208933Z","iopub.status.idle":"2025-10-30T12:16:16.214048Z","shell.execute_reply.started":"2025-10-30T12:16:16.208908Z","shell.execute_reply":"2025-10-30T12:16:16.213348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")\nimg = Image.open(os.path.join('/kaggle/input/csiro-biomass', df['image_path'][0]))\nimg = test_transform(img).to(device).unsqueeze(0)\npreds = []\nfor model_path in best_models:\n    model = CSIRO_Model().to(device)\n    checkpoint = torch.load(f=os.path.join('/kaggle/working/', model_path))\n    new_state_dict = checkpoint[\"model_state_dict\"]\n    model.load_state_dict(new_state_dict, strict=True)\n    print(\"Loaded checkpoint weights!\")\n    del checkpoint, new_state_dict\n\n    model.eval()\n    with torch.inference_mode():\n        pred = model(img)\n    preds.append(pred.expm1())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:16:16.214722Z","iopub.execute_input":"2025-10-30T12:16:16.214895Z","iopub.status.idle":"2025-10-30T12:16:17.846510Z","shell.execute_reply.started":"2025-10-30T12:16:16.214881Z","shell.execute_reply":"2025-10-30T12:16:17.845787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = torch.cat(preds, dim=0).mean(0)\nsubmission = pd.DataFrame({\n    \"sample_id\": df[\"sample_id\"],\n    \"target\": preds.cpu()\n})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-30T12:16:17.847213Z","iopub.execute_input":"2025-10-30T12:16:17.847488Z","iopub.status.idle":"2025-10-30T12:16:17.855341Z","shell.execute_reply.started":"2025-10-30T12:16:17.847470Z","shell.execute_reply":"2025-10-30T12:16:17.854639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}