{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":3721,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":2648,"modelId":310},{"sourceId":3729,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":2656,"modelId":312},{"sourceId":3732,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":2659,"modelId":312}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Score Improvement with PowerTransformer\n\nAfter implementing `PowerTransformer` as a normalizer, the model's score significantly improved from **0.28** to **0.51**. This indicates that normalizing the target variables using `PowerTransformer` had a positive impact on the model's performance.","metadata":{}},{"cell_type":"code","source":"import timm\nimport torch\nimport torch.nn as nn\nimport gc\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nimport torchvision.transforms as T\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint,EarlyStopping\nimport random\nimport os\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import PowerTransformer\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-06T13:20:32.574858Z","iopub.execute_input":"2025-12-06T13:20:32.575110Z","iopub.status.idle":"2025-12-06T13:20:32.587732Z","shell.execute_reply.started":"2025-12-06T13:20:32.575093Z","shell.execute_reply":"2025-12-06T13:20:32.587162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======== Seed ========\ndef seed_everything(seed: int = 42):\n    pl.seed_everything(seed, workers=True)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed) \n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nseed_everything(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T13:20:32.589260Z","iopub.execute_input":"2025-12-06T13:20:32.589486Z","iopub.status.idle":"2025-12-06T13:20:32.607435Z","shell.execute_reply.started":"2025-12-06T13:20:32.589466Z","shell.execute_reply":"2025-12-06T13:20:32.606843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======== Weighted R² ========\ndef weighted_r2_score(y_true: np.ndarray, y_pred: np.ndarray):\n    \"\"\"\n    y_true, y_pred: shape (N, 5)\n    \"\"\"\n    weights = np.array([0.1, 0.1, 0.1, 0.5, 0.2])\n    r2_scores = []\n    for i in range(5):\n        y_t = y_true[:, i]\n        y_p = y_pred[:, i]\n        ss_res = np.sum((y_t - y_p) ** 2)\n        ss_tot = np.sum((y_t - np.mean(y_t)) ** 2)\n        r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n        r2_scores.append(r2)\n    r2_scores = np.array(r2_scores)\n    weighted_r2 = np.sum(r2_scores * weights) / np.sum(weights)\n    return weighted_r2, r2_scores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T13:20:32.608154Z","iopub.execute_input":"2025-12-06T13:20:32.608373Z","iopub.status.idle":"2025-12-06T13:20:32.623091Z","shell.execute_reply.started":"2025-12-06T13:20:32.608349Z","shell.execute_reply":"2025-12-06T13:20:32.622485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======== Dataset ========\nclass ImageRegressionDataset(Dataset):\n    def __init__(self, df,  normalizer,target_cols,transform=None,):\n        self.df = df\n        self.transform = transform\n        self.normalizer = normalizer\n        self.target_cols = target_cols\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image = Image.open('/kaggle/input/csiro-biomass/' + row[\"image_path\"]).convert(\"RGB\")\n        if self.transform:\n            image = self.transform(image)\n        targets = torch.tensor([\n            row[\"Dry_Green_g\"],\n            row[\"Dry_Dead_g\"],\n            row[\"Dry_Clover_g\"],\n            row[\"Dry_Total_g\"],\n            row[\"GDM_g\"],\n        ], dtype=torch.float32)\n        return image, targets\n    \nclass ImageRegressionDatasetTest(Dataset):\n    def __init__(self, df, transform):\n        self.df = df\n        self.transform = transform\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_id = row[\"image_path\"].strip(\"test/.jpg\")\n        image = Image.open('/kaggle/input/csiro-biomass/' + row[\"image_path\"]).convert(\"RGB\")\n        if self.transform:\n            image = self.transform(image)\n        \n        return image , image_id\n        \n    def __len__(self):\n        return len(self.df)\n     ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T13:20:32.623835Z","iopub.execute_input":"2025-12-06T13:20:32.624087Z","iopub.status.idle":"2025-12-06T13:20:32.643745Z","shell.execute_reply.started":"2025-12-06T13:20:32.624070Z","shell.execute_reply":"2025-12-06T13:20:32.643104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======== Model ========\nclass MultiRegressionModel(pl.LightningModule):\n    def regression_head(self, in_features,dropout,out_features):\n        return nn.Sequential(\n            nn.Linear(in_features, 64),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(64, out_features)\n        )\n\n    def __init__(self,model_name,model_location, input_size=1000,  lr=1e-4):\n        super().__init__()\n        self.save_hyperparameters()\n        self.model = timm.create_model(model_name,num_classes=0)\n        ckpt = torch.load(model_location, map_location=\"cpu\")\n        self.model.load_state_dict(ckpt, strict=False)\n        self.in_features = self.model.num_features *2\n        self.heads = nn.ModuleDict({\n            \"Dry\": self.regression_head(self.in_features, 0.2,3),\n            \"GDM_g\": self.regression_head(self.in_features, 0.2,1),\n            \"Dry_Total_g\": self.regression_head(self.in_features, 0.2,1),\n        })\n        self.criterion = nn.SmoothL1Loss()\n        self.val_outputs = []\n\n    def forward(self, x):\n        \n        _, _, H, W = x.shape\n        left = x[:, :, :, :W // 2]\n        right = x[:, :, :, W // 2:]\n\n        feat_left = self.model(left)\n        feat_right = self.model(right)\n\n        features = torch.cat([feat_left, feat_right], dim=1)\n\n        outputs = [\n            self.heads[\"Dry\"](features),\n            self.heads[\"Dry_Total_g\"](features),\n            self.heads[\"GDM_g\"](features),\n        ]\n\n        # Shape: [B, 5]\n        return torch.cat(outputs, dim=1)\n\n\n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.criterion(y_hat, y)\n        self.log(\"train_loss\", loss, on_step=False, on_epoch=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_hat = self(x)\n        loss = self.criterion(y_hat, y)\n        self.val_outputs.append((y_hat.detach().cpu(), y.detach().cpu()))\n        self.log(\"val_loss\", loss, on_step=False, on_epoch=True)\n        return loss\n\n    def on_validation_epoch_end(self):\n        if len(self.val_outputs) == 0:\n            self.log(\"val_weighted_r2\", 0.0, prog_bar=True, on_epoch=True)\n            for name in [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\",  \"Dry_Total_g\",\"GDM_g\"]:\n                self.log(f\"val_r2_{name}\", 0.0, on_epoch=True)\n            self.val_outputs.clear()\n            return\n\n        preds, trues = zip(*self.val_outputs)\n        preds = torch.cat(preds).numpy()\n        trues = torch.cat(trues).numpy()\n        preds = preds.reshape(-1, 5)\n        trues = trues.reshape(-1, 5)\n        weighted_r2, r2s = weighted_r2_score(trues, preds)\n        self.log(\"val_weighted_r2\", weighted_r2, prog_bar=True, on_epoch=True)\n        for i, name in enumerate([\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"Dry_Total_g\", \"GDM_g\",]):\n            self.log(f\"val_r2_{name}\", r2s[i], on_epoch=True)\n        self.val_outputs.clear()\n        return\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=self.hparams.lr)\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n        return {\"optimizer\": optimizer, \"lr_scheduler\": scheduler}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T13:27:19.705012Z","iopub.execute_input":"2025-12-06T13:27:19.705592Z","iopub.status.idle":"2025-12-06T13:27:19.718972Z","shell.execute_reply.started":"2025-12-06T13:27:19.705566Z","shell.execute_reply":"2025-12-06T13:27:19.718392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======== DataModule ========\nclass ImageRegressionDataModule(pl.LightningDataModule):\n    def __init__(self, train_df, valid_df,normalizer,target_cols, batch_size=16, num_workers=4, img_size=1000):\n        super().__init__()\n        self.train_df = train_df\n        self.valid_df = valid_df\n        self.batch_size = batch_size\n        self.num_workers = num_workers\n        self.img_size = img_size\n        self.normalizer = normalizer\n        self.target_cols = target_cols\n    def setup(self, stage=None):\n        train_transform = T.Compose([\n            T.Resize((self.img_size, self.img_size)),\n            T.RandomHorizontalFlip(p=0.5),\n            T.RandomVerticalFlip(p=0.5),\n            T.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.05),\n            T.ToTensor(),\n            T.Normalize([0.485, 0.456, 0.406],\n                        [0.229, 0.224, 0.225])\n        ])\n        val_transform = T.Compose([\n            T.Resize((self.img_size, self.img_size)),\n            T.ToTensor(),\n            T.Normalize([0.485, 0.456, 0.406],\n                        [0.229, 0.224, 0.225])\n        ])\n        self.train_dataset = ImageRegressionDataset(self.train_df, self.normalizer,self.target_cols,train_transform)\n        self.val_dataset = ImageRegressionDataset(self.valid_df,self.normalizer,self.target_cols,val_transform)\n\n    def train_dataloader(self):\n        return DataLoader(self.train_dataset, batch_size=self.batch_size,\n                          shuffle=True, num_workers=self.num_workers)\n\n    def val_dataloader(self):\n        return DataLoader(self.val_dataset, batch_size=self.batch_size,\n                          shuffle=False, num_workers=self.num_workers)\n        \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T13:27:20.216312Z","iopub.execute_input":"2025-12-06T13:27:20.217097Z","iopub.status.idle":"2025-12-06T13:27:20.223878Z","shell.execute_reply.started":"2025-12-06T13:27:20.217065Z","shell.execute_reply":"2025-12-06T13:27:20.223294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trained_models = []\n\n#  Load & reshape data \ntrain_df = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntrain_df = pd.pivot_table(\n    train_df,\n    index='image_path',\n    columns=['target_name'],\n    values='target'\n).reset_index()\ntarget_cols = [\n    \"Dry_Clover_g\", \n    \"Dry_Dead_g\", \n    \"Dry_Green_g\", \n    \"Dry_Total_g\", \n    \"GDM_g\"\n]\npt = PowerTransformer(method='yeo-johnson')\ntrain_df[target_cols] = pt.fit_transform(train_df[target_cols].values)\n\ndef train_model(n_fold = 7,\n               batch_size = 16,\n               image_size = 1000,\n               model_name= \"tf-efficientnet-b0\",\n               model_location= \"/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b0/1/tf_efficientnet_b0_aa-827b6e33.pth\",\n               epochs = 50,\n               trained_models=trained_models):\n    #  K-Fold setup \n    kf = KFold(n_splits=n_fold, random_state=0, shuffle=True)\n    \n    for fold, (train_index, valid_index) in enumerate(kf.split(train_df)):\n    \n        print(f\"\\n================= Fold {fold + 1} =================\")\n    \n        datamodule = ImageRegressionDataModule(\n            train_df.iloc[train_index],\n            train_df.iloc[valid_index],\n            batch_size=batch_size,\n            img_size=image_size,\n            normalizer = pt,\n            target_cols = target_cols\n        )\n    \n        model = MultiRegressionModel(lr=1e-4,model_name= model_name,model_location= model_location)\n    \n        #  Checkpoints \n        ckpt_loss = ModelCheckpoint(\n            monitor=\"val_loss\",\n            save_top_k=1,\n            mode=\"min\",                    \n            filename=f\"best_by_loss_fold{fold}\"\n        )\n    \n        ckpt_r2 = ModelCheckpoint(\n            monitor=\"val_weighted_r2\",\n            save_top_k=1,\n            mode=\"max\",                    \n            filename=f\"best_by_r2_fold{fold}\"\n        )\n    \n        #  Early Stopping \n        earlystopping = EarlyStopping(\n            monitor=\"val_loss\",\n            mode=\"min\",                    \n            patience=8,\n            min_delta=1e-4,\n            verbose=True\n        )\n    \n        #  Trainer \n        trainer = pl.Trainer(\n            max_epochs=epochs,\n            accelerator=\"gpu\" if torch.cuda.is_available() else \"cpu\",\n            callbacks=[ckpt_loss, ckpt_r2, earlystopping],\n            precision=\"16-mixed\",\n            log_every_n_steps=10\n        )\n    \n        #  Train \n        trainer.fit(model, datamodule=datamodule)\n    \n        #  Results \n        print(f\"\\nFold {fold + 1} Results:\")\n        print(f\" Best (val_loss) Path: {ckpt_loss.best_model_path}\")\n        print(f\" Best (val_loss): {ckpt_loss.best_model_score}\")\n    \n        print(f\" Best (val_weighted_r2) Path: {ckpt_r2.best_model_path}\")\n        print(f\" Best (val_weighted_r2): {ckpt_r2.best_model_score}\")\n    \n        #  Save best loss model \n        if ckpt_loss.best_model_path:\n            ckpt = torch.load(ckpt_loss.best_model_path, map_location=\"cpu\")\n            torch.save(\n                ckpt[\"state_dict\"],\n                f\"best_loss_state_dict_fold{fold}.pth\"\n            )\n    \n            best_model_loss = MultiRegressionModel.load_from_checkpoint(\n                ckpt_loss.best_model_path\n            )\n            trained_models.append(best_model_loss)\n    \n        \n        #  Save best R2 model \n        if ckpt_r2.best_model_path:\n            ckpt = torch.load(ckpt_r2.best_model_path, map_location=\"cpu\")\n            torch.save(\n                ckpt[\"state_dict\"],\n                f\"best_r2_state_dict_fold{fold}.pth\"\n            )\n    \n            best_model_r2 = MultiRegressionModel.load_from_checkpoint(\n                ckpt_r2.best_model_path\n            )\n            trained_models.append(best_model_r2)\n    \n        # Cleanup \n        del model\n        del datamodule\n        torch.cuda.empty_cache()\n        gc.collect()\n    \n    \n    print(f\"\\n✅ Finished training {len(trained_models)} best models (val_loss + val_weighted_r2 for each fold)\")\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T13:27:20.243548Z","iopub.execute_input":"2025-12-06T13:27:20.243746Z","iopub.status.idle":"2025-12-06T13:27:20.279481Z","shell.execute_reply.started":"2025-12-06T13:27:20.243731Z","shell.execute_reply":"2025-12-06T13:27:20.278973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_model(n_fold = 7,\n               batch_size = 8,\n               image_size = 1000,\n               model_name= \"tf_efficientnet_cc_b0_4e\",\n               model_location= \"/kaggle/input/tf-efficientnet-condconv/pytorch/tf-efficientnet-cc-b0-4e/1/tf_efficientnet_cc_b0_4e-4362b6b2.pth\",\n               epochs = 50,\n               trained_models=trained_models)\n\"\"\"\ntrain_model(n_fold = 5,\n               batch_size = 16,\n               image_size = 1000,\n               model_name= \"tf_efficientnet_b0\",\n               model_location= \"/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b0/1/tf_efficientnet_b0_aa-827b6e33.pth\",\n               epochs = 50,\n               trained_models=trained_models)\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-06T13:27:20.280526Z","iopub.execute_input":"2025-12-06T13:27:20.280861Z","execution_failed":"2025-12-06T13:30:25.807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef make_prediction(trained_models, test_df, normalizer, image_size=(1000,1000), device=None):\n\n    if device is None:\n        device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n    # unique images only\n    image_paths = pd.DataFrame(test_df['image_path'].unique(), columns=['image_path'])\n\n    transform = T.Compose([\n        T.Resize(image_size),\n        T.ToTensor(),\n        T.Normalize([0.485, 0.456, 0.406],\n                    [0.229, 0.224, 0.225])\n    ])\n\n    test_ds = ImageRegressionDatasetTest(image_paths, transform)\n    test_loader = DataLoader(test_ds, batch_size=16, shuffle=False)\n\n    # Move models to device once\n    for model in trained_models:\n        model.to(device)\n        model.eval()\n\n    results = []\n\n    with torch.no_grad():\n        for images, image_ids in tqdm(test_loader):\n            images = images.to(device)\n\n            fold_preds = []\n            for model in trained_models:\n                outputs = model(images)          # (B,5)\n                fold_preds.append(outputs)\n\n            # Average over folds\n            ensemble_preds = torch.stack(fold_preds).mean(dim=0)  # (B,5)\n            # shape\n            ensemble_preds = ensemble_preds.cpu().numpy().reshape(-1, 5)\n            # back to origin\n            ensemble_preds = normalizer.inverse_transform(ensemble_preds)\n\n            for img_id, pred in zip(image_ids, ensemble_preds):\n                for t_idx, t_name in enumerate(target_cols):\n                    row_id = f\"{img_id}__{t_name}\"\n                    value = float(pred[t_idx])\n                    results.append((row_id, value))\n\n    submission = pd.DataFrame(results, columns=[\"sample_id\", \"target\"])\n    submission.to_csv(\"submission.csv\", index=False)\n\n    return submission\ntest_df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")\n\nmake_prediction(trained_models, test_df, normalizer= pt, image_size=(1000,1000),\n                device=\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-06T13:30:25.807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}