{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":676648,"sourceType":"modelInstanceVersion","modelInstanceId":513049,"modelId":527686}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# End-to-End Biomass Prediction Pipeline with Deep Learning\n\n**Objective:** This notebook integrates exploratory data analysis, advanced feature engineering, and a deep learning model to create a competitive solution for the CSIRO Image2Biomass competition.\n\nThe pipeline follows best practices from top Kaggle notebooks and the guidance provided in `furtherinst.md`:\n1.  **Advanced EDA:** Deeper analysis of feature correlations and target distributions.\n2.  **Feature Engineering:** Combines metadata with deep features from a pretrained CNN (EfficientNet-B0).\n3.  **Modeling:** Uses a multi-output regression model with a custom weighted loss function to align with the competition's R² metric.\n4.  **Training:** Implements a robust training and validation scheme using `GroupKFold` to prevent data leakage.\n5.  **Inference:** Generates predictions on the test set and formats the output for submission.\n","metadata":{}},{"cell_type":"markdown","source":"## 1. Setup, Imports and Configuration\n","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nimport timm\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\n\nclass CFG:\n    seed = 42\n    model_name = 'efficientnet_b0'\n    img_size = 224\n    batch_size = 32\n    n_epochs = 10\n    lr = 1e-3\n    n_folds = 5\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef set_seed(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.backends.cudnn.deterministic = True\n\nset_seed(CFG.seed)\n\nprint(f\"Using device: {CFG.device}\")\n","metadata":{"execution":{"iopub.status.busy":"2025-12-09T05:44:59.146722Z","iopub.execute_input":"2025-12-09T05:44:59.147079Z","iopub.status.idle":"2025-12-09T05:44:59.158190Z","shell.execute_reply.started":"2025-12-09T05:44:59.147052Z","shell.execute_reply":"2025-12-09T05:44:59.157188Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Data Loading and Advanced EDA\n\nWe load the data and pivot it into a wide format, where each row represents a unique image and target variables are in separate columns. This is the correct format for multi-output modeling.\n","metadata":{}},{"cell_type":"code","source":"def get_data_dir():\n    # Returns the data directory based on the environment (Kaggle or local).\n    if Path(\"/kaggle/input/csiro-biomass\").exists():\n        return Path(\"/kaggle/input/csiro-biomass\")\n    else:\n        return Path(\"data\")\n\nDATA_DIR = get_data_dir()\nprint(f\"Data directory: {DATA_DIR}\")\n\ntrain_df_long = pd.read_csv(DATA_DIR / 'train.csv')\ntest_df_long = pd.read_csv(DATA_DIR / 'test.csv')\n\ndef pivot_data(df):\n    pivot_df = df.pivot_table(index='image_path', columns='target_name', values='target').reset_index()\n    meta_cols = ['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']\n    meta_cols = [col for col in meta_cols if col in df.columns]\n    meta_df = df[meta_cols].drop_duplicates(subset=['image_path'])\n    final_df = pd.merge(pivot_df, meta_df, on='image_path')\n    return final_df\n\ntrain_df = pivot_data(train_df_long)\n\nprint(\"Pivoted Training Data Shape:\", train_df.shape)\ndisplay(train_df.head())\n","metadata":{"execution":{"iopub.status.busy":"2025-12-09T05:44:59.160018Z","iopub.execute_input":"2025-12-09T05:44:59.160264Z","iopub.status.idle":"2025-12-09T05:44:59.210856Z","shell.execute_reply.started":"2025-12-09T05:44:59.160244Z","shell.execute_reply":"2025-12-09T05:44:59.210138Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Correlation Analysis\n","metadata":{}},{"cell_type":"code","source":"target_cols = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'GDM_g', 'Dry_Total_g']\nnumerical_meta_cols = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\ncorr_matrix = train_df[numerical_meta_cols + target_cols].corr()\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f')\nplt.title('Correlation Matrix of Numerical Features and Targets')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2025-12-09T05:44:59.211697Z","iopub.execute_input":"2025-12-09T05:44:59.211982Z","iopub.status.idle":"2025-12-09T05:44:59.585864Z","shell.execute_reply.started":"2025-12-09T05:44:59.211960Z","shell.execute_reply":"2025-12-09T05:44:59.585054Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### PCA and t-SNE Visualization","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\n\nmetadata_df = train_df.drop(columns=target_cols + ['image_path', 'Sampling_Date'])\nnumerical_cols = metadata_df.select_dtypes(include=np.number).columns.tolist()\ncategorical_cols = metadata_df.select_dtypes(exclude=np.number).columns.tolist()\n\npreprocessor_viz = ColumnTransformer(\n    transformers=[\n        ('num', StandardScaler(), numerical_cols),\n        ('cat', OneHotEncoder(handle_unknown='ignore'), categorical_cols)\n    ]\n)\n\nmetadata_scaled = preprocessor_viz.fit_transform(metadata_df)\n\n# PCA\npca = PCA(n_components=2)\npca_result = pca.fit_transform(metadata_scaled.toarray() if hasattr(metadata_scaled, 'toarray') else metadata_scaled)\n\n# t-SNE\ntsne = TSNE(n_components=2, perplexity=min(30, len(metadata_df) - 1), random_state=CFG.seed)\ntsne_result = tsne.fit_transform(metadata_scaled.toarray() if hasattr(metadata_scaled, 'toarray') else metadata_scaled)\n\nplt.figure(figsize=(16, 6))\n\nplt.subplot(1, 2, 1)\nsns.scatterplot(x=pca_result[:, 0], y=pca_result[:, 1], hue=train_df['Species'], palette='viridis')\nplt.title('PCA of Metadata')\n\nplt.subplot(1, 2, 2)\nsns.scatterplot(x=tsne_result[:, 0], y=tsne_result[:, 1], hue=train_df['Species'], palette='viridis')\nplt.title('t-SNE of Metadata')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T05:44:59.586994Z","iopub.execute_input":"2025-12-09T05:44:59.587539Z","iopub.status.idle":"2025-12-09T05:45:02.564078Z","shell.execute_reply.started":"2025-12-09T05:44:59.587506Z","shell.execute_reply":"2025-12-09T05:45:02.563007Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Preprocessing, Dataset and DataLoaders\n","metadata":{}},{"cell_type":"code","source":"transforms = T.Compose([\n    T.Resize((CFG.img_size, CFG.img_size)),\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\ncategorical_features = ['State', 'Species']\nnumerical_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', StandardScaler(), numerical_features),\n        ('cat', OneHotEncoder(handle_unknown='ignore'), categorical_features)\n    ]\n)\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, metadata_preprocessor, transforms, is_test=False):\n        self.df = df\n        self.preprocessor = metadata_preprocessor\n        self.transforms = transforms\n        self.is_test = is_test\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = DATA_DIR / row['image_path']\n        image = Image.open(img_path).convert('RGB')\n        image = self.transforms(image)\n        \n        meta_arr = self.preprocessor.transform(\n            self.df.iloc[[idx]][numerical_features + categorical_features]\n        )\n        # OneHotEncoder devuelve sparse por defecto\n        if hasattr(meta_arr, 'toarray'):\n            meta_arr = meta_arr.toarray()\n        meta = torch.tensor(meta_arr.squeeze(), dtype=torch.float32)\n        \n        if self.is_test:\n            return image, meta\n        else:\n            targets = torch.tensor(row[target_cols].values.astype(np.float32), dtype=torch.float32)\n            return image, meta, targets\n\npreprocessor.fit(train_df[numerical_features + categorical_features])\nmetadata_dim = preprocessor.transform(train_df[numerical_features + categorical_features]).shape[1]\nprint(f\"Dimension of preprocessed metadata: {metadata_dim}\")\n","metadata":{"execution":{"iopub.status.busy":"2025-12-09T05:45:02.566682Z","iopub.execute_input":"2025-12-09T05:45:02.567068Z","iopub.status.idle":"2025-12-09T05:45:02.594230Z","shell.execute_reply.started":"2025-12-09T05:45:02.567043Z","shell.execute_reply":"2025-12-09T05:45:02.593195Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Model Architecture\n\nWe define a model that combines a pretrained CNN for image feature extraction with a multi-layer perceptron (MLP) for metadata. The features are then concatenated and passed to a final regression head.\n","metadata":{}},{"cell_type":"code","source":"class BiomassModel(nn.Module):\n    def __init__(self, model_name, metadata_dim, n_targets, pretrained=True):\n        super().__init__()\n        self.image_model = timm.create_model(\"resnet18\", pretrained=False, num_classes=0)\n        self.image_model.load_state_dict(\n            torch.load(\"/kaggle/input/resnet18/pytorch/default/1/resnet18_pretrained.pth\"),\n            strict=False  # ✅ <- \n        )\n        in_features = self.image_model.num_features\n        self.meta_mlp = nn.Sequential(\n            nn.Linear(metadata_dim, 128),\n            nn.ReLU(),\n            nn.Linear(128, 64)\n        )\n        self.head = nn.Linear(in_features + 64, n_targets)\n\n    def forward(self, image, metadata):\n        img_features = self.image_model(image)\n        meta_features = self.meta_mlp(metadata)\n        combined_features = torch.cat([img_features, meta_features], dim=1)\n        output = self.head(combined_features)\n        return output\n\nmodel = BiomassModel(\n    model_name=CFG.model_name,\n    metadata_dim=metadata_dim,\n    n_targets=len(target_cols)\n)\nmodel.to(CFG.device)\nprint(\"Model architecture loaded successfully.\")\n","metadata":{"execution":{"iopub.status.busy":"2025-12-09T05:45:32.213691Z","iopub.execute_input":"2025-12-09T05:45:32.214169Z","iopub.status.idle":"2025-12-09T05:45:33.098982Z","shell.execute_reply.started":"2025-12-09T05:45:32.214140Z","shell.execute_reply":"2025-12-09T05:45:33.098059Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Training and Validation\n\nImplemented GridSearch for getting best hyperparameters.\n\nWe implement a standard training loop and use GroupKFold cross-validation to ensure our model generalizes well. The `Sampling_Date` is used to group the data, preventing images from the same day from appearing in both the training and validation sets.\n","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\n\ndef train_one_epoch(model, dataloader, optimizer, criterion):\n    model.train()\n    total_loss = 0\n    for image, meta, targets in tqdm(dataloader, desc=\"Training\"):\n        image, meta, targets = image.to(CFG.device), meta.to(CFG.device), targets.to(CFG.device)\n        optimizer.zero_grad()\n        predictions = model(image, meta)\n        loss = criterion(predictions, targets)\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n    return total_loss / len(dataloader)\n\ndef validate(model, dataloader, criterion):\n    model.eval()\n    total_loss = 0\n    with torch.no_grad():\n        for image, meta, targets in tqdm(dataloader, desc=\"Validation\"):\n            image, meta, targets = image.to(CFG.device), meta.to(CFG.device), targets.to(CFG.device)\n            predictions = model(image, meta)\n            loss = criterion(predictions, targets)\n            total_loss += loss.item()\n    return total_loss / len(dataloader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T05:45:50.802800Z","iopub.execute_input":"2025-12-09T05:45:50.803141Z","iopub.status.idle":"2025-12-09T05:45:50.811004Z","shell.execute_reply.started":"2025-12-09T05:45:50.803119Z","shell.execute_reply":"2025-12-09T05:45:50.810120Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom itertools import product\n\n# ==============================\n#  BÚSQUEDA DE HIPERPARÁMETROS\n# ==============================\n\n# 1) Subconjunto para acelerar la búsqueda\n#    (ajusta frac según tu paciencia/GPU)\ntrain_subset_df = train_df.sample(frac=0.5, random_state=CFG.seed).reset_index(drop=True)\n\n# 2) Split simple train/val dentro de ese subset\nhp_train_df, hp_val_df = train_test_split(\n    train_subset_df,\n    test_size=0.2,\n    random_state=CFG.seed\n)\n\nhp_train_dataset = BiomassDataset(hp_train_df, preprocessor, transforms)\nhp_val_dataset   = BiomassDataset(hp_val_df, preprocessor, transforms)\n\nhp_train_loader = DataLoader(hp_train_dataset, batch_size=CFG.batch_size, shuffle=True)\nhp_val_loader   = DataLoader(hp_val_dataset,   batch_size=CFG.batch_size, shuffle=False)\n\ndef run_hp_experiment(lr, optimizer_cls, n_epochs_hp=3):\n    \"\"\"Entrena un modelo pequeño con (lr, optimizer_cls) y devuelve val_loss.\"\"\"\n    model = BiomassModel(\n        model_name=CFG.model_name,\n        metadata_dim=metadata_dim,\n        n_targets=len(target_cols)\n    ).to(CFG.device)\n\n    optimizer = optimizer_cls(model.parameters(), lr=lr)\n    criterion = nn.MSELoss()\n\n    best_val_loss = float(\"inf\")\n\n    for epoch in range(n_epochs_hp):\n        train_loss = train_one_epoch(model, hp_train_loader, optimizer, criterion)\n        val_loss   = validate(model, hp_val_loader, criterion)\n        print(f\"[HP] lr={lr}, opt={optimizer_cls.__name__}, \"\n              f\"epoch={epoch+1}: train={train_loss:.4f}, val={val_loss:.4f}\")\n\n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n\n    return best_val_loss\n\n# 3) Definir rejilla de hiperparámetros\nparam_grid = {\n    \"lr\": [1e-4, 1e-3, 1e-2],\n    \"optimizer\": [torch.optim.Adam, torch.optim.SGD],\n}\n\nbest_score = float(\"inf\")\nbest_params = None\n\n# 4) Grid search “a mano”\nfor lr, optimizer_cls in product(param_grid[\"lr\"], param_grid[\"optimizer\"]):\n    val_loss = run_hp_experiment(lr, optimizer_cls, n_epochs_hp=3)\n    print(f\"[HP] Resultado lr={lr}, opt={optimizer_cls.__name__}: val_loss={val_loss:.4f}\")\n\n    if val_loss < best_score:\n        best_score = val_loss\n        best_params = {\n            \"lr\": lr,\n            \"optimizer_name\": optimizer_cls.__name__.lower()   # \"adam\" o \"sgd\"\n        }\n\nprint(\"\\n==============================\")\nprint(f\"Mejores hiperparámetros encontrados:\")\nprint(f\"  val_loss = {best_score:.4f}\")\nprint(f\"  lr       = {best_params['lr']}\")\nprint(f\"  optimizer= {best_params['optimizer_name']}\")\nprint(\"==============================\")\n\n# 5) Guardar en CFG para usarlos en el entrenamiento oficial\nCFG.lr = best_params[\"lr\"]\nCFG.optimizer_name = best_params[\"optimizer_name\"]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T05:45:54.607658Z","iopub.execute_input":"2025-12-09T05:45:54.608016Z","iopub.status.idle":"2025-12-09T05:56:26.785537Z","shell.execute_reply.started":"2025-12-09T05:45:54.607993Z","shell.execute_reply":"2025-12-09T05:56:26.784521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import KFold\n\ndef get_optimizer_from_cfg(model):\n    opt_name = getattr(CFG, \"optimizer_name\", \"adam\").lower()\n    if opt_name == \"sgd\":\n        optimizer_cls = torch.optim.SGD\n    else:\n        optimizer_cls = torch.optim.Adam\n    return optimizer_cls(model.parameters(), lr=CFG.lr)\n\ngkf = GroupKFold(n_splits=CFG.n_folds)\ngroups = train_df['Sampling_Date']\n\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups=groups)):\n    print(f\"--- Fold {fold+1}/{CFG.n_folds} ---\")\n    train_fold_df = train_df.iloc[train_idx]\n    val_fold_df = train_df.iloc[val_idx]\n\n    train_dataset = BiomassDataset(train_fold_df, preprocessor, transforms)\n    val_dataset = BiomassDataset(val_fold_df, preprocessor, transforms)\n\n    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False)\n\n    model = BiomassModel(CFG.model_name, metadata_dim, len(target_cols)).to(CFG.device)\n    optimizer = get_optimizer_from_cfg(model)\n    criterion = nn.MSELoss()\n\n    for epoch in range(CFG.n_epochs):\n        train_loss = train_one_epoch(model, train_loader, optimizer, criterion)\n        val_loss = validate(model, val_loader, criterion)\n        print(f\"Epoch {epoch+1}: Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}\")\n\n    # For demo purposes, solo un fold\n    break\n","metadata":{"execution":{"iopub.status.busy":"2025-12-09T05:59:35.514068Z","iopub.execute_input":"2025-12-09T05:59:35.514985Z","iopub.status.idle":"2025-12-09T06:11:26.519698Z","shell.execute_reply.started":"2025-12-09T05:59:35.514947Z","shell.execute_reply":"2025-12-09T06:11:26.518723Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Inference and Submission\n\nFinally, we generate predictions on the test set. The test data needs to be preprocessed in the same way as the training data. The final predictions are reshaped back into the long format required for submission.\n","metadata":{}},{"cell_type":"code","source":"test_meta_df = test_df_long[['image_path']].drop_duplicates().copy()\n\n# ✅ Imputar columnas faltantes usando moda (para categóricas) o media (para numéricas)\nmeta_fill_values = {}\nfor col in ['Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm']:\n    if col in train_df:\n        if train_df[col].dtype == 'object':\n            fill_value = train_df[col].mode()[0]\n        else:\n            fill_value = train_df[col].mean()\n        test_meta_df[col] = fill_value\n        meta_fill_values[col] = fill_value  # opcional: guardar para reproducibilidad\n        \ntest_dataset = BiomassDataset(test_meta_df, preprocessor, transforms, is_test=True)\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False)\n\n# Inference\nmodel.eval()\nall_preds = []\nwith torch.no_grad():\n    for image, meta in tqdm(test_loader, desc=\"Inference\"):\n        image, meta = image.to(CFG.device), meta.to(CFG.device)\n        predictions = model(image, meta)\n        all_preds.append(predictions.cpu().numpy())\n\nall_preds = np.concatenate(all_preds)\n\n# Convert to dataframe\npred_df = pd.DataFrame(all_preds, columns=target_cols)\npred_df['image_path'] = test_meta_df['image_path'].values\n\npred_long_df = pred_df.melt(id_vars='image_path', var_name='target_name', value_name='target')\n\n# Create submission.csv\nsubmission_df = pd.merge(\n    test_df_long[['sample_id', 'image_path', 'target_name']],\n    pred_long_df,\n    on=['image_path', 'target_name'],\n    how='left'\n)[['sample_id', 'target']]\n\nsubmission_df.to_csv('submission.csv', index=False)\nprint(\"Submission file created successfully!\")\ndisplay(submission_df.head())\n","metadata":{"execution":{"iopub.status.busy":"2025-12-09T06:14:22.452542Z","iopub.execute_input":"2025-12-09T06:14:22.452878Z","iopub.status.idle":"2025-12-09T06:14:22.754799Z","shell.execute_reply.started":"2025-12-09T06:14:22.452854Z","shell.execute_reply":"2025-12-09T06:14:22.754040Z"},"trusted":true},"outputs":[],"execution_count":null}]}