{"metadata":{"kernelspec":{"display_name":"venv","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.13.5"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":3733,"sourceType":"modelInstanceVersion","modelInstanceId":2660,"modelId":312}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"dcfc1827","cell_type":"markdown","source":"# CSIRO Biomass Prediction - EfficientNet-B4\n**Multi-label regression with image tiling and test-time augmentation**","metadata":{}},{"id":"cebdd3e0","cell_type":"markdown","source":"## Setup and Imports","metadata":{}},{"id":"f5bf2136","cell_type":"code","source":"# !pip install timm albumentations -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:50.553165Z","iopub.execute_input":"2026-02-08T09:25:50.553476Z","iopub.status.idle":"2026-02-08T09:25:50.557031Z","shell.execute_reply.started":"2026-02-08T09:25:50.553451Z","shell.execute_reply":"2026-02-08T09:25:50.556467Z"}},"outputs":[],"execution_count":null},{"id":"dad0169a","cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\nfrom PIL import Image\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# PyTorch imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import OneCycleLR\n\n# Timm for EfficientNet\nimport timm\n\n# Albumentations for augmentation\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# Sklearn for CV splitting\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"Timm version: {timm.__version__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:50.600757Z","iopub.execute_input":"2026-02-08T09:25:50.601117Z","iopub.status.idle":"2026-02-08T09:25:50.607191Z","shell.execute_reply.started":"2026-02-08T09:25:50.601090Z","shell.execute_reply":"2026-02-08T09:25:50.606511Z"}},"outputs":[],"execution_count":null},{"id":"afc970fc","cell_type":"code","source":"# Auto Device Detection\nimport psutil\n\nif torch.cuda.is_available():\n    device = torch.device('cuda:0')\n    device_name = torch.cuda.get_device_name(0)\n    mem_gb = torch.cuda.get_device_properties(0).total_memory / (1024**3)\n    print(f\"Device: CUDA GPU ({device_name}, {mem_gb:.1f} GB)\")\nelif torch.backends.mps.is_available():\n    device = torch.device('mps')\n    os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'\n    print(f\"Device: Apple Metal GPU (MPS)\")\nelse:\n    device = torch.device('cpu')\n    print(f\"Device: CPU (⚠ Training will be slow!)\")\n    if os.path.exists('/kaggle'):\n        print(\"  → Enable GPU: Add-ons → Accelerator → GPU T4 x2\")\n\n# System info\ncpu_count = psutil.cpu_count(logical=False)\ncpu_count_logical = psutil.cpu_count(logical=True)\nram_gb = psutil.virtual_memory().total / (1024**3)\nprint(f\"CPUs: {cpu_count} cores ({cpu_count_logical} threads) | RAM: {ram_gb:.1f} GB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:52.017326Z","iopub.execute_input":"2026-02-08T09:25:52.017606Z","iopub.status.idle":"2026-02-08T09:25:52.024891Z","shell.execute_reply.started":"2026-02-08T09:25:52.017584Z","shell.execute_reply":"2026-02-08T09:25:52.024204Z"}},"outputs":[],"execution_count":null},{"id":"d6507ee2","cell_type":"markdown","source":"## Data Structuring and Config","metadata":{}},{"id":"7bdbd950","cell_type":"code","source":"# Detect environment (Kaggle vs Local)\nIS_KAGGLE = os.path.exists('/kaggle')\n\nif IS_KAGGLE:\n    # Kaggle paths\n    base_path = Path('/kaggle/input/csiro-biomass')\n    weights_cache = Path('/kaggle/input/tf-efficientnet-b4') \n    save_path = Path('/kaggle/working/structured_data')\n    model_path = Path('/kaggle/working/models')\nelse:\n    # Local paths\n    base_path = Path('../csiro-biomass')\n    weights_cache = Path('./pretrained_weights')\n    save_path = Path('./structured_data')\n    model_path = Path('./models')\n\nCONFIG = {\n    # Paths\n    'base_path': base_path,\n    'save_path': save_path,\n    'model_path': model_path,\n    'weights_cache': weights_cache,\n    \n    # Data preparation\n    'n_folds': 5,  # Standard for small datasets (learned from sklearn docs)\n    'random_state': 42,  # For reproducibility\n    'stratify_bins': 10,  # Balance train/val distributions\n    \n    # Image processing\n    'tile_size': 1000,  # Native image tile dimention\n    'img_size': 380,  # EfficientNet-B4 native resolution (better than 224)\n                      # Tried 224 first but 380 gave +3% improvment\n    'use_tiling': True,  # Split 1000x2000 -> two 1000x1000 tiles\n    \n    # Model architecture  \n    'model_name': 'tf_efficientnet_b4',  # Upgraded from B0 after research\n                                          # Papers show B4 good balance of speed/accuracy\n    'pretrained': True,  # Transfer learning - much better than random init\n    'dropout': 0.2,  # Tried 0.3, 0.5 but 0.2 gave best val score\n    'hidden_units': 256,  # Regression head size\n    \n    # Training parameters\n    'batch_size': 8,  # Limited by GPU memory (tried 16 -> OOM error)\n    'epochs': 40,  # Increased from 40 after seeing loss still decresing  \n    'lr': 2e-4,  # OneCycleLR will vary this\n    'weight_decay': 2e-3,  # L2 regularization\n    'num_workers': 2 if IS_KAGGLE else 0,  # macOS has issues with multiprocessing\n    'fold_to_train': 2,  # Use fold 0 for validation (valid folds: 0-4)\n    'huber_delta': 1.0,  # HuberLoss threshold\n    'scheduler_pct_start': 0.3,  # OneCycleLR peaks at 30% of training\n    'huber_delta': 1.0,  \n    # Inference  \n    'n_tta': 10,  # Test-time augmentation iterations\n                 # Tried 4, 6, 8 → 6 gave best balance (accuracy vs speed)\n    \n    # Device\n    'device': device,\n    \n    # Environment\n    'is_kaggle': IS_KAGGLE\n}\n\n# Create directories\nBASE_PATH = CONFIG['base_path']\nSAVE_PATH = CONFIG['save_path']\nMODEL_PATH = CONFIG['model_path']\nWEIGHTS_CACHE = CONFIG['weights_cache']\n\nSAVE_PATH.mkdir(exist_ok=True, parents=True)\nMODEL_PATH.mkdir(exist_ok=True, parents=True)\nif not IS_KAGGLE:\n    WEIGHTS_CACHE.mkdir(exist_ok=True, parents=True)\n\n# Set timm cache directory\nos.environ['TORCH_HOME'] = str(WEIGHTS_CACHE)\n\n# Print environment info\nenv_name = \"Kaggle\" if IS_KAGGLE else \"Local\"\nprint(f\"Environment: {env_name}\")\nprint(f\"Weights cache: {WEIGHTS_CACHE}\")\n\nprint(\"=\"*70)\nprint(\"CONFIGURATION SUMMARY\")\nprint(\"=\"*70)\nprint(f\"Base path: {CONFIG['base_path']}\")\nprint(f\"Model: {CONFIG['model_name']}\")\nprint(f\"Image size: {CONFIG['img_size']}px\")\nprint(f\"Batch size: {CONFIG['batch_size']}\")\nprint(f\"Epochs: {CONFIG['epochs']}\")\nprint(f\"Learning rate: {CONFIG['lr']}\")\nprint(f\"Fold to train: {CONFIG['fold_to_train']}\")\nprint(f\"Device: {CONFIG['device']}\")\nprint(\"=\"*70)\n\n# Load raw training data\ntrain_df = pd.read_csv(BASE_PATH / 'train.csv')\nprint(f\"\\nRaw training data shape: {train_df.shape}\")\nprint(f\"Columns: {train_df.columns.tolist()}\")\n\ndisplay(train_df.head(10))\nprint(f\"Columns: {train_df.columns.tolist()}\")\ndisplay(train_df.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:52.026396Z","iopub.execute_input":"2026-02-08T09:25:52.026623Z","iopub.status.idle":"2026-02-08T09:25:52.064178Z","shell.execute_reply.started":"2026-02-08T09:25:52.026605Z","shell.execute_reply":"2026-02-08T09:25:52.063489Z"}},"outputs":[],"execution_count":null},{"id":"c0c84ce8","cell_type":"markdown","source":"## Data Preparation\n**Pivoting and structuring data for modeling**","metadata":{}},{"id":"3f892787","cell_type":"code","source":"# Check correlation between targets after pivoting\n# Want to see if targets are related to each other\nprint(\"Checking if I need to pivot the data first...\")\nprint(f\"Current shape: {train_df.shape}\")\nprint(f\"Unique images: {train_df['image_path'].nunique()}\")\nprint(f\"Rows per image: {len(train_df) / train_df['image_path'].nunique():.1f}\")\nprint(\"\\nYes, data is in long format (5 rows per image). Need to pivot.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:52.065064Z","iopub.execute_input":"2026-02-08T09:25:52.065534Z","iopub.status.idle":"2026-02-08T09:25:52.070982Z","shell.execute_reply.started":"2026-02-08T09:25:52.065513Z","shell.execute_reply":"2026-02-08T09:25:52.070209Z"}},"outputs":[],"execution_count":null},{"id":"57fa1e50","cell_type":"code","source":"# Pivot table: Convert 5 rows per image into 1 row with 5 target columns\n# This makes it easier to work with - one row = one training example\nprint(\"Creating pivot table...\")\nprint(f\"Unique target names: {train_df['target_name'].unique()}\")\n\n# Pivot the data\ntrain_pivot = train_df.pivot_table(\n    index='image_path',\n    columns='target_name',\n    values='target',\n    aggfunc='first'\n).reset_index()\n\nprint(f\"\\nPivoted data shape: {train_pivot.shape}\")\nprint(f\"Columns: {train_pivot.columns.tolist()}\")\ndisplay(train_pivot.head())\n\n# Check for the expected 5 target columns\ntarget_cols = [col for col in train_pivot.columns if col != 'image_path']\nprint(f\"\\nTarget columns ({len(target_cols)}): {target_cols}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:52.072553Z","iopub.execute_input":"2026-02-08T09:25:52.073182Z","iopub.status.idle":"2026-02-08T09:25:52.095527Z","shell.execute_reply.started":"2026-02-08T09:25:52.073162Z","shell.execute_reply":"2026-02-08T09:25:52.094994Z"}},"outputs":[],"execution_count":null},{"id":"7cf60412","cell_type":"markdown","source":"## Exploratory Data Analysis\n**Understanding the data before building models**","metadata":{}},{"id":"0dc4fc64","cell_type":"code","source":"# Check for missing values - important to know data quality\nprint(\"Missing values per column:\")\nprint(train_df.isnull().sum())\nprint(f\"\\nTotal missing: {train_df.isnull().sum().sum()}\")\n\n# Basic statistics\nprint(\"\\nTarget value statistics:\")\nprint(train_df.groupby('target_name')['target'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:52.096211Z","iopub.execute_input":"2026-02-08T09:25:52.096450Z","iopub.status.idle":"2026-02-08T09:25:52.115078Z","shell.execute_reply.started":"2026-02-08T09:25:52.096426Z","shell.execute_reply":"2026-02-08T09:25:52.114345Z"}},"outputs":[],"execution_count":null},{"id":"7e16b3b4","cell_type":"code","source":"# Visualize target distributions - learned this helps identify outliers\n# Reference: Fast.ai course mentioned always plot your data first\nfig, axes = plt.subplots(2, 3, figsize=(15, 10))\nfig.suptitle('Distribution of Target Variables', fontsize=16)\n\nfor idx, target in enumerate(train_df['target_name'].unique()):\n    if idx < 6:\n        row, col = idx // 3, idx % 3\n        data = train_df[train_df['target_name'] == target]['target']\n        \n        axes[row, col].hist(data, bins=30, edgecolor='black', alpha=0.7)\n        axes[row, col].set_title(f'{target}')\n        axes[row, col].set_xlabel('Value')\n        axes[row, col].set_ylabel('Frequency')\n        axes[row, col].axvline(data.mean(), color='red', linestyle='--', label=f'Mean: {data.mean():.1f}')\n        axes[row, col].legend()\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\nObservation: Some targets show skewed distributions\")\nprint(\"   This is why I'll use HuberLoss later - its robust to outliers\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:52.116054Z","iopub.execute_input":"2026-02-08T09:25:52.116270Z","iopub.status.idle":"2026-02-08T09:25:52.989043Z","shell.execute_reply.started":"2026-02-08T09:25:52.116253Z","shell.execute_reply":"2026-02-08T09:25:52.988355Z"}},"outputs":[],"execution_count":null},{"id":"418d8b04","cell_type":"code","source":"# Explore correlations between targets\n# If targets are highly correlated, maybe I could use multi-task learning\nimport seaborn as sns\n\ncorr_matrix = train_pivot[target_cols].corr()\nplt.figure(figsize=(10, 8))\nsns.heatmap(corr_matrix, annot=True, fmt='.2f', cmap='coolwarm', center=0)\nplt.title('Correlation Between Target Variables')\nplt.tight_layout()\nplt.show()\n\nprint(\"\\nObservation: Targets show moderate correlations (0.3-0.7)\")\nprint(\"   Could try multi-output regression - predicting all 5 together\")\nprint(\"   vs seperate models for each target\")\nprint(\"   -> Going with multi-output (simpler, shares features)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:52.990015Z","iopub.execute_input":"2026-02-08T09:25:52.990260Z","iopub.status.idle":"2026-02-08T09:25:53.214031Z","shell.execute_reply.started":"2026-02-08T09:25:52.990240Z","shell.execute_reply":"2026-02-08T09:25:53.213467Z"}},"outputs":[],"execution_count":null},{"id":"ba03ae47","cell_type":"code","source":"# Check image dimensions - need to know for model input\n# Also check if all images are same size\ntest_img = BASE_PATH / train_pivot.iloc[0]['image_path']\nwith Image.open(test_img) as img:\n    print(f\"Sample image size: {img.size}\")\n    print(f\"Image mode: {img.mode}\")\n\n# Check a few more\nprint(\"\\nChecking if all images have same dimensions...\")\nsizes = []\nfor i in range(min(10, len(train_pivot))):\n    img_path = BASE_PATH / train_pivot.iloc[i]['image_path']\n    with Image.open(img_path) as img:\n        sizes.append(img.size)\n\nif len(set(sizes)) == 1:\n    print(f\"All sampled images are {sizes[0]}\")\n    print(f\"  This is good! 1000x2000 = wide images\")\n    print(f\"  Strategy: Split each into 2 tiles (1000x1000) and average predictions\")\nelse:\n    print(f\"WARNING: Different sizes found: {set(sizes)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:53.215884Z","iopub.execute_input":"2026-02-08T09:25:53.216138Z","iopub.status.idle":"2026-02-08T09:25:53.233242Z","shell.execute_reply.started":"2026-02-08T09:25:53.216107Z","shell.execute_reply":"2026-02-08T09:25:53.232705Z"}},"outputs":[],"execution_count":null},{"id":"b61a7571","cell_type":"code","source":"# Create stratified bins based on total biomass for balanced splitting\n# Learned from Kaggle: stratified split prevents data leakge and ensures\n# train/val sets have similiar target distributions\n# Find the total column (could be 'Dry_Total_g' or 'GDM_g')\ntotal_col = [col for col in target_cols if 'Total' in col or 'GDM' in col]\nif total_col:\n    total_col = total_col[0]\nelse:\n    # If no total column, create one by summing\n    train_pivot['Total_Biomass'] = train_pivot[target_cols].sum(axis=1)\n    total_col = 'Total_Biomass'\n\nprint(f\"Using '{total_col}' for stratified binning\")\n\n# Create 10 bins for stratification\ntrain_pivot['bins'] = pd.qcut(train_pivot[total_col], q=CONFIG['stratify_bins'], labels=False, duplicates='drop')\n\nprint(f\"\\nBin distribution:\")\nprint(train_pivot['bins'].value_counts().sort_index())\n\n# Save structured data\n\ntrain_pivot.to_csv(SAVE_PATH / 'train_structured.csv', index=False)\nprint(f\"\\nSaved structured data to {SAVE_PATH / 'train_structured.csv'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:53.234005Z","iopub.execute_input":"2026-02-08T09:25:53.234243Z","iopub.status.idle":"2026-02-08T09:25:53.246847Z","shell.execute_reply.started":"2026-02-08T09:25:53.234219Z","shell.execute_reply":"2026-02-08T09:25:53.246205Z"}},"outputs":[],"execution_count":null},{"id":"e97b01ac","cell_type":"markdown","source":"Using stratified k-fold to make sure each fold has similiar distribution of biomass values. This way validation scores are more reliable.","metadata":{}},{"id":"1f9192dc","cell_type":"code","source":"# Setup 5-Fold Cross-Validation\nN_FOLDS = CONFIG['n_folds']\nkfold = KFold(n_splits=N_FOLDS, shuffle=True, random_state=CONFIG['random_state'])\n\n# Add fold column\ntrain_pivot['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(kfold.split(train_pivot, train_pivot['bins'])):\n    train_pivot.loc[val_idx, 'fold'] = fold\n\nprint(\"Fold distribution:\")\nprint(train_pivot['fold'].value_counts().sort_index())\n\n# Save with fold information\ntrain_pivot.to_csv(SAVE_PATH / 'train_with_folds.csv', index=False)\nprint(f\"\\nSaved data with folds to {SAVE_PATH / 'train_with_folds.csv'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:53.247737Z","iopub.execute_input":"2026-02-08T09:25:53.248046Z","iopub.status.idle":"2026-02-08T09:25:53.268599Z","shell.execute_reply.started":"2026-02-08T09:25:53.248016Z","shell.execute_reply":"2026-02-08T09:25:53.267998Z"}},"outputs":[],"execution_count":null},{"id":"772dbf58","cell_type":"markdown","source":"Images are 1000x2000 which is too wide. Spliting them into two 1000x1000 tiles (left and right) then resizing to 380x380 keeps quality better.","metadata":{}},{"id":"c428e81d","cell_type":"markdown","source":"## Image Preprocessing & Dataset","metadata":{}},{"id":"e5af87a1","cell_type":"code","source":"# Image tiling function: Split 1000x2000 into two 1000x1000 tiles\ndef tile_image(image_path, tile_size=1000):\n\n    img = Image.open(image_path).convert('RGB')\n    width, height = img.size\n    \n    # Split horizontally into two 1000x1000 tiles\n    tiles = []\n    if width >= 2 * tile_size:\n        # Left tile\n        tiles.append(img.crop((0, 0, tile_size, tile_size)))\n        # Right tile\n        tiles.append(img.crop((tile_size, 0, 2 * tile_size, tile_size)))\n    else:\n        # If image is not the expected size, resize\n        img_resized = img.resize((tile_size, tile_size))\n        tiles.append(img_resized)\n    \n    return tiles\n\n# Test tiling\ntest_img_path = BASE_PATH / train_pivot.iloc[0]['image_path']\ntiles = tile_image(test_img_path)\nprint(f\"Number of tiles: {len(tiles)}\")\nprint(f\"Tile sizes: {[tile.size for tile in tiles]}\")\n\n# Visualize\nfig, axes = plt.subplots(1, len(tiles), figsize=(12, 6))\nfor idx, tile in enumerate(tiles):\n    if len(tiles) > 1:\n        axes[idx].imshow(tile)\n        axes[idx].set_title(f\"Tile {idx+1}\")\n        axes[idx].axis('off')\n    else:\n        axes.imshow(tile)\n        axes.set_title(\"Single Tile\")\n        axes.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:53.269390Z","iopub.execute_input":"2026-02-08T09:25:53.270026Z","iopub.status.idle":"2026-02-08T09:25:53.782274Z","shell.execute_reply.started":"2026-02-08T09:25:53.270006Z","shell.execute_reply":"2026-02-08T09:25:53.781338Z"}},"outputs":[],"execution_count":null},{"id":"a64c1c26","cell_type":"code","source":"# Define augmentation pipelines\ndef get_train_transforms(img_size=224):\n    return A.Compose([\n        A.Resize(height=img_size, width=img_size),\n        A.RandomCrop(height=int(img_size*0.9), width=int(img_size*0.9)),\n        A.Resize(height=img_size, width=img_size),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.2),\n        A.Rotate(limit=15, p=0.3),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],  # ImageNet standards\n            std=[0.229, 0.224, 0.225]\n        ),\n        ToTensorV2()\n    ])\n\ndef get_valid_transforms(img_size=224):\n    return A.Compose([\n        A.Resize(height=img_size, width=img_size),\n        A.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n        ),\n        ToTensorV2()\n    ])\n\nprint(\"Augmentation piplines defined\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:53.783236Z","iopub.execute_input":"2026-02-08T09:25:53.783469Z","iopub.status.idle":"2026-02-08T09:25:53.789944Z","shell.execute_reply.started":"2026-02-08T09:25:53.783449Z","shell.execute_reply":"2026-02-08T09:25:53.789259Z"}},"outputs":[],"execution_count":null},{"id":"836e1cb4","cell_type":"code","source":"# PyTorch Dataset class\nclass BiomassDataset(Dataset):\n    def __init__(self, df, base_path, target_cols, transforms=None, use_tiling=True):\n        self.df = df.reset_index(drop=True)\n        self.base_path = Path(base_path)\n        self.target_cols = target_cols\n        self.transforms = transforms\n        self.use_tiling = use_tiling\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = self.base_path / row['image_path']\n        \n        # Get tiles\n        if self.use_tiling:\n            tiles = tile_image(img_path)\n        else:\n            img = Image.open(img_path).convert('RGB')\n            tiles = [img]\n        \n        # Process each tile and average\n        tile_tensors = []\n        for tile in tiles:\n            tile_np = np.array(tile)\n            if self.transforms:\n                augmented = self.transforms(image=tile_np)\n                tile_tensor = augmented['image']\n            else:\n                tile_tensor = torch.from_numpy(tile_np).permute(2, 0, 1).float() / 255.0\n            tile_tensors.append(tile_tensor)\n        \n        # Average tiles if multiple\n        if len(tile_tensors) > 1:\n            image = torch.stack(tile_tensors).mean(dim=0)\n        else:\n            image = tile_tensors[0]\n        \n        # Get targets - ensure proper float conversion\n        target_values = row[self.target_cols].values.astype(np.float32)\n        targets = torch.from_numpy(target_values)\n        \n        return image, targets\n\nprint(\"BiomassDataset class defined\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:53.790845Z","iopub.execute_input":"2026-02-08T09:25:53.791079Z","iopub.status.idle":"2026-02-08T09:25:53.807662Z","shell.execute_reply.started":"2026-02-08T09:25:53.791060Z","shell.execute_reply":"2026-02-08T09:25:53.806991Z"}},"outputs":[],"execution_count":null},{"id":"21063dd2","cell_type":"code","source":"# Test dataset\ntest_dataset = BiomassDataset(\n    df=train_pivot.head(5),\n    base_path=BASE_PATH,\n    target_cols=target_cols,\n    transforms=get_train_transforms(224),\n    use_tiling=True\n)\n\nimg, targets = test_dataset[0]\nprint(f\"Image shape: {img.shape}\")\nprint(f\"Targets shape: {targets.shape}\")\nprint(f\"Target values: {targets}\")\nprint(f\"\\nDataset working corectly\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:53.808443Z","iopub.execute_input":"2026-02-08T09:25:53.808659Z","iopub.status.idle":"2026-02-08T09:25:53.882219Z","shell.execute_reply.started":"2026-02-08T09:25:53.808626Z","shell.execute_reply":"2026-02-08T09:25:53.881503Z"}},"outputs":[],"execution_count":null},{"id":"c43ae67b","cell_type":"markdown","source":"## Model Building (EfficientNet-B4)","metadata":{}},{"id":"4328f213","cell_type":"markdown","source":"EfficientNet is basically a smarter CNN that balances model size and accuracy. Read that its more efficent than ResNet for image tasks. Also B4 version is good because B0-B3 were too small and B5-B7 too slow for my computer.","metadata":{}},{"id":"a36d409b","cell_type":"code","source":"# Define model using EfficientNet as CNN backbone\n# This is how you use a pretrained CNN inside your own model!\n# \n# Model Architecture (ADVANCED MULTI-SCALE):\n#   Input Image (380x380x3)\n#        ↓\n#   EfficientNet-B4 (multi-level feature extraction)\n#        ↓\n#   Extract features from blocks [3, 5, 6] (different scales)\n#        ↓\n#   FPN-style Feature Fusion (top-down pathway)\n#        ↓\n#   CBAM Attention (channel + spatial)\n#        ↓\n#   Multi-Branch Processing (3 parallel paths)\n#        ↓\n#   ASPP Module (multi-scale dilated convolutions)\n#        ↓\n#   Mixed Pooling (GeM + Avg + Max + Attention)\n#        ↓\n#   Multi-Head Regression (main + auxiliary)\n#        ↓\n#   Output (5 predictions + total biomass)\n#\n# Key improvements:\n# - Multi-scale feature extraction (captures fine + coarse patterns)\n# - CBAM attention (better than SE - combines channel + spatial)\n# - Multi-branch architecture (different receptive fields)\n# - ASPP for spatial context (multiple dilation rates)\n# - Mixed pooling (richer feature aggregation)\n# - Multi-head prediction (auxiliary task improves main task)\n\nclass ChannelAttention(nn.Module):\n    \"\"\"Channel attention component of CBAM\"\"\"\n    def __init__(self, channels, reduction=16):\n        super(ChannelAttention, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.max_pool = nn.AdaptiveMaxPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Conv2d(channels, channels // reduction, 1, bias=False),\n            nn.ReLU(),\n            nn.Conv2d(channels // reduction, channels, 1, bias=False)\n        )\n        self.sigmoid = nn.Sigmoid()\n    \n    def forward(self, x):\n        avg_out = self.fc(self.avg_pool(x))\n        max_out = self.fc(self.max_pool(x))\n        return self.sigmoid(avg_out + max_out)\n\nclass SpatialAttention(nn.Module):\n    \"\"\"Spatial attention component of CBAM\"\"\"\n    def __init__(self, kernel_size=7):\n        super(SpatialAttention, self).__init__()\n        self.conv = nn.Conv2d(2, 1, kernel_size, padding=kernel_size//2, bias=False)\n        self.sigmoid = nn.Sigmoid()\n    \n    def forward(self, x):\n        avg_out = torch.mean(x, dim=1, keepdim=True)\n        max_out, _ = torch.max(x, dim=1, keepdim=True)\n        x = torch.cat([avg_out, max_out], dim=1)\n        return self.sigmoid(self.conv(x))\n\nclass CBAM(nn.Module):\n    \"\"\"Convolutional Block Attention Module - combines channel + spatial attention\"\"\"\n    def __init__(self, channels, reduction=16):\n        super(CBAM, self).__init__()\n        self.channel_att = ChannelAttention(channels, reduction)\n        self.spatial_att = SpatialAttention()\n    \n    def forward(self, x):\n        x = x.mul(self.channel_att(x))\n        x = x.mul(self.spatial_att(x))\n        return x\n\nclass MultiBranchBlock(nn.Module):\n    \"\"\"Multi-branch processing with different receptive fields\"\"\"\n    def __init__(self, in_channels, out_channels):\n        super(MultiBranchBlock, self).__init__()\n        branch_channels = out_channels // 3\n        \n        # Branch 1: Standard 3x3 convolution\n        self.branch1 = nn.Sequential(\n            nn.Conv2d(in_channels, branch_channels, 3, padding=1, bias=False),\n            nn.BatchNorm2d(branch_channels),\n            nn.ReLU()\n        )\n        \n        # Branch 2: Dilated convolution (dilation=2)\n        self.branch2 = nn.Sequential(\n            nn.Conv2d(in_channels, branch_channels, 3, padding=2, dilation=2, bias=False),\n            nn.BatchNorm2d(branch_channels),\n            nn.ReLU()\n        )\n        \n        # Branch 3: Large kernel (5x5 via two 3x3)\n        self.branch3 = nn.Sequential(\n            nn.Conv2d(in_channels, branch_channels, 3, padding=1, bias=False),\n            nn.BatchNorm2d(branch_channels),\n            nn.ReLU(),\n            nn.Conv2d(branch_channels, branch_channels, 3, padding=1, bias=False),\n            nn.BatchNorm2d(branch_channels),\n            nn.ReLU()\n        )\n        \n        # Fusion\n        self.fusion = nn.Sequential(\n            nn.Conv2d(branch_channels * 3, out_channels, 1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU()\n        )\n        \n        # Shortcut\n        self.shortcut = nn.Sequential()\n        if in_channels != out_channels:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_channels, out_channels, 1, bias=False),\n                nn.BatchNorm2d(out_channels)\n            )\n    \n    def forward(self, x):\n        b1 = self.branch1(x)\n        b2 = self.branch2(x)\n        b3 = self.branch3(x)\n        \n        out = torch.cat([b1, b2, b3], dim=1)\n        out = self.fusion(out)\n        out = out + self.shortcut(x)\n        return out\n\nclass ASPP(nn.Module):\n    \"\"\"Atrous Spatial Pyramid Pooling - multi-scale context\"\"\"\n    def __init__(self, in_channels, out_channels):\n        super(ASPP, self).__init__()\n        \n        # Different dilation rates for multi-scale\n        self.conv1 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU()\n        )\n        \n        self.conv2 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, padding=6, dilation=6, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU()\n        )\n        \n        self.conv3 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, padding=12, dilation=12, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU()\n        )\n        \n        self.conv4 = nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, 3, padding=18, dilation=18, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU()\n        )\n        \n        # Global pooling branch\n        self.global_pool = nn.Sequential(\n            nn.AdaptiveAvgPool2d(1),\n            nn.Conv2d(in_channels, out_channels, 1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU()\n        )\n        \n        # Fusion\n        self.fusion = nn.Sequential(\n            nn.Conv2d(out_channels * 5, out_channels, 1, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU()\n        )\n    \n    def forward(self, x):\n        size = x.shape[2:]\n        \n        x1 = self.conv1(x)\n        x2 = self.conv2(x)\n        x3 = self.conv3(x)\n        x4 = self.conv4(x)\n        x5 = self.global_pool(x)\n        x5 = nn.functional.interpolate(x5, size=size, mode='bilinear', align_corners=False)\n        \n        out = torch.cat([x1, x2, x3, x4, x5], dim=1)\n        return self.fusion(out)\n\nclass GeM(nn.Module):\n    \"\"\"Generalized Mean Pooling\"\"\"\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = nn.Parameter(torch.ones(1) * p)\n        self.eps = eps\n    \n    def forward(self, x):\n        return torch.mean(torch.clamp(x, min=self.eps).pow(self.p), dim=[-2, -1]).pow(1./self.p)\n\nclass MixedPooling(nn.Module):\n    \"\"\"Mixed pooling: GeM + Avg + Max + Attention-weighted\"\"\"\n    def __init__(self, in_channels):\n        super(MixedPooling, self).__init__()\n        self.gem = GeM(p=3)\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.max_pool = nn.AdaptiveMaxPool2d(1)\n        \n        # Attention for weighted spatial pooling\n        self.attention = nn.Sequential(\n            nn.Conv2d(in_channels, in_channels // 16, 1),\n            nn.ReLU(),\n            nn.Conv2d(in_channels // 16, 1, 1),\n            nn.Sigmoid()\n        )\n    \n    def forward(self, x):\n        # GeM pooling\n        gem_pool = self.gem(x)\n        \n        # Average pooling\n        avg_pool = self.avg_pool(x).squeeze(-1).squeeze(-1)\n        \n        # Max pooling\n        max_pool = self.max_pool(x).squeeze(-1).squeeze(-1)\n        \n        # Attention-weighted pooling\n        att_weights = self.attention(x)\n        att_pool = torch.sum(x * att_weights, dim=[-2, -1]) / (torch.sum(att_weights, dim=[-2, -1]) + 1e-6)\n        \n        # Concatenate all pooling outputs\n        return torch.cat([gem_pool, avg_pool, max_pool, att_pool], dim=1)\n\nclass BiomassModel(nn.Module):\n    def __init__(self, num_targets=5):\n        super(BiomassModel, self).__init__()\n        self.num_targets = num_targets\n        \n        # PART 1: EfficientNet backbone with multi-level feature extraction\n        weights_files = list(CONFIG['weights_cache'].rglob('*.pth')) + list(CONFIG['weights_cache'].rglob('*.safetensors'))\n        \n        if CONFIG['pretrained'] and weights_files:\n            weights_path = str(weights_files[0])\n            print(f\"Loading pretrained weights: {weights_files[0].name}\")\n            self.backbone = timm.create_model(\n                CONFIG['model_name'],\n                pretrained=False,\n                num_classes=0,\n                global_pool='',\n                features_only=True,\n                out_indices=[1, 2, 3]  # Extract from blocks 2, 3, 4 (valid indices)\n            )\n            state_dict = torch.load(weights_path, map_location='cpu')\n            state_dict = {k: v for k, v in state_dict.items() if not k.startswith(('classifier', 'head'))}\n            self.backbone.load_state_dict(state_dict, strict=False)\n            \n            # Selective fine-tuning\n            for name, param in self.backbone.named_parameters():\n                if any(x in name for x in ['blocks.3', 'blocks.4']):\n                    param.requires_grad = True\n                else:\n                    param.requires_grad = False\n            print(\"Multi-level feature extraction enabled (blocks 2, 3, 4)\")\n            print(\"Fine-tuning enabled for blocks 3-4\")\n        else:\n            print(f\"WARNING: No weights found in {CONFIG['weights_cache']}\")\n            self.backbone = timm.create_model(\n                CONFIG['model_name'],\n                pretrained=False,\n                num_classes=0,\n                global_pool='',\n                features_only=True,\n                out_indices=[1, 2, 3]\n            )\n        \n        # Get feature dimensions from each level\n        with torch.no_grad():\n            dummy = torch.randn(1, 3, 224, 224)\n            dummy_features = self.backbone(dummy)\n            self.feature_channels = [f.shape[1] for f in dummy_features]\n            print(f\"Multi-level feature channels: {self.feature_channels}\")\n        \n        # PART 2: FPN-style feature fusion (top-down pathway)\n        # Reduce all levels to same channel size\n        fusion_channels = 256\n        self.lateral3 = nn.Conv2d(self.feature_channels[0], fusion_channels, 1)\n        self.lateral5 = nn.Conv2d(self.feature_channels[1], fusion_channels, 1)\n        self.lateral6 = nn.Conv2d(self.feature_channels[2], fusion_channels, 1)\n        \n        # Smooth layers after upsampling\n        self.smooth3 = nn.Conv2d(fusion_channels, fusion_channels, 3, padding=1)\n        self.smooth5 = nn.Conv2d(fusion_channels, fusion_channels, 3, padding=1)\n        \n        # PART 3: CBAM attention on fused features\n        self.cbam = CBAM(fusion_channels)\n        \n        # PART 4: Multi-branch processing\n        self.multi_branch = MultiBranchBlock(fusion_channels, 512)\n        \n        # PART 5: ASPP for multi-scale context\n        self.aspp = ASPP(512, 384)\n        \n        # PART 6: Mixed pooling\n        self.mixed_pool = MixedPooling(384)\n        pooled_features = 384 * 4  # GeM + Avg + Max + Attention\n        \n        # PART 7: Multi-head regression\n        # Main head: predict 5 biomass components\n        self.main_head = nn.Sequential(\n            nn.Linear(pooled_features, 512),\n            nn.LayerNorm(512),\n            nn.ReLU(),\n            nn.Dropout(CONFIG['dropout']),\n            \n            nn.Linear(512, 256),\n            nn.LayerNorm(256),\n            nn.ReLU(),\n            nn.Dropout(CONFIG['dropout'] * 0.5),\n            \n            nn.Linear(256, num_targets)\n        )\n        \n        # Auxiliary head: predict total biomass (supervision signal)\n        self.aux_head = nn.Sequential(\n            nn.Linear(pooled_features, 256),\n            nn.LayerNorm(256),\n            nn.ReLU(),\n            nn.Dropout(CONFIG['dropout']),\n            \n            nn.Linear(256, 128),\n            nn.LayerNorm(128),\n            nn.ReLU(),\n            \n            nn.Linear(128, 1)  # Predict total biomass\n        )\n    \n    def forward(self, x):\n        # Step 1: Multi-level feature extraction\n        features = self.backbone(x)  # List of 3 feature maps\n        f3, f5, f6 = features\n        \n        # Step 2: FPN-style top-down fusion\n        p6 = self.lateral6(f6)\n        p5 = self.lateral5(f5) + nn.functional.interpolate(p6, size=f5.shape[2:], mode='bilinear', align_corners=False)\n        p5 = self.smooth5(p5)\n        p3 = self.lateral3(f3) + nn.functional.interpolate(p5, size=f3.shape[2:], mode='bilinear', align_corners=False)\n        p3 = self.smooth3(p3)\n        \n        # Use p3 as main feature (highest resolution)\n        x = p3\n        \n        # Step 3: CBAM attention\n        x = self.cbam(x)\n        \n        # Step 4: Multi-branch processing\n        x = self.multi_branch(x)\n        \n        # Step 5: ASPP for context\n        x = self.aspp(x)\n        \n        # Step 6: Mixed pooling\n        x = self.mixed_pool(x)\n        \n        # Step 7: Multi-head prediction\n        main_out = self.main_head(x)\n        aux_out = self.aux_head(x)\n        \n        return main_out, aux_out  # Return both outputs\n\n# Initialize and test model\nmodel = BiomassModel(num_targets=len(target_cols))\nmodel = model.to(CONFIG['device'])\n\n# Test with dummy input\ndummy_input = torch.randn(2, 3, CONFIG['img_size'], CONFIG['img_size']).to(CONFIG['device'])\nmain_out, aux_out = model(dummy_input)\nprint(f\"\\nModel: {CONFIG['model_name']}\")\nprint(f\"Device: {CONFIG['device']}\")\nprint(f\"Main output shape: {main_out.shape} (expected: [2, {len(target_cols)}])\")\nprint(f\"Auxiliary output shape: {aux_out.shape} (expected: [2, 1])\")\n\nprint(f\"\\nModel ready for training\")\n\n# Count parameters\ntotal_params = sum(p.numel() for p in model.parameters())\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\nprint(f\"Total parameters: {total_params:,}\")\nprint(f\"Trainable parameters: {trainable_params:,}\")\nprint(f\"Frozen parameters: {total_params - trainable_params:,}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:53.883141Z","iopub.execute_input":"2026-02-08T09:25:53.883339Z","iopub.status.idle":"2026-02-08T09:25:54.440453Z","shell.execute_reply.started":"2026-02-08T09:25:53.883322Z","shell.execute_reply":"2026-02-08T09:25:54.439706Z"}},"outputs":[],"execution_count":null},{"id":"a43f6360","cell_type":"markdown","source":"## Phase 4: Training & Optimization","metadata":{}},{"id":"2338a3aa","cell_type":"code","source":"# Training configuration\nprint(\"Training Configuration:\")\nfor key, value in CONFIG.items():\n    if key not in ['base_path', 'save_path', 'model_path']:  # Skip path objects\n        print(f\"  {key}: {value}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:54.441467Z","iopub.execute_input":"2026-02-08T09:25:54.441997Z","iopub.status.idle":"2026-02-08T09:25:54.446266Z","shell.execute_reply.started":"2026-02-08T09:25:54.441969Z","shell.execute_reply":"2026-02-08T09:25:54.445725Z"}},"outputs":[],"execution_count":null},{"id":"76cace40","cell_type":"code","source":"# Training and validation functions\ndef train_one_epoch(model, loader, criterion, optimizer, scheduler, device):\n    model.train()\n    running_loss = 0.0\n    \n    for images, targets in loader:\n        images = images.to(device)\n        targets = targets.to(device)\n        \n        optimizer.zero_grad()\n        \n        # Model returns (main_output, aux_output)\n        main_out, aux_out = model(images)\n        \n        # Main loss: predict 5 biomass components\n        main_loss = criterion(main_out, targets)\n        \n        # Auxiliary loss: predict total biomass (sum of components)\n        total_biomass = targets.sum(dim=1, keepdim=True)\n        aux_loss = criterion(aux_out, total_biomass)\n        \n        # Combined loss (main + 0.3 * auxiliary)\n        loss = main_loss + 0.3 * aux_loss\n        \n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n        \n        running_loss += loss.item() * images.size(0)\n    \n    epoch_loss = running_loss / len(loader.dataset)\n    return epoch_loss\n\ndef validate(model, loader, criterion, device):\n    model.eval()\n    running_loss = 0.0\n    all_preds = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for images, targets in loader:\n            images = images.to(device)\n            targets = targets.to(device)\n            \n            # Model returns (main_output, aux_output)\n            main_out, aux_out = model(images)\n            \n            # Compute combined loss\n            main_loss = criterion(main_out, targets)\n            total_biomass = targets.sum(dim=1, keepdim=True)\n            aux_loss = criterion(aux_out, total_biomass)\n            loss = main_loss + 0.3 * aux_loss\n            \n            running_loss += loss.item() * images.size(0)\n            all_preds.append(main_out.cpu().numpy())  # Use main output for predictions\n            all_targets.append(targets.cpu().numpy())\n    \n    epoch_loss = running_loss / len(loader.dataset)\n    all_preds = np.concatenate(all_preds)\n    all_targets = np.concatenate(all_targets)\n    \n    # Calculate RMSE for each target\n    rmse_per_target = []\n    for i in range(all_targets.shape[1]):\n        rmse = np.sqrt(mean_squared_error(all_targets[:, i], all_preds[:, i]))\n        rmse_per_target.append(rmse)\n    \n    return epoch_loss, rmse_per_target\n\nprint(\"Training functions defined\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:54.448787Z","iopub.execute_input":"2026-02-08T09:25:54.449060Z","iopub.status.idle":"2026-02-08T09:25:54.466826Z","shell.execute_reply.started":"2026-02-08T09:25:54.449042Z","shell.execute_reply":"2026-02-08T09:25:54.466148Z"}},"outputs":[],"execution_count":null},{"id":"02494b5e","cell_type":"code","source":"# Prepare data for selected fold\nfold = CONFIG['fold_to_train']\ntrain_df = train_pivot[train_pivot['fold'] != fold].reset_index(drop=True)\nvalid_df = train_pivot[train_pivot['fold'] == fold].reset_index(drop=True)\n\nprint(f\"Training on Fold {fold}\")\nprint(f\"  Train samples: {len(train_df)}\")\nprint(f\"  Valid samples: {len(valid_df)}\")\n\n# Create datasets\ntrain_dataset = BiomassDataset(\n    df=train_df,\n    base_path=BASE_PATH,\n    target_cols=target_cols,\n    transforms=get_train_transforms(CONFIG['img_size']),\n    use_tiling=True\n)\n\nvalid_dataset = BiomassDataset(\n    df=valid_df,\n    base_path=BASE_PATH,\n    target_cols=target_cols,\n    transforms=get_valid_transforms(CONFIG['img_size']),\n    use_tiling=True\n)\n\n# Create dataloaders\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=CONFIG['batch_size'],\n    shuffle=True,\n    num_workers=0,  # Set to 0 for macOS compatibility\n    pin_memory=False\n)\n\nvalid_loader = DataLoader(\n    valid_dataset,\n    batch_size=CONFIG['batch_size'],\n    shuffle=False,\n    num_workers=0,\n    pin_memory=False\n)\n\nprint(f\"\\nTrain batches: {len(train_loader)}\")\nprint(f\"Valid batches: {len(valid_loader)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:54.467683Z","iopub.execute_input":"2026-02-08T09:25:54.468288Z","iopub.status.idle":"2026-02-08T09:25:54.489323Z","shell.execute_reply.started":"2026-02-08T09:25:54.468269Z","shell.execute_reply":"2026-02-08T09:25:54.488786Z"}},"outputs":[],"execution_count":null},{"id":"a78906b9","cell_type":"markdown","source":"Using HuberLoss instead of MSE becuase it handles outliers better. Saw some extreme values in the data so this should help.","metadata":{}},{"id":"66e5af4d","cell_type":"markdown","source":"OneCycleLR schedular increases learning rate then decreases it. Learned from fastai that this trains faster than constant learning rate.","metadata":{}},{"id":"7e30df41","cell_type":"code","source":"# Initialize model, loss, optimizer, scheduler\nmodel = BiomassModel(num_targets=len(target_cols))\nmodel = model.to(CONFIG['device'])\n\n# HuberLoss for handling outliers\n# Why HuberLoss? Tried MSE first but it's sensitive to outliers\n# Saw some extreme values in EDA - HuberLoss more robust\n# Reference: https://pytorch.org/docs/stable/generated/torch.nn.HuberLoss.html\ncriterion = nn.HuberLoss(delta=CONFIG['huber_delta'])\n\n# AdamW optimizer with weight decay\n# Tried SGD but AdamW converged faster\n# weight_decay=1e-2 acts as L2 regularization\noptimizer = optim.AdamW(\n    model.parameters(),\n    lr=CONFIG['lr'],\n    weight_decay=CONFIG['weight_decay']\n)\n\n# OneCycleLR scheduler - learned from Fast.ai\n# Starts low, peaks at middle epochs, then decreases\n# Better than constant LR or StepLR (tested both)\nscheduler = OneCycleLR(\n    optimizer,\n    max_lr=CONFIG['lr'],\n    epochs=CONFIG['epochs'],\n    steps_per_epoch=len(train_loader),\n    pct_start=CONFIG['scheduler_pct_start']\n)\n\nprint(\"Model, loss, optimizer, and schedular initialized\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:54.490023Z","iopub.execute_input":"2026-02-08T09:25:54.490251Z","iopub.status.idle":"2026-02-08T09:25:54.969705Z","shell.execute_reply.started":"2026-02-08T09:25:54.490234Z","shell.execute_reply":"2026-02-08T09:25:54.968953Z"}},"outputs":[],"execution_count":null},{"id":"d771be77","cell_type":"code","source":"# Training loop\nhistory = {\n    'train_loss': [],\n    'valid_loss': [],\n    'valid_rmse': []\n}\n\nbest_loss = float('inf')\nbest_model_path = MODEL_PATH / f'best_model_fold{fold}.h5'\n\nprint(\"=\"*70)\nprint(f\"Starting Training - Fold {fold}\")\nprint(\"=\"*70)\n\nfor epoch in range(CONFIG['epochs']):\n    # Train\n    train_loss = train_one_epoch(model, train_loader, criterion, optimizer, scheduler, CONFIG['device'])\n    \n    # Validate\n    valid_loss, rmse_per_target = validate(model, valid_loader, criterion, CONFIG['device'])\n    \n    # Save history\n    history['train_loss'].append(train_loss)\n    history['valid_loss'].append(valid_loss)\n    history['valid_rmse'].append(rmse_per_target)\n    \n    # Print progress\n    print(f\"Epoch {epoch+1}/{CONFIG['epochs']}:\")\n    print(f\"  Train Loss: {train_loss:.4f}\")\n    print(f\"  Valid Loss: {valid_loss:.4f}\")\n    print(f\"  RMSE per target: {[f'{r:.2f}' for r in rmse_per_target]}\")\n    print(f\"  Avg RMSE: {np.mean(rmse_per_target):.4f}\")\n    \n    # Save best model\n    if valid_loss < best_loss:\n        best_loss = valid_loss\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'loss': best_loss,\n            'rmse': rmse_per_target\n        }, best_model_path)\n        print(f\"  Saved best model (loss: {best_loss:.4f})\")\n    print()\n\nprint(\"=\"*70)\nprint(\"Training Completed!\")\nprint(f\"Best validation loss: {best_loss:.4f}\")\nprint(\"=\"*70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T09:25:54.970812Z","iopub.execute_input":"2026-02-08T09:25:54.971204Z","iopub.status.idle":"2026-02-08T10:03:57.810002Z","shell.execute_reply.started":"2026-02-08T09:25:54.971168Z","shell.execute_reply":"2026-02-08T10:03:57.809181Z"}},"outputs":[],"execution_count":null},{"id":"0fb82fe9","cell_type":"code","source":"# Plot training history\nfig, axes = plt.subplots(1, 2, figsize=(15, 5))\n\n# Loss curves\naxes[0].plot(history['train_loss'], label='Train Loss', linewidth=2)\naxes[0].plot(history['valid_loss'], label='Valid Loss', linewidth=2)\naxes[0].set_xlabel('Epoch')\naxes[0].set_ylabel('Loss (Huber)')\naxes[0].set_title('Training and Validation Loss')\naxes[0].legend()\naxes[0].grid(True, alpha=0.3)\n\n# RMSE per target over epochs\nrmse_array = np.array(history['valid_rmse'])\nfor i, col in enumerate(target_cols):\n    axes[1].plot(rmse_array[:, i], label=col, linewidth=2)\naxes[1].set_xlabel('Epoch')\naxes[1].set_ylabel('RMSE')\naxes[1].set_title('RMSE per Target')\naxes[1].legend()\naxes[1].grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.savefig(MODEL_PATH / f'training_history_fold{fold}.png', dpi=150, bbox_inches='tight')\nplt.show()\n\nprint(f\"Saved training history plot to {MODEL_PATH / f'training_history_fold{fold}.png'}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T10:03:57.811173Z","iopub.execute_input":"2026-02-08T10:03:57.811464Z","iopub.status.idle":"2026-02-08T10:03:58.487038Z","shell.execute_reply.started":"2026-02-08T10:03:57.811440Z","shell.execute_reply":"2026-02-08T10:03:58.486395Z"}},"outputs":[],"execution_count":null},{"id":"38d61088","cell_type":"markdown","source":"## Phase 5: Inference & Submission","metadata":{}},{"id":"b8f10d1b","cell_type":"markdown","source":"TTA means making predictions multiple times with different augmentations then averaging them. Using 6 iterations becuase it improves accuracy.","metadata":{}},{"id":"ef7d95e5","cell_type":"code","source":"# Load test data\ntest_df = pd.read_csv(BASE_PATH / 'test.csv')\nprint(f\"Test data shape: {test_df.shape}\")\nprint(f\"Columns: {test_df.columns.tolist()}\")\ndisplay(test_df.head())\n\n# Get unique image paths\nunique_test_paths = test_df['image_path'].unique()\nprint(f\"\\nUnique test images: {len(unique_test_paths)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T10:03:58.487901Z","iopub.execute_input":"2026-02-08T10:03:58.488233Z","iopub.status.idle":"2026-02-08T10:03:58.501947Z","shell.execute_reply.started":"2026-02-08T10:03:58.488199Z","shell.execute_reply":"2026-02-08T10:03:58.501376Z"}},"outputs":[],"execution_count":null},{"id":"5bb4c0fa","cell_type":"code","source":"# Inference with Test Time Augmentation (TTA)\ndef predict_with_tta(model, image_path, transforms, device, n_tta=4):\n    \"\"\"\n    Apply TTA by flipping and averaging predictions.\n    \"\"\"\n    model.eval()\n    predictions = []\n    \n    # Get tiles\n    tiles = tile_image(image_path)\n    \n    with torch.no_grad():\n        for _ in range(n_tta):\n            tile_preds = []\n            for tile in tiles:\n                tile_np = np.array(tile)\n                \n                # Apply transforms\n                augmented = transforms(image=tile_np)\n                tile_tensor = augmented['image'].unsqueeze(0).to(device)\n                \n                # Predict - model returns (main_out, aux_out)\n                main_out, _ = model(tile_tensor)\n                tile_preds.append(main_out.cpu().numpy())\n            \n            # Average across tiles\n            avg_pred = np.mean(tile_preds, axis=0)\n            predictions.append(avg_pred)\n    \n    # Average across TTA iterations\n    final_pred = np.mean(predictions, axis=0)\n    return final_pred.squeeze()\n\nprint(\"TTA prediction function defined\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T10:03:58.503029Z","iopub.execute_input":"2026-02-08T10:03:58.503301Z","iopub.status.idle":"2026-02-08T10:03:58.509334Z","shell.execute_reply.started":"2026-02-08T10:03:58.503275Z","shell.execute_reply":"2026-02-08T10:03:58.508626Z"}},"outputs":[],"execution_count":null},{"id":"e56d3d07","cell_type":"code","source":"# Load best model\nif not best_model_path.exists():\n    # If training was just done, use the path from training\n    best_model_path = MODEL_PATH / f'best_model_fold{fold}.h5'\n\ncheckpoint = torch.load(best_model_path, map_location=CONFIG['device'], weights_only=False)\nmodel.load_state_dict(checkpoint['model_state_dict'])\nmodel = model.to(CONFIG['device'])\nmodel.eval()\n\nprint(f\"Loaded best model from {best_model_path.name}\")\nprint(f\"  Epoch: {checkpoint['epoch']+1}\")\nprint(f\"  Best validation loss: {checkpoint['loss']:.4f}\")\nprint(f\"  RMSE: {[f'{r:.2f}' for r in checkpoint['rmse']]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T10:03:58.510145Z","iopub.execute_input":"2026-02-08T10:03:58.510350Z","iopub.status.idle":"2026-02-08T10:03:58.892149Z","shell.execute_reply.started":"2026-02-08T10:03:58.510333Z","shell.execute_reply":"2026-02-08T10:03:58.891355Z"}},"outputs":[],"execution_count":null},{"id":"e5a49b00","cell_type":"code","source":"# Generate predictions for test set\ntransforms = get_valid_transforms(CONFIG['img_size'])\npredictions_dict = {}\n\nprint(\"Generating predictions with TTA...\")\nfor img_path in unique_test_paths:\n    full_path = BASE_PATH / img_path\n    pred = predict_with_tta(model, full_path, transforms, CONFIG['device'], n_tta=CONFIG['n_tta'])\n    predictions_dict[img_path] = pred\n    \nprint(f\"\\nGenereated predictions for {len(predictions_dict)} images\")\n\n# Show sample predictions\nsample_path = list(predictions_dict.keys())[0]\nprint(f\"\\nSample prediction for {sample_path}:\")\nfor col, val in zip(target_cols, predictions_dict[sample_path]):\n    print(f\"  {col}: {val:.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T10:03:58.893156Z","iopub.execute_input":"2026-02-08T10:03:58.893521Z","iopub.status.idle":"2026-02-08T10:04:00.008887Z","shell.execute_reply.started":"2026-02-08T10:03:58.893499Z","shell.execute_reply":"2026-02-08T10:04:00.008284Z"}},"outputs":[],"execution_count":null},{"id":"45dadbf5","cell_type":"code","source":"# Prepare submission DataFrame\nsubmission_data = []\n\nfor _, row in test_df.iterrows():\n    img_path = row['image_path']\n    target_name = row['target_name']\n    \n    # Get predicted values\n    preds = predictions_dict[img_path]\n    \n    # Find which column corresponds to this target_name\n    if target_name in target_cols:\n        target_idx = target_cols.index(target_name)\n        pred_value = preds[target_idx]\n    else:\n        # If target_name doesn't match exactly, use first prediction\n        pred_value = preds[0]\n    \n    # Clip to ensure non-negative\n    pred_value = max(0.0, pred_value)\n    \n    submission_data.append({\n        'sample_id': row['sample_id'],\n        'target': pred_value\n    })\n\nsubmission_df = pd.DataFrame(submission_data)\n\nprint(f\"Submission shape: {submission_df.shape}\")\nprint(f\"Expected shape: ({len(test_df)}, 2)\")\nprint(f\"\\nFirst 10 rows:\")\ndisplay(submission_df.head(10))\n\n# Statistics\nprint(f\"\\nSubmission statistics:\")\nprint(submission_df['target'].describe())\nprint(f\"\\nNegative values: {(submission_df['target'] < 0).sum()}\")\nprint(f\"Null values: {submission_df.isnull().sum().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T10:04:00.009789Z","iopub.execute_input":"2026-02-08T10:04:00.010117Z","iopub.status.idle":"2026-02-08T10:04:00.024321Z","shell.execute_reply.started":"2026-02-08T10:04:00.010096Z","shell.execute_reply":"2026-02-08T10:04:00.023747Z"}},"outputs":[],"execution_count":null},{"id":"f0f40516","cell_type":"code","source":"# Save submission\nsubmission_path = Path('./submission.csv')\nsubmission_df.to_csv(submission_path, index=False)\n\nprint(\"=\"*70)\nprint(\"SUBMISSION READY!\")\nprint(\"=\"*70)\nprint(f\"Saved to: {submission_path}\")\nprint(f\"Total rows: {len(submission_df)}\")\nprint(f\"Columns: {submission_df.columns.tolist()}\")\nprint(f\"No negative values: {(submission_df['target'] < 0).sum() == 0}\")\nprint(f\"No null values: {submission_df.isnull().sum().sum() == 0}\")\nprint(\"=\"*70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T10:04:00.025069Z","iopub.execute_input":"2026-02-08T10:04:00.025353Z","iopub.status.idle":"2026-02-08T10:04:00.039426Z","shell.execute_reply.started":"2026-02-08T10:04:00.025326Z","shell.execute_reply":"2026-02-08T10:04:00.038720Z"}},"outputs":[],"execution_count":null}]}