{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Load & Initial Inspection","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:10.687897Z","iopub.execute_input":"2025-12-25T22:30:10.688186Z","iopub.status.idle":"2025-12-25T22:30:14.497258Z","shell.execute_reply.started":"2025-12-25T22:30:10.688158Z","shell.execute_reply":"2025-12-25T22:30:14.496200Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BASE_PATH = \"/kaggle/input/csiro-biomass\"\n\ntrain_df = pd.read_csv(f\"{BASE_PATH}/train.csv\")\ntest_df  = pd.read_csv(f\"{BASE_PATH}/test.csv\")\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:14.499496Z","iopub.execute_input":"2025-12-25T22:30:14.499887Z","iopub.status.idle":"2025-12-25T22:30:14.550533Z","shell.execute_reply.started":"2025-12-25T22:30:14.499858Z","shell.execute_reply":"2025-12-25T22:30:14.549227Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Structural Validation","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Column Overview","metadata":{}},{"cell_type":"code","source":"train_df.info()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:14.551550Z","iopub.execute_input":"2025-12-25T22:30:14.551822Z","iopub.status.idle":"2025-12-25T22:30:14.577362Z","shell.execute_reply.started":"2025-12-25T22:30:14.551796Z","shell.execute_reply":"2025-12-25T22:30:14.576491Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.2 Missing Values","metadata":{}},{"cell_type":"code","source":"train_df.isna().sum().sort_values(ascending=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:14.578535Z","iopub.execute_input":"2025-12-25T22:30:14.579568Z","iopub.status.idle":"2025-12-25T22:30:14.597139Z","shell.execute_reply.started":"2025-12-25T22:30:14.579530Z","shell.execute_reply":"2025-12-25T22:30:14.596197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Target Variable EDA","metadata":{}},{"cell_type":"markdown","source":"## 3.1 Target Distribution by Biomass Type","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nfor i, t in enumerate(train_df[\"target_name\"].unique()):\n    plt.subplot(2,3,i+1)\n    subset = train_df[train_df[\"target_name\"] == t][\"target\"]\n    plt.hist(subset, bins=30)\n    plt.title(t)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:14.598201Z","iopub.execute_input":"2025-12-25T22:30:14.598506Z","iopub.status.idle":"2025-12-25T22:30:15.596735Z","shell.execute_reply.started":"2025-12-25T22:30:14.598469Z","shell.execute_reply":"2025-12-25T22:30:15.595844Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2 Target Scale Comparison","metadata":{}},{"cell_type":"code","source":"train_df.groupby(\"target_name\")[\"target\"].describe()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:15.597985Z","iopub.execute_input":"2025-12-25T22:30:15.598326Z","iopub.status.idle":"2025-12-25T22:30:15.650402Z","shell.execute_reply.started":"2025-12-25T22:30:15.598292Z","shell.execute_reply":"2025-12-25T22:30:15.649487Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Image-Level Aggregation (One Row per Image)","metadata":{}},{"cell_type":"code","source":"pivot_df = train_df.pivot_table(\n    index=[\"sample_id\", \"image_path\", \"Sampling_Date\", \"State\",\n           \"Species\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\"],\n    columns=\"target_name\",\n    values=\"target\"\n).reset_index()\n\npivot_df.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:15.653347Z","iopub.execute_input":"2025-12-25T22:30:15.653661Z","iopub.status.idle":"2025-12-25T22:30:15.694077Z","shell.execute_reply.started":"2025-12-25T22:30:15.653633Z","shell.execute_reply":"2025-12-25T22:30:15.693301Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Metadata EDA","metadata":{}},{"cell_type":"markdown","source":"## 5.1 Sampling Date","metadata":{}},{"cell_type":"code","source":"# 1. Convert to datetime (if not already done)\npivot_df[\"Sampling_Date\"] = pd.to_datetime(pivot_df[\"Sampling_Date\"])\n\n# 2. Extract Year, Month, and Day Name\npivot_df[\"Year\"] = pivot_df[\"Sampling_Date\"].dt.year\npivot_df[\"Month\"] = pivot_df[\"Sampling_Date\"].dt.month_name()\npivot_df[\"Day_of_Week\"] = pivot_df[\"Sampling_Date\"].dt.day_name()\n\n# 3. Visualization: Samples by Month-Year\n# We create a 'Year-Month' string for sorting and plotting\npivot_df[\"Year_Month\"] = pivot_df[\"Sampling_Date\"].dt.to_period(\"M\")\n\nplt.figure(figsize=(20, 6))\npivot_df[\"Year_Month\"].value_counts().sort_index().plot(kind=\"bar\", color=\"skyblue\")\nplt.title(\"Sampling Frequency by Month and Year\")\nplt.xlabel(\"Month-Year\")\nplt.ylabel(\"Count\")\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()\n\n# 4. Visualization: Samples by Day of the Week\nplt.figure(figsize=(12, 6))\npivot_df[\"Day_of_Week\"].value_counts().reindex([\n    'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'\n]).plot(kind=\"bar\", color=\"salmon\")\nplt.title(\"Sampling Distribution by Day of Week\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:32:08.113441Z","iopub.execute_input":"2025-12-25T22:32:08.114066Z","iopub.status.idle":"2025-12-25T22:32:08.478113Z","shell.execute_reply.started":"2025-12-25T22:32:08.114034Z","shell.execute_reply":"2025-12-25T22:32:08.477223Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5.2 Geographic Distribution (State)","metadata":{}},{"cell_type":"code","source":"pivot_df[\"State\"].value_counts().plot(kind=\"bar\")\nplt.title(\"Samples by Australian State\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:15.895119Z","iopub.execute_input":"2025-12-25T22:30:15.895478Z","iopub.status.idle":"2025-12-25T22:30:16.029386Z","shell.execute_reply.started":"2025-12-25T22:30:15.895410Z","shell.execute_reply":"2025-12-25T22:30:16.028499Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5.3 Species Composition","metadata":{}},{"cell_type":"code","source":"pivot_df[\"Species\"].str.split(\"_\").explode().value_counts().head(10)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:16.030548Z","iopub.execute_input":"2025-12-25T22:30:16.030973Z","iopub.status.idle":"2025-12-25T22:30:16.041508Z","shell.execute_reply.started":"2025-12-25T22:30:16.030938Z","shell.execute_reply":"2025-12-25T22:30:16.040732Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5.4 NDVI & Height","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(12,4))\n\nsns.histplot(pivot_df[\"Pre_GSHH_NDVI\"], ax=ax[0])\nax[0].set_title(\"NDVI Distribution\")\n\nsns.histplot(pivot_df[\"Height_Ave_cm\"], ax=ax[1])\nax[1].set_title(\"Average Height Distribution\")\n\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:16.042583Z","iopub.execute_input":"2025-12-25T22:30:16.042923Z","iopub.status.idle":"2025-12-25T22:30:16.463317Z","shell.execute_reply.started":"2025-12-25T22:30:16.042888Z","shell.execute_reply":"2025-12-25T22:30:16.462627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Correlation Analysis","metadata":{}},{"cell_type":"markdown","source":"## 6.1 Target–Target Correlation","metadata":{}},{"cell_type":"code","source":"targets = [\n    \"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\",\n    \"GDM_g\", \"Dry_Total_g\"\n]\n\nsns.heatmap(pivot_df[targets].corr(), annot=True, cmap=\"coolwarm\")\nplt.title(\"Biomass Target Correlation\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:16.464289Z","iopub.execute_input":"2025-12-25T22:30:16.464583Z","iopub.status.idle":"2025-12-25T22:30:16.683409Z","shell.execute_reply.started":"2025-12-25T22:30:16.464552Z","shell.execute_reply":"2025-12-25T22:30:16.682568Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6.2 Metadata vs Targets","metadata":{}},{"cell_type":"code","source":"sns.pairplot(\n    pivot_df,\n    vars=[\"Pre_GSHH_NDVI\", \"Height_Ave_cm\"],\n    hue=None\n)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:16.684623Z","iopub.execute_input":"2025-12-25T22:30:16.684969Z","iopub.status.idle":"2025-12-25T22:30:17.342030Z","shell.execute_reply.started":"2025-12-25T22:30:16.684935Z","shell.execute_reply":"2025-12-25T22:30:17.341208Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. Image EDA","metadata":{}},{"cell_type":"markdown","source":"## 7.1 Image Existence Check","metadata":{}},{"cell_type":"code","source":"IMAGE_DIR = f\"{BASE_PATH}/train\"\n\nmissing = []\nfor p in pivot_df[\"image_path\"]:\n    if not os.path.exists(os.path.join(BASE_PATH, p)):\n        missing.append(p)\n\nprint(\"Missing images:\", len(missing))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:17.343039Z","iopub.execute_input":"2025-12-25T22:30:17.343322Z","iopub.status.idle":"2025-12-25T22:30:17.769783Z","shell.execute_reply.started":"2025-12-25T22:30:17.343297Z","shell.execute_reply":"2025-12-25T22:30:17.768973Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.2 Image Size & Mode Consistency","metadata":{}},{"cell_type":"code","source":"from collections import Counter\n\nsizes, modes = [], []\n\nfor p in pivot_df[\"image_path\"].sample(200, random_state=42):\n    img = Image.open(os.path.join(BASE_PATH, p))\n    sizes.append(img.size)\n    modes.append(img.mode)\n\nCounter(sizes).most_common(5), Counter(modes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:17.771065Z","iopub.execute_input":"2025-12-25T22:30:17.771617Z","iopub.status.idle":"2025-12-25T22:30:19.128825Z","shell.execute_reply.started":"2025-12-25T22:30:17.771589Z","shell.execute_reply":"2025-12-25T22:30:19.128113Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.3 Visual Inspection","metadata":{}},{"cell_type":"code","source":"def show_images(df, n=9):\n    sample = df.sample(n)\n    plt.figure(figsize=(9,9))\n\n    for i, p in enumerate(sample[\"image_path\"]):\n        img = Image.open(os.path.join(BASE_PATH, p)).convert(\"RGB\")\n        plt.subplot(3,3,i+1)\n        plt.imshow(img)\n        plt.axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()\n\nshow_images(pivot_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:19.129800Z","iopub.execute_input":"2025-12-25T22:30:19.130087Z","iopub.status.idle":"2025-12-25T22:30:21.222616Z","shell.execute_reply.started":"2025-12-25T22:30:19.130063Z","shell.execute_reply":"2025-12-25T22:30:21.221524Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7.4 Pixel Intensity Diagnostics","metadata":{}},{"cell_type":"code","source":"pixel_means = []\n\nfor p in pivot_df[\"image_path\"].sample(300):\n    img = np.array(Image.open(os.path.join(BASE_PATH, p)).convert(\"RGB\"))\n    pixel_means.append(img.mean())\n\nplt.hist(pixel_means, bins=30)\nplt.title(\"Mean Pixel Intensity Distribution\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:21.223734Z","iopub.execute_input":"2025-12-25T22:30:21.224051Z","iopub.status.idle":"2025-12-25T22:30:43.176841Z","shell.execute_reply.started":"2025-12-25T22:30:21.224013Z","shell.execute_reply":"2025-12-25T22:30:43.176052Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. Train–Test Consistency Check","metadata":{}},{"cell_type":"code","source":"print(\"Train targets:\", train_df[\"target_name\"].unique())\nprint(\"Test targets:\", test_df[\"target_name\"].unique())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T22:30:43.177955Z","iopub.execute_input":"2025-12-25T22:30:43.178246Z","iopub.status.idle":"2025-12-25T22:30:43.184178Z","shell.execute_reply.started":"2025-12-25T22:30:43.178221Z","shell.execute_reply":"2025-12-25T22:30:43.183380Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 9. Key Takeaways from Exploratory Data Analysis (EDA)","metadata":{}},{"cell_type":"markdown","source":"---\n\n## 1. Dataset Structure and Integrity\n\n* The training dataset contains **1,785 rows and 9 variables**, representing **357 unique images**, each repeated across **five biomass targets**.\n* The test dataset follows the same target schema, ensuring **full train–test compatibility**.\n* **No missing values** were detected across any feature or target, indicating high data quality and no need for imputation.\n* All referenced image files exist and are accessible, eliminating risks of silent data-loading failures during training.\n\n---\n\n## 2. Target Variable Characteristics\n\n* Each biomass component (`Dry_Green_g`, `Dry_Dead_g`, `Dry_Clover_g`, `GDM_g`, `Dry_Total_g`) has **exactly 357 observations**, confirming a **balanced target distribution**.\n* Biomass values exhibit:\n\n  * **Strong right skew**\n  * **Zero-inflation**, particularly for `Dry_Clover_g`\n  * **Heterogeneous scales**, with means ranging from ~6 g to ~45 g\n* These properties necessitate **target normalization or transformation** prior to regression modeling.\n\n---\n\n## 3. Independence of Biomass Targets\n\n* Pairwise correlation analysis reveals **near-zero correlation among all biomass targets**.\n\n### Interpretation\n\n* Each biomass component behaves as an **independent regression problem**.\n* `Dry_Total_g` is **not a linear combination** of the other components in this dataset.\n* Shared modeling does **not benefit from target interdependence**, but rather from **shared visual features**.\n\n### Modeling Implication\n\n* A **shared CNN backbone with independent output heads** is appropriate.\n* Multi-task learning will rely on **representation sharing**, not target correlation.\n\n---\n\n## 4. Metadata Insights\n\n* Pasture species distribution is **highly imbalanced**, with Clover and Ryegrass dominating the dataset.\n* This imbalance may introduce **species-specific prediction bias**, particularly for underrepresented species.\n* NDVI (`Pre_GSHH_NDVI`) and pasture height (`Height_Ave_cm`) are continuous, well-distributed variables and represent **strong auxiliary signals** for biomass estimation.\n\n---\n\n## 5. Image Characteristics\n\n* All images are:\n\n  * Uniformly sized (**2000 × 1000 pixels**)\n  * RGB format\n* The absence of variation in image resolution simplifies preprocessing and allows **fixed-size resizing** without distortion concerns.\n* Visual inspection confirms:\n\n  * Variability in lighting and background\n  * Presence of natural noise typical of field conditions\n\n### Implication\n\n* Image augmentation (brightness, contrast, random crop) is **recommended and justified**.\n\n---\n\n## 6. Train–Test Consistency\n\n* The same five biomass targets are present in both training and test datasets.\n* This confirms **no target leakage or schema mismatch**, enabling straightforward inference and submission.\n\n---\n\n## 7. Overall Readiness for Modeling\n\n* The dataset is **clean, consistent, and modeling-ready**.\n* Key challenges lie in:\n\n  * Non-linear relationships\n  * Target heterogeneity\n  * Species imbalance\n* The problem is best framed as **multi-output, non-linear regression using CNN-based feature extraction**.\n\n---\n\n## Summary Statement (One-Line)\n\n> *The EDA reveals a clean, well-structured multimodal dataset with independent biomass targets, uniform high-resolution imagery, and strong auxiliary signals, making it suitable for CNN-based multi-output regression with normalized targets and robust augmentation strategies.*\n\n---","metadata":{}}]}