{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:04.498826Z","iopub.execute_input":"2025-12-08T21:22:04.499637Z","iopub.status.idle":"2025-12-08T21:22:04.885220Z","shell.execute_reply.started":"2025-12-08T21:22:04.499590Z","shell.execute_reply":"2025-12-08T21:22:04.884226Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# R² Calculation","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# weights used in competition\nTARGET_WEIGHTS = {\n    \"Dry_Green_g\": 0.1,\n    \"Dry_Dead_g\": 0.1,\n    \"Dry_Clover_g\": 0.1,\n    \"GDM_g\": 0.2,\n    \"Dry_Total_g\": 0.5\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:04.887134Z","iopub.execute_input":"2025-12-08T21:22:04.887706Z","iopub.status.idle":"2025-12-08T21:22:04.893355Z","shell.execute_reply.started":"2025-12-08T21:22:04.887676Z","shell.execute_reply":"2025-12-08T21:22:04.891869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def weighted_r2(df, pred_col=\"pred\", true_col=\"target\", target_col=\"target_name\"):\n    y_true = df[true_col].values\n    y_pred = df[pred_col].values\n    \n    w = train.target_name.map(lambda x: TARGET_WEIGHTS[x])\n    y_w = sum(y_true*w)/sum(w)\n    \n    ss_res = sum(w*(y_true-y_pred)**2)\n    ss_tot = sum(w*(y_true-y_w)**2)\n\n    R = 1 - (ss_res/ss_tot)\n    return R","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:04.894387Z","iopub.execute_input":"2025-12-08T21:22:04.894718Z","iopub.status.idle":"2025-12-08T21:22:04.925303Z","shell.execute_reply.started":"2025-12-08T21:22:04.894684Z","shell.execute_reply":"2025-12-08T21:22:04.923634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# PART 1 — Quick Overview","metadata":{}},{"cell_type":"code","source":"train_path = '/kaggle/input/csiro-biomass/train.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:04.928755Z","iopub.execute_input":"2025-12-08T21:22:04.929142Z","iopub.status.idle":"2025-12-08T21:22:04.951375Z","shell.execute_reply.started":"2025-12-08T21:22:04.929109Z","shell.execute_reply":"2025-12-08T21:22:04.949672Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(train_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:04.953060Z","iopub.execute_input":"2025-12-08T21:22:04.953361Z","iopub.status.idle":"2025-12-08T21:22:05.009790Z","shell.execute_reply.started":"2025-12-08T21:22:04.953338Z","shell.execute_reply":"2025-12-08T21:22:05.008846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.shape)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:05.010636Z","iopub.execute_input":"2025-12-08T21:22:05.010888Z","iopub.status.idle":"2025-12-08T21:22:05.047437Z","shell.execute_reply.started":"2025-12-08T21:22:05.010869Z","shell.execute_reply":"2025-12-08T21:22:05.046414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.image_path.nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:05.048508Z","iopub.execute_input":"2025-12-08T21:22:05.049005Z","iopub.status.idle":"2025-12-08T21:22:05.060261Z","shell.execute_reply.started":"2025-12-08T21:22:05.048969Z","shell.execute_reply":"2025-12-08T21:22:05.059166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"1785/357","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:05.061474Z","iopub.execute_input":"2025-12-08T21:22:05.061762Z","iopub.status.idle":"2025-12-08T21:22:05.081615Z","shell.execute_reply.started":"2025-12-08T21:22:05.061737Z","shell.execute_reply":"2025-12-08T21:22:05.080778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:05.082856Z","iopub.execute_input":"2025-12-08T21:22:05.083205Z","iopub.status.idle":"2025-12-08T21:22:05.109980Z","shell.execute_reply.started":"2025-12-08T21:22:05.083182Z","shell.execute_reply":"2025-12-08T21:22:05.108966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:05.112385Z","iopub.execute_input":"2025-12-08T21:22:05.112676Z","iopub.status.idle":"2025-12-08T21:22:05.138636Z","shell.execute_reply.started":"2025-12-08T21:22:05.112653Z","shell.execute_reply":"2025-12-08T21:22:05.137217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()\ntrain.describe(include='all')\ntrain.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:05.139786Z","iopub.execute_input":"2025-12-08T21:22:05.140077Z","iopub.status.idle":"2025-12-08T21:22:05.212275Z","shell.execute_reply.started":"2025-12-08T21:22:05.140055Z","shell.execute_reply":"2025-12-08T21:22:05.211144Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# PART 2 — Target Variable Analysis","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:05.213463Z","iopub.execute_input":"2025-12-08T21:22:05.213777Z","iopub.status.idle":"2025-12-08T21:22:05.218388Z","shell.execute_reply.started":"2025-12-08T21:22:05.213746Z","shell.execute_reply":"2025-12-08T21:22:05.217442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(12,6))\nsns.histplot(data=train, x=\"target\", kde=True, bins=40)\nplt.title(\"Distribution of target\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:05.219328Z","iopub.execute_input":"2025-12-08T21:22:05.219575Z","iopub.status.idle":"2025-12-08T21:22:06.847787Z","shell.execute_reply.started":"2025-12-08T21:22:05.219555Z","shell.execute_reply":"2025-12-08T21:22:06.845999Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Interpretation of the Target Distribution (Based on the Plot)**\n\n* The distribution is heavily right-skewed\n\nMost target values are small (0–20 g), and only a small number of samples have very large biomass values (100–180+ g).\nThis long right tail indicates strong positive skewness.\nThis is typical for biomass datasets where:\nMany plots have low biomass (young plants, winter sampling)\nOnly some plots reach very high biomass (dense/mature vegetation)\n\n* A large peak occurs near zero\n\nThe first bars show a very high concentration of values close to zero, meaning:\nSome targets (like Dry_Clover_g and Dry_Dead_g) are frequently zero or near-zero\nEven Dry_Green_g has many small values during early growth periods\nThis suggests a zero-inflated structure, which affects modeling.\n\n* As the target increases, the number of observations declines smoothly\n\nFrom ~20 g onward:\nCounts steadily decrease\nThe KDE curve shows a gradual decay.\nHigh-biomass samples are rare.\nThis indicates:\n\n1.     Predicting high values is harder (less data to learn from)\n\n2. Outliers are biologically real, not noise\n\n\n* The range is wide: from 0 up to ~180 g\n\nThe target spans nearly two orders of magnitude.\nThis means:\nThe model must learn relationships across very different scales.\nLoss functions may overweight large errors from high-biomass samples.\nThis is why the competition uses weighted R².\n\n* Log-transforming targets will likely improve model performance\n\nGiven the clear exponential decay pattern, using:\ny_log = np.log1p(target)\n\nwould:\nMake the distribution more Gaussian.\nReduce the influence of outliers.\nStabilize gradients for neural networks.\nImprove performance for linear models, gradient boosting, and CNN regressors.\nThis matches what most top solutions do in biomass prediction tasks.\n\n**Summary** \n\nThe target distribution is strongly right-skewed, with a large concentration of small biomass values near zero and a long decreasing tail extending beyond 150 g. This indicates substantial heterogeneity in biomass levels, typical of vegetation datasets where many plots contain sparse biomass and only a minority reach very high densities. The wide range and skewness suggest that modeling will benefit from log-transformations, robust loss functions, and careful handling of outliers.","metadata":{}},{"cell_type":"code","source":"train.target_name.unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:06.849304Z","iopub.execute_input":"2025-12-08T21:22:06.849754Z","iopub.status.idle":"2025-12-08T21:22:06.857579Z","shell.execute_reply.started":"2025-12-08T21:22:06.849729Z","shell.execute_reply":"2025-12-08T21:22:06.856414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- **Dry_Total_g** is the “parent” variable — it includes *every biomass component*.  \n- **Dry_Green_g**, **Dry_Dead_g**, and **Dry_Clover_g** sum up (approximately) to Dry_Total_g.\n- **GDM_g** represents *all green biomass* and overlaps with:\n  - Dry_Green_g  \n  - The green portion of Clover  \n\nSo in simple terms:\n\n- **Dry_Total_g** = Green + Dead + Clover  \n- **GDM_g** = Green + Green Clover  \n- **Dry_Green_g** = Green biomass excluding clover  \n\nThus:\n- **GDM_g** is a *subset* of Total  \n- **Green and Clover contribute to both GDM and Dry_Total**  \n- **Dead contributes only to Total**\n","metadata":{}},{"cell_type":"code","source":"# 1. Extract image_id from sample_id\ntrain[\"image_id\"] = train[\"sample_id\"].str.split(\"__\").str[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:06.858884Z","iopub.execute_input":"2025-12-08T21:22:06.859199Z","iopub.status.idle":"2025-12-08T21:22:06.888442Z","shell.execute_reply.started":"2025-12-08T21:22:06.859177Z","shell.execute_reply":"2025-12-08T21:22:06.887259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Pivot to wide format: one row per sample_id, one column per target_name\nwide_train = train.pivot_table(\n    index=[\"image_id\", \"image_path\", \"Sampling_Date\", \"State\", \"Species\", \n           \"Pre_GSHH_NDVI\", \"Height_Ave_cm\"],\n    columns=\"target_name\",\n    values=\"target\"\n).reset_index()\n\nwide_train.set_index('image_id', inplace = True)\nwide_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:06.889605Z","iopub.execute_input":"2025-12-08T21:22:06.889952Z","iopub.status.idle":"2025-12-08T21:22:06.953597Z","shell.execute_reply.started":"2025-12-08T21:22:06.889898Z","shell.execute_reply":"2025-12-08T21:22:06.952150Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Make sure all expected columns exist\nexpected_cols = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\nprint(\"Missing columns:\", [c for c in expected_cols if c not in wide_train.columns])\n\n# 2. Check Equation 1:\n# Dry_Total_g ≈ Dry_Green_g + Dry_Dead_g + Dry_Clover_g\n\neq1_lhs = wide_train[\"Dry_Total_g\"]\neq1_rhs = wide_train[\"Dry_Green_g\"] + wide_train[\"Dry_Dead_g\"] + wide_train[\"Dry_Clover_g\"]\neq1_diff = eq1_lhs - eq1_rhs  # if perfect, should be ~0\n\nprint(\"\\n=== Equation 1: Dry_Total_g vs sum of components ===\")\nprint(\"Diff summary (Total - (Green + Dead + Clover)):\")\nprint(eq1_diff.describe())\n\n# how many are almost exact? (tolerance can be tuned)\ntolerance = 1e-3  # 0.001 g\neq1_within_tol = (eq1_diff.abs() <= tolerance).mean()\nprint(f\"Fraction of samples within {tolerance} g: {eq1_within_tol:.4f}\")\n\nprint(\"Max absolute difference:\", eq1_diff.abs().max())\n\n\n# 3. Check Equation 2:\n# GDM_g ≈ Dry_Green_g + Dry_Clover_g\n\neq2_lhs = wide_train[\"GDM_g\"]\neq2_rhs = wide_train[\"Dry_Green_g\"] + wide_train[\"Dry_Clover_g\"]\neq2_diff = eq2_lhs - eq2_rhs\n\nprint(\"\\n=== Equation 2: GDM_g vs Green + Clover ===\")\nprint(\"Diff summary (GDM - (Green + Clover)):\")\nprint(eq2_diff.describe())\n\neq2_within_tol = (eq2_diff.abs() <= tolerance).mean()\nprint(f\"Fraction of samples within {tolerance} g: {eq2_within_tol:.4f}\")\n\nprint(\"Max absolute difference:\", eq2_diff.abs().max())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:06.954852Z","iopub.execute_input":"2025-12-08T21:22:06.955244Z","iopub.status.idle":"2025-12-08T21:22:06.976276Z","shell.execute_reply.started":"2025-12-08T21:22:06.955211Z","shell.execute_reply":"2025-12-08T21:22:06.974983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(12,6))\nsns.histplot(data=train, x=\"target\", hue=\"target_name\", kde=True, bins=40)\nplt.title(\"Distribution of target (all targets)\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:06.980853Z","iopub.execute_input":"2025-12-08T21:22:06.981189Z","iopub.status.idle":"2025-12-08T21:22:07.703778Z","shell.execute_reply.started":"2025-12-08T21:22:06.981166Z","shell.execute_reply":"2025-12-08T21:22:07.702455Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 📊 Target Distribution Analysis (All Biomass Targets)\n\nThe distribution plot shows how each of the five biomass targets behaves across the dataset. Even though the targets measure different biomass components, they share several statistical patterns that influence model design.\n\n---\n\n**1. All targets are strongly right-skewed**\nAcross all target types:\n- Most observations are small.\n- A minority of samples extend toward very high values (long right tail).\n\nThis skewness suggests:\n- Log-transformations may improve model performance.\n- Outliers reflect real biological variation, not noise.\n\n---\n\n**2. Dry_Total_g has the largest range**\n- Extends beyond **150 g**, dominating the upper tail of the distribution.\n- Shows the smoothest long-tail decay.\n\nBecause it captures the *entire* biomass, it drives most of the variance in the dataset.\n\n➡️ This aligns with the competition assigning **0.5 weight** to Dry_Total_g in the metric.\n\n---\n\n**3. GDM_g closely tracks Dry_Total_g**\n- Similar shape, slightly smaller scale.\n- Indicates strong correlation with Dry_Total_g.\n\n➡️ Multitask models can exploit shared structure between these two targets.\n\n---\n\n**4. Dry_Green_g is moderate with a peak around 10–20 g**\n- Most values fall in the lower–mid range.\n- Tail extends up to ~60–80 g.\n\nThis target is influenced by plant phenology, NDVI, and growth season.\n\n---\n\n**5. Dry_Dead_g is heavily concentrated near zero**\n- Large spike at 0–5 g.\n- Moderate tail reaching ~40–50 g.\n\nThis reflects seasonal senescence—dead material increases later in the year.\n\n➡️ The distribution is **zero-inflated**.\n\n---\n\n**6. Dry_Clover_g is the smallest and most zero-heavy**\n- Extremely concentrated near zero.\n- Rarely exceeds 10–15 g.\n\nThis makes it the **hardest** target to predict reliably.\n\n---\n\n**📌 Summary**\nThe five biomass targets differ in scale but share right-skewed distributions.  \n- **Dry_Total_g** and **GDM_g** span the widest ranges.  \n- **Dry_Green_g** and **Dry_Dead_g** occupy intermediate ranges.  \n- **Dry_Clover_g** is tightly clustered near zero.  \n\nThese patterns imply:\n- Use log-transforms or robust losses.\n- Consider multitask learning for shared relationships.\n- Expect higher variability in predicting large biomass values.\n\n---\n","metadata":{}},{"cell_type":"markdown","source":"# PART 3 — Feature Analysis","metadata":{}},{"cell_type":"markdown","source":"## Species Effects","metadata":{}},{"cell_type":"code","source":"species_count = train.Species.value_counts()\nspecies_count.index\nspecies_count.values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:07.704819Z","iopub.execute_input":"2025-12-08T21:22:07.705099Z","iopub.status.idle":"2025-12-08T21:22:07.715129Z","shell.execute_reply.started":"2025-12-08T21:22:07.705077Z","shell.execute_reply":"2025-12-08T21:22:07.713653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"species_count","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:07.716493Z","iopub.execute_input":"2025-12-08T21:22:07.716878Z","iopub.status.idle":"2025-12-08T21:22:07.751130Z","shell.execute_reply.started":"2025-12-08T21:22:07.716852Z","shell.execute_reply":"2025-12-08T21:22:07.750044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"species_count = train.Species.value_counts()\n\nplt.figure(figsize=(14,6))\nax = sns.barplot(\n    x=species_count.index,\n    y=species_count.values,\n)\nax.bar_label(ax.containers[0])\nplt.xticks(rotation=75)\nplt.title(\"Count of each Species\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:07.752530Z","iopub.execute_input":"2025-12-08T21:22:07.753246Z","iopub.status.idle":"2025-12-08T21:22:08.192584Z","shell.execute_reply.started":"2025-12-08T21:22:07.753157Z","shell.execute_reply":"2025-12-08T21:22:08.191294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14,6))\nsns.barplot(data=train, x=\"Species\", y=\"target\", hue=\"target_name\")\nplt.xticks(rotation=75)\nplt.title(\"Species by Target\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:08.193497Z","iopub.execute_input":"2025-12-08T21:22:08.193780Z","iopub.status.idle":"2025-12-08T21:22:10.350268Z","shell.execute_reply.started":"2025-12-08T21:22:08.193756Z","shell.execute_reply":"2025-12-08T21:22:10.349266Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Date effects","metadata":{}},{"cell_type":"code","source":"wide_train[\"Sampling_Date\"] = pd.to_datetime(wide_train[\"Sampling_Date\"])\nwide_train[\"Month\"] = wide_train[\"Sampling_Date\"].dt.month\nwide_train[\"DayOfYear\"] = wide_train[\"Sampling_Date\"].dt.dayofyear","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:10.351993Z","iopub.execute_input":"2025-12-08T21:22:10.352445Z","iopub.status.idle":"2025-12-08T21:22:10.370227Z","shell.execute_reply.started":"2025-12-08T21:22:10.352411Z","shell.execute_reply":"2025-12-08T21:22:10.368972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"Sampling_Date\"] = pd.to_datetime(train[\"Sampling_Date\"])\ntrain[\"Month\"] = train[\"Sampling_Date\"].dt.month\ntrain[\"DayOfYear\"] = train[\"Sampling_Date\"].dt.dayofyear","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:25:06.581912Z","iopub.execute_input":"2025-12-08T21:25:06.582322Z","iopub.status.idle":"2025-12-08T21:25:06.594268Z","shell.execute_reply.started":"2025-12-08T21:25:06.582297Z","shell.execute_reply":"2025-12-08T21:25:06.593049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wide_train[\"Month\"].min(), wide_train[\"Month\"].max(), wide_train.Month.nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:10.371632Z","iopub.execute_input":"2025-12-08T21:22:10.372348Z","iopub.status.idle":"2025-12-08T21:22:10.406788Z","shell.execute_reply.started":"2025-12-08T21:22:10.372305Z","shell.execute_reply":"2025-12-08T21:22:10.405254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"month_counts = wide_train[\"Month\"].value_counts().sort_index()\nplt.figure(figsize=(14,6))\nax = sns.barplot(x=month_counts.index, y=month_counts.values)\nax.bar_label(ax.containers[0])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:10.408216Z","iopub.execute_input":"2025-12-08T21:22:10.408693Z","iopub.status.idle":"2025-12-08T21:22:10.686376Z","shell.execute_reply.started":"2025-12-08T21:22:10.408655Z","shell.execute_reply":"2025-12-08T21:22:10.685036Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**❗ Observations:**\n\n* No month 3 (March)\n* No month 12 (December) Sampling was simply not done in those periods → not an error.\n* Strong imbalance: months 5–11 dominate, months 1, 2, 4 are tiny.\n","metadata":{}},{"cell_type":"markdown","source":"### Add Season (categorical)","metadata":{}},{"cell_type":"code","source":"wide_train.State.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:22:10.688901Z","iopub.execute_input":"2025-12-08T21:22:10.689311Z","iopub.status.idle":"2025-12-08T21:22:10.702617Z","shell.execute_reply.started":"2025-12-08T21:22:10.689272Z","shell.execute_reply":"2025-12-08T21:22:10.699814Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Seasons follow the Southern Hemisphere calendar, because all states are from Australiia ","metadata":{}},{"cell_type":"code","source":"def get_season_southern(month):\n    if month in [12, 1, 2]:\n        return \"Summer\"\n    elif month in [3, 4, 5]:\n        return \"Fall\"\n    elif month in [6, 7, 8]:\n        return \"Winter\"\n    else:  # 9,10,11\n        return \"Spring\"\n\nwide_train[\"Season\"] = wide_train[\"Month\"].apply(get_season_southern)\ntrain[\"Season\"] = train[\"Month\"].apply(get_season_southern)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:25:54.228642Z","iopub.execute_input":"2025-12-08T21:25:54.229003Z","iopub.status.idle":"2025-12-08T21:25:54.238249Z","shell.execute_reply.started":"2025-12-08T21:25:54.228978Z","shell.execute_reply":"2025-12-08T21:25:54.237216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14,6))\nsns.boxplot(data=train, x=\"Month\", y=\"target\", hue=\"target_name\")\nplt.title(\"Target Value by Month and Target Type\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:25:16.491468Z","iopub.execute_input":"2025-12-08T21:25:16.491779Z","iopub.status.idle":"2025-12-08T21:25:17.289423Z","shell.execute_reply.started":"2025-12-08T21:25:16.491757Z","shell.execute_reply":"2025-12-08T21:25:17.288481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14,6))\nsns.boxplot(data=train, x=\"Month\", y=\"target\")\nplt.title(\"Target Value by Month and Target Type\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:25:17.674665Z","iopub.execute_input":"2025-12-08T21:25:17.675024Z","iopub.status.idle":"2025-12-08T21:25:18.121820Z","shell.execute_reply.started":"2025-12-08T21:25:17.675000Z","shell.execute_reply":"2025-12-08T21:25:18.120738Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Boxplot of target by season","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.boxplot(data=train, x=\"Season\", y=\"target\", hue=\"target_name\")\nplt.title(\"Biomass Distribution by Season and Target\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:40:32.708734Z","iopub.execute_input":"2025-12-08T21:40:32.709108Z","iopub.status.idle":"2025-12-08T21:40:33.413995Z","shell.execute_reply.started":"2025-12-08T21:40:32.709084Z","shell.execute_reply":"2025-12-08T21:40:33.412923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.boxplot(data=wide_train, x=\"Season\", y=\"Dry_Total_g\")\nplt.title(\"Biomass Distribution by Season\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:40:37.827262Z","iopub.execute_input":"2025-12-08T21:40:37.827565Z","iopub.status.idle":"2025-12-08T21:40:38.041961Z","shell.execute_reply.started":"2025-12-08T21:40:37.827545Z","shell.execute_reply":"2025-12-08T21:40:38.040548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"targets = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\nfig, axes = plt.subplots(2, 3, figsize=(18, 10))\naxes = axes.flatten()\n\nfor i, t in enumerate(targets):\n    sns.boxplot(data=wide_train, x=\"Season\", y=t, ax=axes[i])\n    axes[i].set_title(f\"{t} by Season\")\n    axes[i].set_xlabel(\"Season\")\n    axes[i].set_ylabel(t)\n\n# hide last empty subplot if 5 targets\naxes[-1].axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:40:44.047874Z","iopub.execute_input":"2025-12-08T21:40:44.048539Z","iopub.status.idle":"2025-12-08T21:40:45.094260Z","shell.execute_reply.started":"2025-12-08T21:40:44.048513Z","shell.execute_reply":"2025-12-08T21:40:45.093357Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ⭐ Key Insights for Modeling from Seasonal Analysis\n\nThe seasonal patterns observed across Dry_Green_g, Dry_Dead_g, Dry_Clover_g, GDM_g, and Dry_Total_g\nprovide several important signals that can significantly improve model performance.\n\n---\n**1. Season is a highly predictive feature**\nAll biomass components exhibit clear seasonal structure:\n\n- **Summer** → highest green and total biomass  \n- **Fall** → highest dead biomass  \n- **Winter** → lowest biomass overall  \n- **Spring** → active green regrowth  \n\nModels should incorporate:\n- `Season` (categorical)\n- Cyclical month encodings (`sin(month)`, `cos(month)`)\n\nThis will help the model capture plant phenology.\n\n---\n\n**2. Winter samples have distinct behavior**\nWinter shows:\n- Very low green biomass  \n- Lower total biomass  \n- Less variation overall  \n\nThis makes winter observations a smaller, more homogeneous cluster.\nModels must learn this pattern to avoid overestimating winter biomass.\n\n---\n\n**3. Summer displays high variability and many outliers**\nSummer biomass ranges from:\n- Very low (heat/drought-stressed plots)\n- To extremely high (peak growth)\n\nThis suggests the need for:\n- Robust loss functions (e.g., Huber, log-cosh)\n- Log-transforming the target to stabilize large values\n\n---\n\n**4. Green and dead biomass show opposite seasonal trends**\n- **Dry_Green_g & GDM_g** peak in Spring/Summer  \n- **Dry_Dead_g** peaks in Fall  \n\nThis complementary behavior is ideal for multitask learning models:\npredicting one target helps constrain the others.\n\n---\n\n **5. Total biomass reflects both seasonal growth and senescence**\nDry_Total_g combines green + dead + clover components, producing:\n- Highest values in Summer  \n- Moderate values in Spring/Fall  \n- Lowest values in Winter  \n\nBecause Dry_Total_g has the highest evaluation weight (0.5), capturing\nits seasonal pattern is critical for leaderboard performance.\n\n---\n\n**6. Seasonal trends confirm ecological validity of the dataset**\nThe observed patterns:\n- Match known Southern Hemisphere (Australia) vegetation cycles  \n- Validate that data quality is consistent and biologically plausible  \n- Suggest no major labeling errors or measurement inconsistencies  \n\nThis increases confidence in building season-aware models.\n\n---\n\n**Summary**\nSeasonal dynamics strongly influence all biomass targets. Incorporating seasonality—through categorical\nseasons, month embeddings, or time-aware modeling—will significantly improve predictive performance,\nespecially for Dry_Total_g and GDM_g.\n\n","metadata":{}},{"cell_type":"markdown","source":"## Pre_GSHH_NDVI","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\nsns.histplot(wide_train['Pre_GSHH_NDVI'], kde=True, bins=30)\nplt.title(\"Distribution of Pre_GSHH_NDVI\")\nplt.xlabel(\"NDVI\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:40:55.287066Z","iopub.execute_input":"2025-12-08T21:40:55.287395Z","iopub.status.idle":"2025-12-08T21:40:55.592892Z","shell.execute_reply.started":"2025-12-08T21:40:55.287372Z","shell.execute_reply":"2025-12-08T21:40:55.591668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8,4))\nsns.boxplot(x=wide_train['Pre_GSHH_NDVI'])\nplt.title(\"NDVI Outlier Check\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:40:56.674257Z","iopub.execute_input":"2025-12-08T21:40:56.674572Z","iopub.status.idle":"2025-12-08T21:40:56.817777Z","shell.execute_reply.started":"2025-12-08T21:40:56.674550Z","shell.execute_reply":"2025-12-08T21:40:56.816747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nsns.boxplot(data=wide_train, x=\"Season\", y=\"Pre_GSHH_NDVI\")\nsns.swarmplot(data=wide_train, x=\"Season\", y=\"Pre_GSHH_NDVI\", color=\"0.25\", size=3)\nplt.title(\"NDVI Across Seasons\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:40:59.901737Z","iopub.execute_input":"2025-12-08T21:40:59.902112Z","iopub.status.idle":"2025-12-08T21:41:00.383252Z","shell.execute_reply.started":"2025-12-08T21:40:59.902088Z","shell.execute_reply":"2025-12-08T21:41:00.382333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.boxplot(data=wide_train, x=\"State\", y=\"Pre_GSHH_NDVI\")\nsns.swarmplot(data=wide_train, x=\"State\", y=\"Pre_GSHH_NDVI\", color=\"0.25\", size=3)\nplt.title(\"NDVI Distribution by State\")\nplt.xlabel(\"State\")\nplt.ylabel(\"NDVI\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:03.048335Z","iopub.execute_input":"2025-12-08T21:41:03.048652Z","iopub.status.idle":"2025-12-08T21:41:03.552989Z","shell.execute_reply.started":"2025-12-08T21:41:03.048629Z","shell.execute_reply":"2025-12-08T21:41:03.551988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for t in targets:\n    plt.figure(figsize=(7,5))\n    sns.scatterplot(data=wide_train, x=\"Pre_GSHH_NDVI\", y=t, hue=\"Season\")\n    plt.title(f\"NDVI vs {t}\")\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:06.491525Z","iopub.execute_input":"2025-12-08T21:41:06.491860Z","iopub.status.idle":"2025-12-08T21:41:08.286525Z","shell.execute_reply.started":"2025-12-08T21:41:06.491837Z","shell.execute_reply":"2025-12-08T21:41:08.285462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(7,5))\nsns.scatterplot(data=wide_train, x=\"Pre_GSHH_NDVI\", y=\"Height_Ave_cm\", hue=\"Season\")\nplt.title(\"NDVI vs Height\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:08.287891Z","iopub.execute_input":"2025-12-08T21:41:08.288216Z","iopub.status.idle":"2025-12-08T21:41:08.615037Z","shell.execute_reply.started":"2025-12-08T21:41:08.288194Z","shell.execute_reply":"2025-12-08T21:41:08.614035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = wide_train[[\n    \"Pre_GSHH_NDVI\", \"Dry_Clover_g\", \"Dry_Dead_g\", \n    \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\", \"Height_Ave_cm\"\n]].corr()\n\nplt.figure(figsize=(10,7))\nsns.heatmap(corr, annot=True, cmap=\"YlGnBu\", fmt=\".2f\")\nplt.title(\"Correlation Heatmap with NDVI\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:12.188261Z","iopub.execute_input":"2025-12-08T21:41:12.188559Z","iopub.status.idle":"2025-12-08T21:41:12.584820Z","shell.execute_reply.started":"2025-12-08T21:41:12.188538Z","shell.execute_reply":"2025-12-08T21:41:12.583660Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Height","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\nsns.histplot(wide_train['Height_Ave_cm'], kde=True, bins=30)\nplt.title(\"Distribution of Height_Ave_cm\")\nplt.xlabel(\"Height (cm)\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:15.765672Z","iopub.execute_input":"2025-12-08T21:41:15.766045Z","iopub.status.idle":"2025-12-08T21:41:16.043211Z","shell.execute_reply.started":"2025-12-08T21:41:15.766020Z","shell.execute_reply":"2025-12-08T21:41:16.042169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(8,4))\nsns.boxplot(x=wide_train['Height_Ave_cm'])\nplt.title(\"Height Outlier Check\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:17.283689Z","iopub.execute_input":"2025-12-08T21:41:17.283993Z","iopub.status.idle":"2025-12-08T21:41:17.429079Z","shell.execute_reply.started":"2025-12-08T21:41:17.283973Z","shell.execute_reply":"2025-12-08T21:41:17.427898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nsns.boxplot(data=wide_train, x=\"Season\", y=\"Height_Ave_cm\")\nsns.swarmplot(data=wide_train, x=\"Season\", y=\"Height_Ave_cm\", color=\"0.25\", size=3)\nplt.title(\"Height Across Seasons\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:18.709905Z","iopub.execute_input":"2025-12-08T21:41:18.710241Z","iopub.status.idle":"2025-12-08T21:41:19.557597Z","shell.execute_reply.started":"2025-12-08T21:41:18.710218Z","shell.execute_reply":"2025-12-08T21:41:19.556334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10,6))\nsns.boxplot(data=wide_train, x=\"State\", y=\"Height_Ave_cm\")\nplt.title(\"Height Distribution by State\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:24.149397Z","iopub.execute_input":"2025-12-08T21:41:24.149696Z","iopub.status.idle":"2025-12-08T21:41:24.363808Z","shell.execute_reply.started":"2025-12-08T21:41:24.149677Z","shell.execute_reply":"2025-12-08T21:41:24.362948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(7,5))\nsns.scatterplot(data=wide_train, x=\"Pre_GSHH_NDVI\", y=\"Height_Ave_cm\", hue=\"Season\")\nplt.title(\"NDVI vs Height\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:28.954769Z","iopub.execute_input":"2025-12-08T21:41:28.955111Z","iopub.status.idle":"2025-12-08T21:41:29.270531Z","shell.execute_reply.started":"2025-12-08T21:41:28.955088Z","shell.execute_reply":"2025-12-08T21:41:29.269522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfor t in targets:\n    plt.figure(figsize=(7,5))\n    sns.scatterplot(data=wide_train, x=\"Height_Ave_cm\", y=t, hue=\"Season\")\n    plt.title(f\"Height vs {t}\")\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:31.705615Z","iopub.execute_input":"2025-12-08T21:41:31.705948Z","iopub.status.idle":"2025-12-08T21:41:33.240720Z","shell.execute_reply.started":"2025-12-08T21:41:31.705911Z","shell.execute_reply":"2025-12-08T21:41:33.239615Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## State","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.boxplot(data=wide_train, x=\"State\", y=\"Pre_GSHH_NDVI\")\nsns.swarmplot(data=wide_train, x=\"State\", y=\"Pre_GSHH_NDVI\", color=\"0.25\", size=3)\nplt.title(\"NDVI Distribution by State\")\nplt.xlabel(\"State\")\nplt.ylabel(\"NDVI\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:33.769576Z","iopub.execute_input":"2025-12-08T21:41:33.769881Z","iopub.status.idle":"2025-12-08T21:41:34.238219Z","shell.execute_reply.started":"2025-12-08T21:41:33.769861Z","shell.execute_reply":"2025-12-08T21:41:34.237376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.boxplot(data=wide_train, x=\"State\", y=\"Dry_Total_g\")\nsns.swarmplot(data=wide_train, x=\"State\", y=\"Dry_Total_g\", color=\"0.25\", size=3)\nplt.title(\"Dry_Total_g Distribution by State\")\nplt.xlabel(\"State\")\nplt.ylabel(\"Dry_Total_g (grams)\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:42.228811Z","iopub.execute_input":"2025-12-08T21:41:42.229199Z","iopub.status.idle":"2025-12-08T21:41:42.983283Z","shell.execute_reply.started":"2025-12-08T21:41:42.229174Z","shell.execute_reply":"2025-12-08T21:41:42.981906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"targets = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\nfig, axes = plt.subplots(2, 3, figsize=(18, 10))\naxes = axes.flatten()\n\nfor i, t in enumerate(targets):\n    ax = axes[i]\n    sns.boxplot(data=wide_train, x=\"State\", y=t, ax=ax)\n    sns.swarmplot(data=wide_train, x=\"State\", y=t, ax=ax, color=\"0.25\", size=3)\n    ax.set_title(f\"{t} by State\")\n    ax.set_xlabel(\"State\")\n    ax.set_ylabel(t)\n\n# Hide last empty plot\naxes[-1].axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:41:49.254187Z","iopub.execute_input":"2025-12-08T21:41:49.254546Z","iopub.status.idle":"2025-12-08T21:41:55.502108Z","shell.execute_reply.started":"2025-12-08T21:41:49.254518Z","shell.execute_reply":"2025-12-08T21:41:55.501286Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engeeniring ","metadata":{}},{"cell_type":"markdown","source":"## Temperatures","metadata":{}},{"cell_type":"code","source":"# === OFFICIAL BOM TEMPERATURES ===\n\nstate_season_temp = {\n    (\"NSW\", \"Winter\"): 12.3, (\"NSW\", \"Spring\"): 16.9, (\"NSW\", \"Summer\"): 22.1, (\"NSW\", \"Fall\"): 17.3,\n    (\"Vic\", \"Winter\"): 10.0, (\"Vic\", \"Spring\"): 14.3, (\"Vic\", \"Summer\"): 20.4, (\"Vic\", \"Fall\"): 15.6,\n    (\"QLD\", \"Winter\"): 15.7, (\"QLD\", \"Spring\"): 20.8, (\"QLD\", \"Summer\"): 25.6, (\"QLD\", \"Fall\"): 21.2,\n    (\"SA\",  \"Winter\"): 12.0, (\"SA\",  \"Spring\"): 16.3, (\"SA\",  \"Summer\"): 22.8, (\"SA\",  \"Fall\"): 17.4,\n    (\"WA\",  \"Winter\"): 13.1, (\"WA\",  \"Spring\"): 17.4, (\"WA\",  \"Summer\"): 23.7, (\"WA\",  \"Fall\"): 18.2,\n    (\"Tas\",\"Winter\"): 8.6,  (\"Tas\",\"Spring\"): 12.5, (\"Tas\",\"Summer\"): 17.5, (\"Tas\",\"Fall\"): 13.0,\n    (\"NT\", \"Winter\"): 25.4, (\"NT\", \"Spring\"): 28.5, (\"NT\", \"Summer\"): 29.3, (\"NT\", \"Fall\"): 27.7,\n    (\"ACT\",\"Winter\"): 6.5,  (\"ACT\",\"Spring\"): 12.1, (\"ACT\",\"Summer\"): 20.2, (\"ACT\",\"Fall\"): 14.7,\n}\n\nseason_mean_temp = {\n    \"Winter\": 12.0, \"Spring\": 17.0, \"Summer\": 23.0, \"Fall\": 16.0,\n}\n\ndef map_state_season_temp(row):\n    key = (row[\"State\"], row[\"Season\"])\n    if key in state_season_temp:\n        return state_season_temp[key]\n    else:\n        return season_mean_temp[row[\"Season\"]]\n\n# Main FE feature\nwide_train[\"State_Season_temp\"] = wide_train.apply(map_state_season_temp, axis=1)\n\n# Additional temperature feature\nwide_train[\"Season_temp_global\"] = wide_train[\"Season\"].map(season_mean_temp)\n\n# Interactions\nwide_train[\"Temp_NDVI\"]   = wide_train[\"State_Season_temp\"] * wide_train[\"Pre_GSHH_NDVI\"]\nwide_train[\"Temp_Height\"] = wide_train[\"State_Season_temp\"] * wide_train[\"Height_Ave_cm\"]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:48:57.483095Z","iopub.execute_input":"2025-12-08T21:48:57.483476Z","iopub.status.idle":"2025-12-08T21:48:57.505036Z","shell.execute_reply.started":"2025-12-08T21:48:57.483450Z","shell.execute_reply":"2025-12-08T21:48:57.503300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wide_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:49:13.399652Z","iopub.execute_input":"2025-12-08T21:49:13.400032Z","iopub.status.idle":"2025-12-08T21:49:13.422263Z","shell.execute_reply.started":"2025-12-08T21:49:13.400006Z","shell.execute_reply":"2025-12-08T21:49:13.421349Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Railfall","metadata":{}},{"cell_type":"code","source":"# === OFFICIAL BOM RAINFALL VALUES ===\n\nstate_season_rain = {\n    (\"NSW\", \"Winter\"): 342, (\"NSW\", \"Spring\"): 267, (\"NSW\", \"Summer\"): 275, (\"NSW\", \"Fall\"): 367,\n    (\"Vic\", \"Winter\"): 132, (\"Vic\", \"Spring\"): 167, (\"Vic\", \"Summer\"): 136, (\"Vic\", \"Fall\"): 151,\n    (\"QLD\",\"Winter\"): 56,  (\"QLD\",\"Spring\"): 171, (\"QLD\",\"Summer\"): 420, (\"QLD\",\"Fall\"): 246,\n    (\"SA\", \"Winter\"): 217, (\"SA\", \"Spring\"): 124, (\"SA\", \"Summer\"): 56,  (\"SA\", \"Fall\"): 115,\n    (\"WA\", \"Winter\"): 476, (\"WA\", \"Spring\"): 233, (\"WA\", \"Summer\"): 40,  (\"WA\", \"Fall\"): 113,\n    (\"Tas\",\"Winter\"): 158, (\"Tas\",\"Spring\"): 170, (\"Tas\",\"Summer\"): 150, (\"Tas\",\"Fall\"): 168,\n    (\"NT\", \"Winter\"): 2,   (\"NT\", \"Spring\"): 70,  (\"NT\", \"Summer\"): 1075,(\"NT\", \"Fall\"): 320,\n    (\"ACT\",\"Winter\"): 108, (\"ACT\",\"Spring\"): 174, (\"ACT\",\"Summer\"): 155, (\"ACT\",\"Fall\"): 146,\n}\n\n# Global fallback (Australia mean seasonal rainfall)\nseason_mean_rain = {\n    \"Winter\": 175,\n    \"Spring\": 180,\n    \"Summer\": 250,\n    \"Fall\":   185,\n}\n\ndef map_state_season_rain(row):\n    key = (row[\"State\"], row[\"Season\"])\n    if key in state_season_rain:\n        return state_season_rain[key]\n    else:\n        return season_mean_rain[row[\"Season\"]]\n\n# Main rainfall feature\nwide_train[\"State_Season_rain\"] = wide_train.apply(map_state_season_rain, axis=1)\n\n# Global seasonal rainfall\nwide_train[\"Season_rain_global\"] = wide_train[\"Season\"].map(season_mean_rain)\n\n# Rainfall interactions\nwide_train[\"Rain_NDVI\"]   = wide_train[\"State_Season_rain\"] * wide_train[\"Pre_GSHH_NDVI\"]\nwide_train[\"Rain_Height\"] = wide_train[\"State_Season_rain\"] * wide_train[\"Height_Ave_cm\"]\nwide_train[\"Rain_Temp\"]   = wide_train[\"State_Season_rain\"] * wide_train[\"State_Season_temp\"]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:53:50.862333Z","iopub.execute_input":"2025-12-08T21:53:50.862706Z","iopub.status.idle":"2025-12-08T21:53:50.887071Z","shell.execute_reply.started":"2025-12-08T21:53:50.862680Z","shell.execute_reply":"2025-12-08T21:53:50.885609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wide_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:54:02.611582Z","iopub.execute_input":"2025-12-08T21:54:02.612000Z","iopub.status.idle":"2025-12-08T21:54:02.661673Z","shell.execute_reply.started":"2025-12-08T21:54:02.611973Z","shell.execute_reply":"2025-12-08T21:54:02.660080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"climate_cols = [\n    \"State_Season_temp\", \"Season_temp_global\",\n    \"State_Season_rain\", \"Season_rain_global\",\n    \"Temp_NDVI\", \"Temp_Height\",\n    \"Rain_NDVI\", \"Rain_Height\",\n    \"Pre_GSHH_NDVI\", \"Height_Ave_cm\",\n    \"Dry_Total_g\", \"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\"\n]\n\ncorr = wide_train[climate_cols].corr()\n\nplt.figure(figsize=(14, 12))\nsns.heatmap(corr, annot=True, cmap=\"YlGnBu\", fmt=\".2f\", linewidths=.5)\nplt.title(\"Heatmap: Climate Features vs NDVI and Biomass Targets\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:58:24.745771Z","iopub.execute_input":"2025-12-08T21:58:24.746925Z","iopub.status.idle":"2025-12-08T21:58:25.652997Z","shell.execute_reply.started":"2025-12-08T21:58:24.746893Z","shell.execute_reply":"2025-12-08T21:58:25.651852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wide_train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-08T21:54:45.092981Z","iopub.execute_input":"2025-12-08T21:54:45.094330Z","iopub.status.idle":"2025-12-08T21:54:45.101124Z","shell.execute_reply.started":"2025-12-08T21:54:45.094296Z","shell.execute_reply":"2025-12-08T21:54:45.100161Z"}},"outputs":[],"execution_count":null}]}