{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **SUMMARIZED HERE**\n\n> The meta data columns have a major imapct on the biomass values. If we can able to use it in the model training it may give an edge.\n\n\n1. Dry_Clover_g:  Dry mass of clover-only portion of the sample.\n2. Dry_Dead_g: Dry mass of dead material in the sample.\n3. Dry_Green_g: Dry mass of green (live) vegetation.\n4. Dry_Total_g: Total dry biomass = Dry_Clover + Dry_Dead + Dry_Green\n5. GDM_g (Green Dry Matter): Dry mass of green dry matter only (subset of total biomass). Key indicator for grazing decisions","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib.lines import Line2D","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.617267Z","iopub.execute_input":"2025-11-21T06:06:52.617611Z","iopub.status.idle":"2025-11-21T06:06:52.622216Z","shell.execute_reply.started":"2025-11-21T06:06:52.617587Z","shell.execute_reply":"2025-11-21T06:06:52.621262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv_path = \"/kaggle/input/csiro-biomass/train.csv\"\ntrain_image_dir_path =\"/kaggle/input/csiro-biomass/train\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.623660Z","iopub.execute_input":"2025-11-21T06:06:52.623937Z","iopub.status.idle":"2025-11-21T06:06:52.645334Z","shell.execute_reply.started":"2025-11-21T06:06:52.623917Z","shell.execute_reply":"2025-11-21T06:06:52.644220Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Read Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(train_csv_path)\n\ntrain_df.head(6)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.646278Z","iopub.execute_input":"2025-11-21T06:06:52.646523Z","iopub.status.idle":"2025-11-21T06:06:52.682066Z","shell.execute_reply.started":"2025-11-21T06:06:52.646504Z","shell.execute_reply":"2025-11-21T06:06:52.681271Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Process & Strcture Data","metadata":{}},{"cell_type":"markdown","source":"## Validate lables for all images","metadata":{}},{"cell_type":"code","source":"group_sizes = train_df.groupby(\"image_path\").size()\nbad_size_groups = group_sizes[group_sizes != 5]\n\nprint(f\"Total Number of Images: {len(group_sizes)}\")\nprint(f\"Images With InComplete Lables: {bad_size_groups.shape[0]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.683959Z","iopub.execute_input":"2025-11-21T06:06:52.684787Z","iopub.status.idle":"2025-11-21T06:06:52.691784Z","shell.execute_reply.started":"2025-11-21T06:06:52.684763Z","shell.execute_reply":"2025-11-21T06:06:52.690862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"expected_types = {\n    \"Dry_Clover_g\",\n    \"Dry_Dead_g\",\n    \"Dry_Green_g\",\n    \"Dry_Total_g\",\n    \"GDM_g\",\n}\n\ndef has_all_types(x):\n    return set(x) == expected_types\n\ntype_check = train_df.groupby(\"image_path\")[\"target_name\"].apply(has_all_types)\n\n# Groups that are missing or have wrong types\nbad_type_groups = type_check[~type_check]\n\nprint(f\"Images with incorrect target_name types: {len(bad_type_groups)}\")\nif len(bad_type_groups) > 0:\n    print(bad_type_groups.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.692970Z","iopub.execute_input":"2025-11-21T06:06:52.693357Z","iopub.status.idle":"2025-11-21T06:06:52.718049Z","shell.execute_reply.started":"2025-11-21T06:06:52.693335Z","shell.execute_reply":"2025-11-21T06:06:52.716935Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":">No Issuses in the target_name field for all images","metadata":{}},{"cell_type":"markdown","source":"## Combine 5 rows into a single row per image","metadata":{}},{"cell_type":"code","source":"train_df.groupby(\"image_path\").get_group(\"train/ID1035947949.jpg\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.718966Z","iopub.execute_input":"2025-11-21T06:06:52.719358Z","iopub.status.idle":"2025-11-21T06:06:52.734144Z","shell.execute_reply.started":"2025-11-21T06:06:52.719337Z","shell.execute_reply":"2025-11-21T06:06:52.733399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.groupby(\"image_path\").get_group(\"train/ID1011485656.jpg\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.735289Z","iopub.execute_input":"2025-11-21T06:06:52.735631Z","iopub.status.idle":"2025-11-21T06:06:52.763232Z","shell.execute_reply.started":"2025-11-21T06:06:52.735596Z","shell.execute_reply":"2025-11-21T06:06:52.762358Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> Above we can see, for each image the sampling date, state, species, & other meta cols data are same. So we can combine into a single row.","metadata":{}},{"cell_type":"code","source":"meta_cols = [\n    \"image_path\",\n    \"Sampling_Date\",\n    \"State\",\n    \"Species\",\n    \"Pre_GSHH_NDVI\",\n    \"Height_Ave_cm\",\n]\n\nmeta_df = (\n    train_df[meta_cols]\n    .groupby(\"image_path\", as_index=False)\n    .agg(\"first\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.764324Z","iopub.execute_input":"2025-11-21T06:06:52.764744Z","iopub.status.idle":"2025-11-21T06:06:52.784941Z","shell.execute_reply.started":"2025-11-21T06:06:52.764716Z","shell.execute_reply":"2025-11-21T06:06:52.783825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pivot_df = (\n    train_df\n    .pivot_table(\n        index=\"image_path\",\n        columns=\"target_name\",\n        values=\"target\",\n        aggfunc=\"first\"  # safe since each type appears once per image_path\n    )\n    .reset_index()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.787113Z","iopub.execute_input":"2025-11-21T06:06:52.787403Z","iopub.status.idle":"2025-11-21T06:06:52.810590Z","shell.execute_reply.started":"2025-11-21T06:06:52.787375Z","shell.execute_reply":"2025-11-21T06:06:52.809595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = meta_df.merge(pivot_df, on=\"image_path\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.811651Z","iopub.execute_input":"2025-11-21T06:06:52.811967Z","iopub.status.idle":"2025-11-21T06:06:52.820087Z","shell.execute_reply.started":"2025-11-21T06:06:52.811941Z","shell.execute_reply":"2025-11-21T06:06:52.818755Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Splitting sampling date to 3 columns","metadata":{}},{"cell_type":"code","source":"# Convert to datetime (your format is YYYY/M/D)\ntrain_df[\"Sampling_Date\"] = pd.to_datetime(train_df[\"Sampling_Date\"], format=\"%Y/%m/%d\")\n\n# Extract year, month, day\ntrain_df[\"sampling_year\"]  = train_df[\"Sampling_Date\"].dt.year\ntrain_df[\"sampling_month\"] = train_df[\"Sampling_Date\"].dt.month\ntrain_df[\"sampling_day\"]   = train_df[\"Sampling_Date\"].dt.day\n\ntrain_df.drop(columns=[\"Sampling_Date\"], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.821087Z","iopub.execute_input":"2025-11-21T06:06:52.821383Z","iopub.status.idle":"2025-11-21T06:06:52.841817Z","shell.execute_reply.started":"2025-11-21T06:06:52.821356Z","shell.execute_reply":"2025-11-21T06:06:52.840719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.844288Z","iopub.execute_input":"2025-11-21T06:06:52.844938Z","iopub.status.idle":"2025-11-21T06:06:52.869876Z","shell.execute_reply.started":"2025-11-21T06:06:52.844913Z","shell.execute_reply":"2025-11-21T06:06:52.869097Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Analysis","metadata":{}},{"cell_type":"markdown","source":"## Data Types","metadata":{}},{"cell_type":"code","source":"train_df.dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.870936Z","iopub.execute_input":"2025-11-21T06:06:52.871232Z","iopub.status.idle":"2025-11-21T06:06:52.889461Z","shell.execute_reply.started":"2025-11-21T06:06:52.871204Z","shell.execute_reply":"2025-11-21T06:06:52.888356Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> - Image Path has an image\n> - State & Species are categorical data\n> - All other data are numerical data","metadata":{}},{"cell_type":"markdown","source":"## Check for missing values","metadata":{}},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.890610Z","iopub.execute_input":"2025-11-21T06:06:52.891177Z","iopub.status.idle":"2025-11-21T06:06:52.909436Z","shell.execute_reply.started":"2025-11-21T06:06:52.891151Z","shell.execute_reply":"2025-11-21T06:06:52.908489Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> There is no missing value in any rows & columns","metadata":{}},{"cell_type":"markdown","source":"# Data Distibutions","metadata":{}},{"cell_type":"markdown","source":"## Numerical Columns","metadata":{}},{"cell_type":"markdown","source":"GDM_g is a broader measurement of green dry matter than Dry_Green_g.\nFor low biomass plots → GDM ≈ Dry_Green\nFor higher biomass plots → additional green material appears, increasing GDM\nOutliers represent lush green clover-heavy plots\nThis difference explains why GDM is often used as a pasture quality indicator, not just quantity\n\nIn modeling:\nBoth features should be treated separately.\nGDM contains extra green information beyond Dry_Green that helps predict total biomass and forage quality.","metadata":{}},{"cell_type":"code","source":"diff = train_df[\"GDM_g\"] - train_df[\"Dry_Green_g\"]\n\nplt.figure(figsize=(8,5))\nsns.histplot(diff, kde=True, bins=30)\nplt.title(\"Distribution of (GDM_g - Dry_Green_g)\")\nplt.xlabel(\"Difference (g)\")\nplt.ylabel(\"Count\")\nplt.tight_layout()\nplt.show()\n\n\nplt.figure(figsize=(6,4))\nsns.boxplot(x=diff)\nplt.title(\"Boxplot: GDM_g - Dry_Green_g\")\nplt.xlabel(\"Difference (g)\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:52.910797Z","iopub.execute_input":"2025-11-21T06:06:52.911095Z","iopub.status.idle":"2025-11-21T06:06:53.744278Z","shell.execute_reply.started":"2025-11-21T06:06:52.911071Z","shell.execute_reply":"2025-11-21T06:06:53.743400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# select only numeric columns\nnumeric_cols = train_df.select_dtypes(include=[\"number\"]).columns\nfor col in numeric_cols:\n    data = train_df[col].dropna()  # drop NaNs so stats are clean\n    \n    # Basic stats\n    desc = data.describe()  # count, mean, std, min, 25%, 50%, 75%, max\n    \n    # Extra percentiles\n    p1, p5, p95, p99 = np.percentile(data, [1, 5, 95, 99])\n    \n    # --- Outlier detection (IQR) ---\n    q1 = desc[\"25%\"]\n    q3 = desc[\"75%\"]\n    iqr = q3 - q1\n    lower = q1 - 1.5 * iqr\n    upper = q3 + 1.5 * iqr\n    outlier_mask = (data < lower) | (data > upper)\n    n_outliers = outlier_mask.sum()\n    total = data.shape[0]\n    outlier_pct = 100 * n_outliers / total if total > 0 else 0.0\n    \n    # Build a multi-line label text for the legend\n    stats_text = (\n        f\"count    = {desc['count']:.0f}\\n\"\n        f\"min      = {desc['min']:.3f}\\n\"\n        f\"1%       = {p1:.3f}\\n\"\n        f\"5%       = {p5:.3f}\\n\"\n        f\"25%      = {desc['25%']:.3f}\\n\"\n        f\"median   = {desc['50%']:.3f}\\n\"\n        f\"75%      = {desc['75%']:.3f}\\n\"\n        f\"95%      = {p95:.3f}\\n\"\n        f\"99%      = {p99:.3f}\\n\"\n        f\"max      = {desc['max']:.3f}\\n\"\n        f\"outliers = {n_outliers} ({outlier_pct:.2f}%)\\n\"\n        f\"lower_IQR= {lower:.3f}\\n\"\n        f\"upper_IQR= {upper:.3f}\"\n    )\n    \n    # Create figure\n    plt.figure(figsize=(10, 4))\n    ax = sns.boxplot(x=data)\n    \n    ax.set_title(f\"Boxplot of {col}\")\n    ax.set_xlabel(col)\n    \n    # Dummy handle just to host the text in legend\n    dummy_handle = Line2D([], [], color=\"none\", label=stats_text)\n    ax.legend(handles=[dummy_handle], loc=\"upper right\", frameon=True, fontsize=8)\n    \n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:53.745156Z","iopub.execute_input":"2025-11-21T06:06:53.745449Z","iopub.status.idle":"2025-11-21T06:06:56.193724Z","shell.execute_reply.started":"2025-11-21T06:06:53.745422Z","shell.execute_reply":"2025-11-21T06:06:56.192788Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Categorical Data","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15, 4))\nsns.countplot(data=train_df, x=\"Species\")\nplt.xticks(rotation='vertical')\nplt.title(\"Category counts for Species\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:56.194650Z","iopub.execute_input":"2025-11-21T06:06:56.194941Z","iopub.status.idle":"2025-11-21T06:06:56.595868Z","shell.execute_reply.started":"2025-11-21T06:06:56.194920Z","shell.execute_reply":"2025-11-21T06:06:56.594991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 4))\nsns.countplot(data=train_df, x=\"State\")\nplt.xticks(rotation=45)\nplt.title(\"Category counts for State\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:56.596865Z","iopub.execute_input":"2025-11-21T06:06:56.597557Z","iopub.status.idle":"2025-11-21T06:06:56.825963Z","shell.execute_reply.started":"2025-11-21T06:06:56.597525Z","shell.execute_reply":"2025-11-21T06:06:56.825090Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Distribution Grouped By Categorical Col","metadata":{}},{"cell_type":"markdown","source":"## Total g Box Plot for Species","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15, 8))\nplt.xticks(rotation='vertical') \nsns.boxplot(data=train_df, x=\"Species\", y=\"Dry_Total_g\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:56.826891Z","iopub.execute_input":"2025-11-21T06:06:56.827127Z","iopub.status.idle":"2025-11-21T06:06:57.338168Z","shell.execute_reply.started":"2025-11-21T06:06:56.827110Z","shell.execute_reply":"2025-11-21T06:06:57.336936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Total g Box Plot for State","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.xticks(rotation='vertical') \nsns.boxplot(data=train_df, x=\"State\", y=\"Dry_Total_g\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:57.341229Z","iopub.execute_input":"2025-11-21T06:06:57.341476Z","iopub.status.idle":"2025-11-21T06:06:57.604349Z","shell.execute_reply.started":"2025-11-21T06:06:57.341458Z","shell.execute_reply":"2025-11-21T06:06:57.603215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(15, 10))\nplt.xticks(rotation='vertical') \nsns.barplot(\n    data=train_df,\n    x=\"Species\",\n    y=\"Dry_Total_g\",   # or other numeric target/feature\n    estimator=\"mean\",\n    ci=\"sd\"\n)\nplt.title(\"Mean Dry_Total_g per Species\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:57.605665Z","iopub.execute_input":"2025-11-21T06:06:57.606017Z","iopub.status.idle":"2025-11-21T06:06:58.053617Z","shell.execute_reply.started":"2025-11-21T06:06:57.605982Z","shell.execute_reply":"2025-11-21T06:06:58.052810Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Relation of total_g with respect to state & species.\n\n> Some species gave better yield in some states","metadata":{}},{"cell_type":"code","source":"ct = pd.crosstab(train_df[\"State\"], train_df[\"Species\"])\nplt.figure(figsize=(10, 10))\nplt.xticks(rotation='vertical') \nsns.heatmap(ct, annot=True, fmt=\"d\")\nplt.title(\"State × Species frequency\")\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:58.054589Z","iopub.execute_input":"2025-11-21T06:06:58.054905Z","iopub.status.idle":"2025-11-21T06:06:58.625733Z","shell.execute_reply.started":"2025-11-21T06:06:58.054877Z","shell.execute_reply":"2025-11-21T06:06:58.624861Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"> Both the State & Species has the biggest factor in deciding the biomass","metadata":{}},{"cell_type":"markdown","source":"# Drift of total_g between classess\n\n> Hellinger distance measures distribution drift between two probability distributions.\n> - 0.0 → identical distributions\n> - 1.0 → completely different distributions\n\n\nIf higher the drift, then the class has bigger imact on total_g","metadata":{}},{"cell_type":"markdown","source":"## Distribution Drift b/w Species & total_g ","metadata":{}},{"cell_type":"code","source":"def hellinger(p, q):\n    \"\"\"\n    Computes the Hellinger distance between two probability distributions.\n    p, q: 1D numpy arrays that sum to 1.\n    \"\"\"\n    p = np.asarray(p)\n    q = np.asarray(q)\n    return np.sqrt(np.sum((np.sqrt(p) - np.sqrt(q))**2)) / np.sqrt(2)\n\n\ndef hellinger_distance_matrix(df, cat_col, value_col, bins=30):\n    \"\"\"\n    Compute pairwise Hellinger distance matrix for distributions of `value_col`\n    across each category in `cat_col`.\n\n    Returns:\n      - dist_df: square DataFrame [categories x categories] of Hellinger distances\n    \"\"\"\n    categories = df[cat_col].dropna().unique()\n    categories = np.sort(categories)  # just for consistent ordering\n\n    # common bin edges for all histograms\n    vals = df[value_col].dropna()\n    bin_edges = np.linspace(vals.min(), vals.max(), bins)\n\n    # compute histograms per category (as probability distributions)\n    histograms = {}\n    for cat in categories:\n        data = df.loc[df[cat_col] == cat, value_col].dropna()\n        hist, _ = np.histogram(data, bins=bin_edges, density=False)\n        # convert to probability distribution\n        total = hist.sum()\n        if total == 0:\n            # avoid division by zero; keep zeros\n            prob = np.zeros_like(hist, dtype=float)\n        else:\n            prob = hist / total\n        histograms[cat] = prob\n\n    # build pairwise distance matrix\n    dist_mat = np.zeros((len(categories), len(categories)))\n    for i, c1 in enumerate(categories):\n        for j, c2 in enumerate(categories):\n            if i == j:\n                dist_mat[i, j] = 0.0\n            elif i < j:\n                h = hellinger(histograms[c1], histograms[c2])\n                dist_mat[i, j] = h\n                dist_mat[j, i] = h\n\n    dist_df = pd.DataFrame(dist_mat, index=categories, columns=categories)\n    return dist_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:58.626670Z","iopub.execute_input":"2025-11-21T06:06:58.627012Z","iopub.status.idle":"2025-11-21T06:06:58.637282Z","shell.execute_reply.started":"2025-11-21T06:06:58.626983Z","shell.execute_reply":"2025-11-21T06:06:58.636246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"hellinger_species = hellinger_distance_matrix(\n    train_df,\n    cat_col=\"Species\",\n    value_col=\"Dry_Total_g\",\n    bins=30\n)\n\nplt.figure(figsize=(15, 15))\nsns.heatmap(\n    hellinger_species,\n    annot=True,\n    fmt=\".3f\",\n    cmap=\"viridis\"\n)\nplt.title(\"Hellinger Distance - Dry_Total_g Distribution Drift by Species\")\nplt.ylabel(\"Species\")\nplt.xlabel(\"Species\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:58.638660Z","iopub.execute_input":"2025-11-21T06:06:58.638979Z","iopub.status.idle":"2025-11-21T06:06:59.876347Z","shell.execute_reply.started":"2025-11-21T06:06:58.638956Z","shell.execute_reply":"2025-11-21T06:06:59.875211Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Distribution Drift b/w State & total_g ","metadata":{}},{"cell_type":"code","source":"hellinger_state = hellinger_distance_matrix(\n    train_df,\n    cat_col=\"State\",\n    value_col=\"Dry_Total_g\",\n    bins=30\n)\n\nplt.figure(figsize=(5, 5))\nsns.heatmap(\n    hellinger_state,\n    annot=True,\n    fmt=\".3f\",\n    cmap=\"viridis\"\n)\nplt.title(\"Hellinger Distance - Dry_Total_g Distribution Drift by State\")\nplt.ylabel(\"State\")\nplt.xlabel(\"State\")\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:06:59.876987Z","iopub.execute_input":"2025-11-21T06:06:59.877272Z","iopub.status.idle":"2025-11-21T06:07:00.195093Z","shell.execute_reply.started":"2025-11-21T06:06:59.877252Z","shell.execute_reply":"2025-11-21T06:07:00.194186Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Columns Correlation Between them","metadata":{}},{"cell_type":"markdown","source":"## Pearson Correlation","metadata":{}},{"cell_type":"code","source":"def plot_pearson_corr(df, figsize=(10,8)):\n    corr = df.corr(numeric_only=True)\n\n    plt.figure(figsize=figsize)\n    sns.heatmap(corr, annot=True, cmap=\"coolwarm\", fmt=\".2f\")\n    plt.title(\"Pearson Correlation (Linear)\")\n    plt.tight_layout()\n    plt.show()\n    \n    return corr\n\n\npearson_corr = plot_pearson_corr(train_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:00.196302Z","iopub.execute_input":"2025-11-21T06:07:00.196617Z","iopub.status.idle":"2025-11-21T06:07:00.782865Z","shell.execute_reply.started":"2025-11-21T06:07:00.196588Z","shell.execute_reply":"2025-11-21T06:07:00.782018Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Mutual Information (numeric + categorical vs numeric)","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_regression, mutual_info_classif\nimport pandas as pd\n\ndef compute_mutual_information(df, target):\n    X = df.drop(columns=[target])\n    y = df[target]\n\n    # numeric vs numeric/categorical → MI Regression\n    mi = mutual_info_regression(\n        X.select_dtypes(include=['number']),\n        y,\n        random_state=0\n    )\n    \n    mi_df = pd.DataFrame({\n        \"feature\": X.select_dtypes(include=['number']).columns,\n        \"MI\": mi\n    }).sort_values(\"MI\", ascending=False)\n\n    return mi_df\n\n\n\ndef plot_mutual_information(mi_df, target_name):\n    plt.figure(figsize=(8,5))\n    sns.barplot(data=mi_df, x=\"MI\", y=\"feature\")\n    plt.title(f\"Mutual Information (Feature → {target_name})\")\n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:00.783815Z","iopub.execute_input":"2025-11-21T06:07:00.784103Z","iopub.status.idle":"2025-11-21T06:07:00.791588Z","shell.execute_reply.started":"2025-11-21T06:07:00.784082Z","shell.execute_reply":"2025-11-21T06:07:00.790631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:00.793221Z","iopub.execute_input":"2025-11-21T06:07:00.794054Z","iopub.status.idle":"2025-11-21T06:07:00.810931Z","shell.execute_reply.started":"2025-11-21T06:07:00.794005Z","shell.execute_reply":"2025-11-21T06:07:00.810086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mi_df = compute_mutual_information(train_df, \"Dry_Total_g\")\nplot_mutual_information(mi_df, \"Dry_Total_g\")\n\nmi_df = compute_mutual_information(train_df, \"GDM_g\")\nplot_mutual_information(mi_df, \"GDM_g\")\n\nmi_df = compute_mutual_information(train_df, \"Dry_Green_g\")\nplot_mutual_information(mi_df, \"Dry_Green_g\")\n\nmi_df = compute_mutual_information(train_df, \"Dry_Dead_g\")\nplot_mutual_information(mi_df, \"Dry_Dead_g\")\n\nmi_df = compute_mutual_information(train_df, \"Dry_Clover_g\")\nplot_mutual_information(mi_df, \"Dry_Clover_g\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:00.811841Z","iopub.execute_input":"2025-11-21T06:07:00.812121Z","iopub.status.idle":"2025-11-21T06:07:02.468084Z","shell.execute_reply.started":"2025-11-21T06:07:00.812101Z","shell.execute_reply":"2025-11-21T06:07:02.466990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Image Data Analysis","metadata":{}},{"cell_type":"markdown","source":"## Prepare the image data","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.feature_selection import mutual_info_regression\n\n# So plots look nice\nsns.set(style=\"whitegrid\")\n\n# Adjust this if needed (root folder of images)\nIMAGE_ROOT = \".\"   # e.g. \"/kaggle/input/pasture-biomass\"\n\ndef full_path(rel_path):\n    return \"/kaggle/input/csiro-biomass/\"+rel_path\n\n# Add full path column\ntrain_df[\"image_full_path\"] = train_df[\"image_path\"].apply(full_path)\n\n# Keep only rows where the image file actually exists\nexisting_df = train_df[train_df[\"image_full_path\"].apply(os.path.exists)]\n\nN_SAMPLES = min(100, len(existing_df))\nprint(f\"Sampling {N_SAMPLES} images from {len(existing_df)} available files\")\n\ndf_sample = existing_df.sample(N_SAMPLES, random_state=42).reset_index(drop=True)\ndf_sample.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:02.468932Z","iopub.execute_input":"2025-11-21T06:07:02.469224Z","iopub.status.idle":"2025-11-21T06:07:02.787139Z","shell.execute_reply.started":"2025-11-21T06:07:02.469204Z","shell.execute_reply":"2025-11-21T06:07:02.786275Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load & Extract Color features","metadata":{}},{"cell_type":"code","source":"def load_image(path):\n    \"\"\"Load image as RGB using OpenCV.\"\"\"\n    img_bgr = cv2.imread(path)\n    if img_bgr is None:\n        raise ValueError(f\"Could not load image at {path}\")\n    img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n    return img_rgb\n\ndef extract_color_features(img, hist_bins=32):\n    \"\"\"\n    Extract basic color statistics and normalized histograms\n    from RGB and HSV color spaces.\n    Returns a flat dict of features.\n    \"\"\"\n    features = {}\n    \n    img = img.astype(np.float32)\n    \n    # --- RGB channels ---\n    R = img[:, :, 0].flatten()\n    G = img[:, :, 1].flatten()\n    B = img[:, :, 2].flatten()\n    \n    for name, arr in zip([\"R\", \"G\", \"B\"], [R, G, B]):\n        features[f\"mean_{name}\"] = float(np.mean(arr))\n        features[f\"std_{name}\"]  = float(np.std(arr))\n        features[f\"min_{name}\"]  = float(np.min(arr))\n        features[f\"max_{name}\"]  = float(np.max(arr))\n    \n    # --- HSV ---\n    img_hsv = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_RGB2HSV)\n    H = img_hsv[:, :, 0].flatten()\n    S = img_hsv[:, :, 1].flatten()\n    V = img_hsv[:, :, 2].flatten()\n    \n    for name, arr in zip([\"H\", \"S\", \"V\"], [H, S, V]):\n        features[f\"mean_{name}\"] = float(np.mean(arr))\n        features[f\"std_{name}\"]  = float(np.std(arr))\n        features[f\"min_{name}\"]  = float(np.min(arr))\n        features[f\"max_{name}\"]  = float(np.max(arr))\n    \n    # --- RGB histograms (normalized) ---\n    for name, arr in zip([\"R\", \"G\", \"B\"], [R, G, B]):\n        hist, _ = np.histogram(arr, bins=hist_bins, range=(0, 255), density=False)\n        hist = hist.astype(np.float32)\n        if hist.sum() > 0:\n            hist /= hist.sum()\n        for i, h_val in enumerate(hist):\n            features[f\"hist_{name}_{i}\"] = float(h_val)\n\n    # --- Hue histogram (normalized) ---\n    hist_H, _ = np.histogram(H, bins=hist_bins, range=(0, 180), density=False)\n    hist_H = hist_H.astype(np.float32)\n    if hist_H.sum() > 0:\n        hist_H /= hist_H.sum()\n    for i, h_val in enumerate(hist_H):\n        features[f\"hist_H_{i}\"] = float(h_val)\n\n    return features\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:02.788098Z","iopub.execute_input":"2025-11-21T06:07:02.788421Z","iopub.status.idle":"2025-11-21T06:07:02.800467Z","shell.execute_reply.started":"2025-11-21T06:07:02.788394Z","shell.execute_reply":"2025-11-21T06:07:02.799332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"color_features_list = []\n\nfor idx, row in df_sample.iterrows():\n    path = row[\"image_full_path\"]\n    img = load_image(path)\n    feats = extract_color_features(img, hist_bins=32)\n    feats[\"image_path\"] = row[\"image_path\"]\n    feats[\"Dry_Total_g\"] = float(row[\"Dry_Total_g\"])\n    color_features_list.append(feats)\n\ncolor_df = pd.DataFrame(color_features_list)\n\nprint(\"Color feature dataframe shape:\", color_df.shape)\ncolor_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:02.801339Z","iopub.execute_input":"2025-11-21T06:07:02.801638Z","iopub.status.idle":"2025-11-21T06:07:28.571136Z","shell.execute_reply.started":"2025-11-21T06:07:02.801611Z","shell.execute_reply":"2025-11-21T06:07:28.570105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = [c for c in color_df.columns if c not in [\"image_path\", \"Dry_Total_g\"]]\nX = color_df[feature_cols]\ny = color_df[\"Dry_Total_g\"]\n\nprint(\"Number of features:\", len(feature_cols))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:28.572295Z","iopub.execute_input":"2025-11-21T06:07:28.572939Z","iopub.status.idle":"2025-11-21T06:07:28.579607Z","shell.execute_reply.started":"2025-11-21T06:07:28.572917Z","shell.execute_reply":"2025-11-21T06:07:28.578658Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Correlation of Image Color Feature & Dry_Total_g","metadata":{}},{"cell_type":"markdown","source":"## Pearson","metadata":{}},{"cell_type":"code","source":"def plot_pearson_corr_with_target(X, y, top_k=30):\n    corr_vals = []\n    for col in X.columns:\n        # guard against constant columns\n        if X[col].nunique() <= 1:\n            corr = 0.0\n        else:\n            corr = np.corrcoef(X[col], y)[0, 1]\n        corr_vals.append((col, corr))\n    \n    corr_df = pd.DataFrame(corr_vals, columns=[\"feature\", \"pearson_corr\"])\n    corr_df[\"abs_corr\"] = corr_df[\"pearson_corr\"].abs()\n    corr_df = corr_df.sort_values(\"abs_corr\", ascending=False).head(top_k)\n    \n    plt.figure(figsize=(8, max(6, 0.3 * len(corr_df))))\n    sns.barplot(data=corr_df, x=\"pearson_corr\", y=\"feature\")\n    plt.title(\"Top Pearson Correlations with Dry_Total_g\")\n    plt.axvline(0, color=\"black\", linewidth=0.8)\n    plt.tight_layout()\n    plt.show()\n    \n    return corr_df\n\npearson_top = plot_pearson_corr_with_target(X, y, top_k=30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:28.580774Z","iopub.execute_input":"2025-11-21T06:07:28.581233Z","iopub.status.idle":"2025-11-21T06:07:29.116740Z","shell.execute_reply.started":"2025-11-21T06:07:28.581204Z","shell.execute_reply":"2025-11-21T06:07:29.115893Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Mutual Information (MI) with Dry_Total_g","metadata":{}},{"cell_type":"code","source":"def compute_and_plot_mi(X, y, top_k=30):\n    mi = mutual_info_regression(X, y, random_state=0)\n    mi_df = pd.DataFrame({\"feature\": X.columns, \"MI\": mi})\n    mi_df = mi_df.sort_values(\"MI\", ascending=False).head(top_k)\n    \n    plt.figure(figsize=(8, max(6, 0.3 * len(mi_df))))\n    sns.barplot(data=mi_df, x=\"MI\", y=\"feature\")\n    plt.title(\"Top Mutual Information Scores with Dry_Total_g\")\n    plt.tight_layout()\n    plt.show()\n    \n    return mi_df\n\nmi_top = compute_and_plot_mi(X, y, top_k=30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:29.117653Z","iopub.execute_input":"2025-11-21T06:07:29.117967Z","iopub.status.idle":"2025-11-21T06:07:29.821270Z","shell.execute_reply.started":"2025-11-21T06:07:29.117940Z","shell.execute_reply":"2025-11-21T06:07:29.820327Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## HSIC (Hilbert–Schmidt Independence Criterion)","metadata":{}},{"cell_type":"code","source":"def rbf_kernel_1d(x, sigma=None):\n    x = x.reshape(-1, 1).astype(np.float64)\n    dists = (x - x.T) ** 2\n    vals = dists[dists > 0]\n    if sigma is None:\n        if len(vals) == 0:\n            sigma = 1.0\n        else:\n            sigma = np.median(np.sqrt(vals))\n        if sigma == 0:\n            sigma = 1.0\n    K = np.exp(-dists / (2 * sigma**2))\n    return K\n\ndef compute_and_plot_hsic(X, y, top_k=30):\n    yK = rbf_kernel_1d(y.values)\n    n = len(y)\n    H = np.eye(n) - np.ones((n, n)) / n\n    \n    hsic_scores = []\n    for col in X.columns:\n        x_arr = X[col].values\n        xK = rbf_kernel_1d(x_arr)\n        hsic = np.trace(xK @ H @ yK @ H) / ((n - 1) ** 2)\n        hsic_scores.append((col, hsic))\n    \n    hsic_df = pd.DataFrame(hsic_scores, columns=[\"feature\", \"HSIC\"])\n    hsic_df = hsic_df.sort_values(\"HSIC\", ascending=False).head(top_k)\n    \n    plt.figure(figsize=(8, max(6, 0.3 * len(hsic_df))))\n    sns.barplot(data=hsic_df, x=\"HSIC\", y=\"feature\")\n    plt.title(\"Top HSIC Scores with Dry_Total_g\")\n    plt.tight_layout()\n    plt.show()\n    \n    return hsic_df\n\nhsic_top = compute_and_plot_hsic(X, y, top_k=30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:29.822004Z","iopub.execute_input":"2025-11-21T06:07:29.822284Z","iopub.status.idle":"2025-11-21T06:07:30.596259Z","shell.execute_reply.started":"2025-11-21T06:07:29.822266Z","shell.execute_reply":"2025-11-21T06:07:30.595355Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Image Color Analysis \n\n\n- **RGB image**: The original image with 3 channels.\n- **ExG — Excess Green**: High when G is much stronger than R and B → very green. Low or negative when area is more reddish, bluish, or grayish\n- **ExR — Excess Red**: High when red is strong relative to green. Captures reddish / brown / dry tones better.\n- **ExGR** : High when green is strong AND red is weak. Low when red dominates or green is weak\n- **VARI** : Visible Atmospherically Resistant Index. Compares Green vs Red. Designed to be more robust to lighting and atmospheric changes.\n- **NDI_GR** : If G ≫ R → value near +1 → very green. If R ≫ G → value near −1 → more red/dry/soil\n- G**reen cover mask** : Binary mask, True where Green is the dominant channel\n\n\n> All these information can be added as a extra channels to model as input.","metadata":{}},{"cell_type":"code","source":"# If you don't have this yet, re-define:\ndef load_image_rgb(path):\n    \"\"\"Load image as RGB (float32, 0–1).\"\"\"\n    img_bgr = cv2.imread(path)\n    if img_bgr is None:\n        raise ValueError(f\"Could not load image at {path}\")\n    img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)\n    img_rgb = img_rgb.astype(np.float32) / 255.0\n    return img_rgb\n\ndef plot_image_color_features(img, title_prefix=\"\", eps=1e-6):\n    \"\"\"\n    img: RGB float32 [0,1], shape (H, W, 3)\n    Plots RGB + index maps in a single row.\n    \"\"\"\n    R = img[:, :, 0]\n    G = img[:, :, 1]\n    B = img[:, :, 2]\n\n    # Color indices\n    ExG    = 2*G - R - B\n    ExR    = 1.4*R - G\n    ExGR   = ExG - ExR\n    VARI   = (G - R) / (G + R - B + eps)\n    NDI_GR = (G - R) / (G + R + eps)\n    green_mask = (G > R) & (G > B)\n\n    # print(img.shape, ExG.shape, ExR.shape, ExGR.shape, VARI.shape, NDI_GR.shape, green_mask.shape)\n\n    # Helper for plotting with percentile clipping\n    def show_index(ax, arr, title, cmap=\"viridis\"):\n        vmin, vmax = np.percentile(arr, [2, 98])\n        ax.imshow(arr, cmap=cmap, vmin=vmin, vmax=vmax)\n        ax.set_title(title, fontsize=9)\n        ax.axis(\"off\")\n\n    fig, axes = plt.subplots(1, 7, figsize=(18,9))\n    \n    # 1) RGB\n    axes[0].imshow(img)\n    axes[0].set_title(f\"{title_prefix} RGB\", fontsize=9)\n    axes[0].axis(\"off\")\n\n    # 2) ExG\n    show_index(axes[1], ExG, \"ExG (Excess Green)\", cmap=\"Greens\")\n\n    # 3) ExR\n    show_index(axes[2], ExR, \"ExR (Excess Red)\", cmap=\"Reds\")\n\n    # 4) ExGR\n    show_index(axes[3], ExGR, \"ExGR (ExG - ExR)\")\n\n    # 5) VARI\n    show_index(axes[4], VARI, \"VARI\")\n\n    # 6) NDI_GR\n    show_index(axes[5], NDI_GR, \"NDI_GR\")\n\n    # 7) Green cover mask\n    axes[6].imshow(green_mask, cmap=\"gray\")\n    axes[6].set_title(\"Green mask\\n(G>R & G>B)\", fontsize=9)\n    axes[6].axis(\"off\")\n\n    plt.tight_layout()\n    plt.show()\n\n\nfor i, row in df_sample.sample(30).iterrows():\n    img = load_image_rgb(row[\"image_full_path\"])\n    biomass = row[\"Dry_Total_g\"]\n    title_prefix = f\"{row['image_path']} | Dry_Total_g={biomass:.1f}\"\n    plot_image_color_features(img, title_prefix=title_prefix)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T06:07:30.597131Z","iopub.execute_input":"2025-11-21T06:07:30.597355Z","iopub.status.idle":"2025-11-21T06:08:37.727187Z","shell.execute_reply.started":"2025-11-21T06:07:30.597338Z","shell.execute_reply":"2025-11-21T06:08:37.725953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}