{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# *CSIRO - Image2Biomass Prediction*","metadata":{}},{"cell_type":"code","source":"\nDATA_PATH = \"/kaggle/input/csiro-biomass/train.csv\"\nSAVE_DIR = \"/kaggle/working/eda_results\"  \nimport os\nfrom pathlib import Path\nPath(SAVE_DIR).mkdir(parents=True, exist_ok=True)\n\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom math import isfinite\n\n%matplotlib inline\npd.set_option(\"display.max_columns\", 120)\npd.set_option(\"display.width\", 180)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.631427Z","iopub.execute_input":"2025-10-31T16:32:07.631695Z","iopub.status.idle":"2025-10-31T16:32:07.637828Z","shell.execute_reply.started":"2025-10-31T16:32:07.631678Z","shell.execute_reply":"2025-10-31T16:32:07.637188Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load Data & Basic Info","metadata":{}},{"cell_type":"code","source":"# Load CSV\ndata = pd.read_csv(DATA_PATH)\nprint(f\"Data shape: {data.shape}\")\ndisplay(data.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.644077Z","iopub.execute_input":"2025-10-31T16:32:07.644317Z","iopub.status.idle":"2025-10-31T16:32:07.667888Z","shell.execute_reply.started":"2025-10-31T16:32:07.644301Z","shell.execute_reply":"2025-10-31T16:32:07.667261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Column types & memory usage\nmem_usage_mb = data.memory_usage(deep=True).sum() / (1024**2)\nprint(f\"Memory usage: {mem_usage_mb:.2f} MB\")\ndisplay(pd.DataFrame({\"dtype\": data.dtypes}))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.668876Z","iopub.execute_input":"2025-10-31T16:32:07.669072Z","iopub.status.idle":"2025-10-31T16:32:07.679066Z","shell.execute_reply.started":"2025-10-31T16:32:07.669056Z","shell.execute_reply":"2025-10-31T16:32:07.678293Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Missing Values Check","metadata":{}},{"cell_type":"code","source":"missing_count = data.isna().sum()\nmissing_pct = (missing_count / len(data) * 100).round(2)\nmissing_table = pd.DataFrame({\"missing_count\": missing_count, \"missing_pct\": missing_pct})\ndisplay(missing_table)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.679887Z","iopub.execute_input":"2025-10-31T16:32:07.680171Z","iopub.status.idle":"2025-10-31T16:32:07.696823Z","shell.execute_reply.started":"2025-10-31T16:32:07.680149Z","shell.execute_reply":"2025-10-31T16:32:07.696110Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Column Categorization","metadata":{}},{"cell_type":"code","source":"# Numeric vs categorical\nnumeric_cols = data.select_dtypes(include=[np.number]).columns.tolist()\nlow_cardinality_numeric = [c for c in numeric_cols if data[c].nunique() <= 10]\ncategorical_cols = [c for c in data.columns if c not in numeric_cols]\ncategorical_like_cols = sorted(set(categorical_cols + low_cardinality_numeric))\nstrict_numeric_cols = [c for c in numeric_cols if c not in low_cardinality_numeric]\n\nprint(\"Strict numeric columns:\", len(strict_numeric_cols))\nprint(\"Categorical-like columns:\", len(categorical_like_cols))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.698459Z","iopub.execute_input":"2025-10-31T16:32:07.698653Z","iopub.status.idle":"2025-10-31T16:32:07.711151Z","shell.execute_reply.started":"2025-10-31T16:32:07.698639Z","shell.execute_reply":"2025-10-31T16:32:07.710562Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Descriptive Statistics","metadata":{}},{"cell_type":"code","source":"# Numeric summary\nif strict_numeric_cols:\n    display(data[strict_numeric_cols].describe().T)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.711790Z","iopub.execute_input":"2025-10-31T16:32:07.712054Z","iopub.status.idle":"2025-10-31T16:32:07.734401Z","shell.execute_reply.started":"2025-10-31T16:32:07.712040Z","shell.execute_reply":"2025-10-31T16:32:07.733871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Categorical summary\nif categorical_like_cols:\n    cat_summary = pd.DataFrame({\n        \"unique_values\": [data[c].nunique() for c in categorical_like_cols],\n        \"top_value\": [data[c].mode()[0] if data[c].notna().any() else np.nan for c in categorical_like_cols],\n        \"top_count\": [data[c].value_counts().iloc[0] if data[c].notna().any() else np.nan for c in categorical_like_cols]\n    }, index=categorical_like_cols)\n    display(cat_summary)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.735007Z","iopub.execute_input":"2025-10-31T16:32:07.735261Z","iopub.status.idle":"2025-10-31T16:32:07.752383Z","shell.execute_reply.started":"2025-10-31T16:32:07.735239Z","shell.execute_reply":"2025-10-31T16:32:07.751731Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Helper Functions","metadata":{}},{"cell_type":"code","source":"def save_plot(fig_title):\n    \"\"\"Save matplotlib figure to SAVE_DIR\"\"\"\n    fname = \"\".join(ch if ch.isalnum() or ch in \"-_.\" else \"_\" for ch in fig_title)\n    plt.savefig(Path(SAVE_DIR)/f\"{fname}.png\", bbox_inches=\"tight\", dpi=150)\n    plt.show()\n    plt.close()\n\ndef fd_bins(series):\n    \"\"\"Freedman-Diaconis bin calculation\"\"\"\n    s = series.dropna().astype(float)\n    if s.size < 2: return 10\n    q1, q3 = np.percentile(s, [25, 75])\n    iqr = q3 - q1\n    if iqr <= 0: return min(max(s.nunique(), 10), 50)\n    width = 2 * iqr * (len(s) ** (-1/3))\n    if width <= 0: return 30\n    return int(np.clip((s.max() - s.min()) / width, 10, 100))\n\ndef iqr_outlier_percentage(series):\n    \"\"\"Calculate percentage of outliers using IQR method\"\"\"\n    s = series.dropna().astype(float)\n    if s.empty: return np.nan\n    q1, q3 = np.percentile(s, [25, 75])\n    iqr = q3 - q1\n    if iqr <= 0: return 0.0\n    lower, upper = q1 - 1.5*iqr, q3 + 1.5*iqr\n    return ((s < lower) | (s > upper)).mean() * 100\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.753016Z","iopub.execute_input":"2025-10-31T16:32:07.753224Z","iopub.status.idle":"2025-10-31T16:32:07.760098Z","shell.execute_reply.started":"2025-10-31T16:32:07.753209Z","shell.execute_reply":"2025-10-31T16:32:07.759374Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Univariate Analysis (Numeric)","metadata":{}},{"cell_type":"code","source":"numeric_summary = []\n\nfor col in strict_numeric_cols:\n    series = data[col].dropna().astype(float)\n    print(f\"\\n--- Column: {col} ---\")\n    display(series.describe(percentiles=[.01,.05,.25,.5,.75,.95,.99]).to_frame().T)\n    \n    # Outliers\n    pct_out = iqr_outlier_percentage(series)\n    print(f\"IQR outlier %: {pct_out:.2f}%\")\n    \n    # Histogram\n    plt.figure(figsize=(8,4))\n    if not series.empty:\n        plt.hist(series, bins=fd_bins(series))\n    else:\n        plt.text(0.5, 0.5, \"No data\", ha=\"center\")\n    plt.title(f\"Histogram: {col}\")\n    plt.xlabel(col)\n    plt.ylabel(\"Count\")\n    save_plot(f\"hist_{col}\")\n    \n    # Boxplot\n    plt.figure(figsize=(6,4))\n    if not series.empty:\n        plt.boxplot(series, vert=True, labels=[col], showfliers=True)\n    plt.title(f\"Boxplot: {col}\")\n    save_plot(f\"box_{col}\")\n    \n    numeric_summary.append({\"column\": col, \"iqr_outlier_pct\": pct_out})\n    \n# Summary Table\ndisplay(pd.DataFrame(numeric_summary).sort_values(\"iqr_outlier_pct\", ascending=False))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:07.761737Z","iopub.execute_input":"2025-10-31T16:32:07.761921Z","iopub.status.idle":"2025-10-31T16:32:09.426051Z","shell.execute_reply.started":"2025-10-31T16:32:07.761908Z","shell.execute_reply":"2025-10-31T16:32:09.425478Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Categorical Analysis","metadata":{}},{"cell_type":"code","source":"for col in categorical_like_cols:\n    plt.figure(figsize=(8,4))\n    vc = data[col].astype(\"category\").value_counts().head(20)\n    vc.plot(kind=\"bar\")\n    plt.title(f\"Top 20 categories: {col}\")\n    plt.xlabel(col)\n    plt.ylabel(\"Count\")\n    save_plot(f\"bar_{col}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:09.426830Z","iopub.execute_input":"2025-10-31T16:32:09.427033Z","iopub.status.idle":"2025-10-31T16:32:11.676584Z","shell.execute_reply.started":"2025-10-31T16:32:09.427018Z","shell.execute_reply":"2025-10-31T16:32:11.675785Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Correlation (Numeric)","metadata":{}},{"cell_type":"code","source":"if len(strict_numeric_cols) >= 2:\n    corr_matrix = data[strict_numeric_cols].corr(numeric_only=True)\n    plt.figure(figsize=(6,5))\n    im = plt.imshow(corr_matrix, interpolation=\"nearest\")\n    plt.colorbar(im, fraction=0.046, pad=0.04)\n    plt.title(\"Correlation Heatmap\")\n    plt.xticks(range(len(strict_numeric_cols)), strict_numeric_cols, rotation=90)\n    plt.yticks(range(len(strict_numeric_cols)), strict_numeric_cols)\n    save_plot(\"correlation_heatmap\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:11.677474Z","iopub.execute_input":"2025-10-31T16:32:11.677659Z","iopub.status.idle":"2025-10-31T16:32:12.137805Z","shell.execute_reply.started":"2025-10-31T16:32:11.677645Z","shell.execute_reply":"2025-10-31T16:32:12.137042Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Target Column Inference","metadata":{}},{"cell_type":"code","source":"def find_target_column(df):\n    \"\"\"Heuristic to detect numeric target column\"\"\"\n    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()\n    keywords = [\"target\",\"label\",\"biomass\",\"agb\",\"yield\",\"gdm\"]\n    candidates = [c for c in df.columns if any(k in c.lower() for k in keywords)]\n    numeric_candidates = [c for c in candidates if c in numeric_cols]\n    if numeric_candidates:\n        numeric_candidates.sort(key=lambda c: df[c].isna().sum())\n        return numeric_candidates[0]\n    return None\n\nTARGET = find_target_column(data)\nprint(\"Inferred target column:\", TARGET)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:12.139593Z","iopub.execute_input":"2025-10-31T16:32:12.139896Z","iopub.status.idle":"2025-10-31T16:32:12.145863Z","shell.execute_reply.started":"2025-10-31T16:32:12.139878Z","shell.execute_reply":"2025-10-31T16:32:12.145288Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Target Distribution","metadata":{}},{"cell_type":"code","source":"if TARGET and TARGET in data.columns:\n    plt.figure(figsize=(6,4))\n    y = data[TARGET]\n    if y.dtype.kind in \"ifu\":\n        s = y.dropna().astype(float)\n        if not s.empty:\n            plt.hist(s, bins=30)\n        else:\n            plt.text(0.5,0.5,\"No target data\", ha=\"center\")\n    else:\n        vc = y.astype(\"category\").value_counts()\n        vc.plot(kind=\"bar\")\n    plt.title(f\"Target distribution: {TARGET}\")\n    plt.xlabel(TARGET)\n    plt.ylabel(\"Count\")\n    save_plot(f\"target_dist_{TARGET}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:12.146685Z","iopub.execute_input":"2025-10-31T16:32:12.146866Z","iopub.status.idle":"2025-10-31T16:32:12.462221Z","shell.execute_reply.started":"2025-10-31T16:32:12.146851Z","shell.execute_reply":"2025-10-31T16:32:12.461484Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Numeric Predictors vs Target","metadata":{}},{"cell_type":"code","source":"if TARGET and TARGET in data.columns and data[TARGET].dtype.kind in \"ifu\":\n    for col in [c for c in strict_numeric_cols if c != TARGET]:\n        plt.figure(figsize=(5,4))\n        xs, ys = data[col].astype(float), data[TARGET].astype(float)\n        mask = xs.notna() & ys.notna()\n        if mask.sum():\n            plt.scatter(xs[mask], ys[mask], s=10, alpha=0.6)\n        else:\n            plt.text(0.5,0.5,\"No overlapping data\", ha=\"center\")\n        plt.title(f\"{col} vs {TARGET}\")\n        plt.xlabel(col)\n        plt.ylabel(TARGET)\n        save_plot(f\"scatter_{col}_vs_{TARGET}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:12.463068Z","iopub.execute_input":"2025-10-31T16:32:12.463379Z","iopub.status.idle":"2025-10-31T16:32:13.096044Z","shell.execute_reply.started":"2025-10-31T16:32:12.463354Z","shell.execute_reply":"2025-10-31T16:32:13.095315Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Categorical Predictors vs Target","metadata":{}},{"cell_type":"code","source":"if TARGET and TARGET in data.columns and len(categorical_like_cols) > 0 and data[TARGET].dtype.kind in \"ifu\":\n    for col in categorical_like_cols:\n        tmp = data[[col,TARGET]].dropna()\n        if tmp.empty: continue\n        top_levels = tmp[col].value_counts().head(12).index\n        data_list = [tmp[tmp[col]==lvl][TARGET].values for lvl in top_levels]\n        plt.figure(figsize=(8,4))\n        plt.boxplot(data_list, labels=[str(l) for l in top_levels], showfliers=False)\n        plt.title(f\"{TARGET} by {col} (Top 12)\")\n        plt.xlabel(col)\n        plt.ylabel(TARGET)\n        save_plot(f\"box_{TARGET}_by_{col}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:13.096874Z","iopub.execute_input":"2025-10-31T16:32:13.097436Z","iopub.status.idle":"2025-10-31T16:32:15.152031Z","shell.execute_reply.started":"2025-10-31T16:32:13.097412Z","shell.execute_reply":"2025-10-31T16:32:15.151265Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Outlier Summary Table","metadata":{}},{"cell_type":"code","source":"outlier_rows = []\nfor col in strict_numeric_cols:\n    s = data[col].dropna().astype(float)\n    if s.empty:\n        outlier_rows.append((col, np.nan, np.nan, np.nan))\n        continue\n    q1, q3 = np.percentile(s,[25,75])\n    iqr = q3 - q1\n    if iqr <= 0:\n        outlier_rows.append((col,q1,q3,0.0))\n        continue\n    lower, upper = q1-1.5*iqr, q3+1.5*iqr\n    pct = ((s<lower) | (s>upper)).mean()*100\n    outlier_rows.append((col, round(q1,4), round(q3,4), round(pct,3)))\n\noutlier_summary = pd.DataFrame(outlier_rows, columns=[\"column\",\"Q1\",\"Q3\",\"%_outliers_IQR\"])\ndisplay(outlier_summary.sort_values(\"%_outliers_IQR\", ascending=False))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.152894Z","iopub.execute_input":"2025-10-31T16:32:15.153443Z","iopub.status.idle":"2025-10-31T16:32:15.167538Z","shell.execute_reply.started":"2025-10-31T16:32:15.153423Z","shell.execute_reply":"2025-10-31T16:32:15.166847Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CSIRO-specific Pivot / Wide-table Creation","metadata":{}},{"cell_type":"code","source":"EXPECTED_TARGETS = [\"Dry_Green_g\",\"Dry_Dead_g\",\"Dry_Clover_g\",\"GDM_g\",\"Dry_Total_g\"]\n\n# Pivot long -> wide\npivot_df = data.pivot_table(\n    index=\"image_path\",\n    columns=\"target_name\",\n    values=\"target\",\n    aggfunc=\"mean\"\n)\npivot_df = pivot_df.reindex(columns=EXPECTED_TARGETS)\n\n# Merge metadata (first row per image)\nmeta_cols = [c for c in [\"Sampling_Date\",\"State\",\"Pre_GSHH_NDVI\",\"Height_Ave_cm\",\"Species\"] if c in data.columns]\nmeta = data.drop_duplicates(subset=[\"image_path\"]).set_index(\"image_path\")[meta_cols]\nwide_df = pivot_df.join(meta, how=\"left\").reset_index()\n\nprint(\"Wide table shape:\", wide_df.shape)\ndisplay(wide_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.168673Z","iopub.execute_input":"2025-10-31T16:32:15.168866Z","iopub.status.idle":"2025-10-31T16:32:15.190776Z","shell.execute_reply.started":"2025-10-31T16:32:15.168851Z","shell.execute_reply":"2025-10-31T16:32:15.190235Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Target Relationships Check","metadata":{}},{"cell_type":"code","source":"# Dry_Total residual\nresidual_total = wide_df[\"Dry_Total_g\"] - (wide_df[\"Dry_Green_g\"].fillna(0) + wide_df[\"Dry_Dead_g\"].fillna(0) + wide_df[\"Dry_Clover_g\"].fillna(0))\nplt.figure(figsize=(6,4))\nplt.hist(residual_total.dropna(), bins=fd_bins(residual_total))\nplt.title(\"Residual: Dry_Total_g - sum(Green+Dead+Clover)\")\nplt.xlabel(\"Residual\"); plt.ylabel(\"Count\")\nplt.show()\n\n# GDM vs Dry_Green scatter\nplt.figure(figsize=(5,4))\nmask = wide_df[\"GDM_g\"].notna() & wide_df[\"Dry_Green_g\"].notna()\nplt.scatter(wide_df.loc[mask,\"Dry_Green_g\"], wide_df.loc[mask,\"GDM_g\"], s=10, alpha=0.6)\nplt.title(\"Dry_Green_g vs GDM_g\")\nplt.xlabel(\"Dry_Green_g\"); plt.ylabel(\"GDM_g\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.191400Z","iopub.execute_input":"2025-10-31T16:32:15.191637Z","iopub.status.idle":"2025-10-31T16:32:15.495468Z","shell.execute_reply.started":"2025-10-31T16:32:15.191622Z","shell.execute_reply":"2025-10-31T16:32:15.494722Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Optional Aggregated Plots","metadata":{}},{"cell_type":"code","source":"# Daily trends\nif \"Sampling_Date\" in wide_df.columns and np.issubdtype(wide_df[\"Sampling_Date\"].dtype, np.datetime64):\n    daily_mean = wide_df.groupby(wide_df[\"Sampling_Date\"].dt.date)[EXPECTED_TARGETS].mean()\n    daily_mean.plot(figsize=(10,4))\n    plt.title(\"Daily mean of targets\")\n    plt.xticks(rotation=45)\n    plt.show()\n\n# State-level\nif \"State\" in wide_df.columns:\n    state_grp = wide_df.groupby(\"State\")[EXPECTED_TARGETS].mean().sort_values(\"Dry_Total_g\", ascending=False).head(10)\n    state_grp[\"Dry_Total_g\"].plot(kind=\"bar\", figsize=(8,4))\n    plt.title(\"Top 10 States by mean Dry_Total_g\")\n    plt.ylabel(\"Dry_Total_g\")\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.496331Z","iopub.execute_input":"2025-10-31T16:32:15.496597Z","iopub.status.idle":"2025-10-31T16:32:15.628654Z","shell.execute_reply.started":"2025-10-31T16:32:15.496568Z","shell.execute_reply":"2025-10-31T16:32:15.628040Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ML Algorithm","metadata":{}},{"cell_type":"code","source":"# Basic ML imports\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import mean_squared_error, r2_score\n\n# Models\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nimport xgboost as xgb\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.630961Z","iopub.execute_input":"2025-10-31T16:32:15.631570Z","iopub.status.idle":"2025-10-31T16:32:15.637099Z","shell.execute_reply.started":"2025-10-31T16:32:15.631550Z","shell.execute_reply":"2025-10-31T16:32:15.636301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = wide_df.copy()  \nTARGET = \"Dry_Total_g\" \n\n# Drop rows with missing target\ndf = df.dropna(subset=[TARGET])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.637910Z","iopub.execute_input":"2025-10-31T16:32:15.638167Z","iopub.status.idle":"2025-10-31T16:32:15.651935Z","shell.execute_reply.started":"2025-10-31T16:32:15.638152Z","shell.execute_reply":"2025-10-31T16:32:15.651330Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Identify features","metadata":{}},{"cell_type":"code","source":"# Numeric features\nnumeric_features = df.select_dtypes(include=[np.number]).columns.tolist()\nnumeric_features = [c for c in numeric_features if c != TARGET]\n\n# Categorical features\ncategorical_features = df.select_dtypes(exclude=[np.number]).columns.tolist()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.652606Z","iopub.execute_input":"2025-10-31T16:32:15.653097Z","iopub.status.idle":"2025-10-31T16:32:15.664631Z","shell.execute_reply.started":"2025-10-31T16:32:15.653073Z","shell.execute_reply":"2025-10-31T16:32:15.664046Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train-Test Split","metadata":{}},{"cell_type":"code","source":"X = df[numeric_features + categorical_features]\ny = df[TARGET]\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\nprint(\"Train shape:\", X_train.shape, \"Test shape:\", X_test.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.665379Z","iopub.execute_input":"2025-10-31T16:32:15.665655Z","iopub.status.idle":"2025-10-31T16:32:15.682896Z","shell.execute_reply.started":"2025-10-31T16:32:15.665633Z","shell.execute_reply":"2025-10-31T16:32:15.682329Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"code","source":"# Numeric: scale | Categorical: one-hot encode\npreprocessor = ColumnTransformer(transformers=[\n    ('num', StandardScaler(), numeric_features),\n    ('cat', OneHotEncoder(handle_unknown='ignore'), categorical_features)\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.683553Z","iopub.execute_input":"2025-10-31T16:32:15.683814Z","iopub.status.idle":"2025-10-31T16:32:15.695838Z","shell.execute_reply.started":"2025-10-31T16:32:15.683794Z","shell.execute_reply":"2025-10-31T16:32:15.695221Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Model Pipelines","metadata":{}},{"cell_type":"code","source":"# Linear Regression\nlr_pipeline = Pipeline([\n    ('preprocess', preprocessor),\n    ('model', LinearRegression())\n])\n\n# Random Forest\nrf_pipeline = Pipeline([\n    ('preprocess', preprocessor),\n    ('model', RandomForestRegressor(n_estimators=200, random_state=42))\n])\n\n# XGBoost\nxgb_pipeline = Pipeline([\n    ('preprocess', preprocessor),\n    ('model', xgb.XGBRegressor(n_estimators=200, learning_rate=0.1, random_state=42))\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.696532Z","iopub.execute_input":"2025-10-31T16:32:15.696760Z","iopub.status.idle":"2025-10-31T16:32:15.709173Z","shell.execute_reply.started":"2025-10-31T16:32:15.696746Z","shell.execute_reply":"2025-10-31T16:32:15.708569Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train & Evaluate","metadata":{}},{"cell_type":"code","source":"pipelines = {\n    'Linear Regression': lr_pipeline,\n    'Random Forest': rf_pipeline,\n    'XGBoost': xgb_pipeline\n}\n\nresults = []\n\nfor name, pipe in pipelines.items():\n    pipe.fit(X_train, y_train)                # Train the model\n    y_pred = pipe.predict(X_test)             # Predict on test set\n    \n    rmse = mean_squared_error(y_test, y_pred, squared=False)  # Root Mean Squared Error\n    r2 = r2_score(y_test, y_pred)                               # R-squared\n    \n    results.append({'Model': name, 'RMSE': rmse, 'R2': r2})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:15.709794Z","iopub.execute_input":"2025-10-31T16:32:15.710032Z","iopub.status.idle":"2025-10-31T16:32:16.739382Z","shell.execute_reply.started":"2025-10-31T16:32:15.710008Z","shell.execute_reply":"2025-10-31T16:32:16.738825Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results_df = pd.DataFrame(results)\n\nresults_df['RMSE'] = results_df['RMSE'].round(3)\nresults_df['R2'] = results_df['R2'].round(3)\nresults_df = results_df.sort_values('RMSE')  # RMSE  sort\ndisplay(results_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:16.739836Z","iopub.execute_input":"2025-10-31T16:32:16.739998Z","iopub.status.idle":"2025-10-31T16:32:16.749394Z","shell.execute_reply.started":"2025-10-31T16:32:16.739982Z","shell.execute_reply":"2025-10-31T16:32:16.748527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model = rf_pipeline  # ধরো Random Forest best\ny_pred = best_model.predict(X_test)\n\nplt.figure(figsize=(6,6))\nplt.scatter(y_test, y_pred, alpha=0.6)\nplt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--')  # perfect prediction line\nplt.xlabel(\"Actual Dry_Total_g\")\nplt.ylabel(\"Predicted Dry_Total_g\")\nplt.title(\"Actual vs Predicted (Random Forest)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:16.750938Z","iopub.execute_input":"2025-10-31T16:32:16.751255Z","iopub.status.idle":"2025-10-31T16:32:16.911518Z","shell.execute_reply.started":"2025-10-31T16:32:16.751238Z","shell.execute_reply":"2025-10-31T16:32:16.910481Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Importance (Tree-based models)","metadata":{}},{"cell_type":"code","source":"# Only for RF or XGB\nmodel = rf_pipeline.named_steps['model']\n# Preprocessed feature names\nohe_cols = rf_pipeline.named_steps['preprocess'].named_transformers_['cat'].get_feature_names_out(categorical_features)\nall_features = numeric_features + list(ohe_cols)\n\nimportances = model.feature_importances_\nfeat_imp = pd.DataFrame({'feature': all_features, 'importance': importances}).sort_values('importance', ascending=False)\nplt.figure(figsize=(10,6))\nsns.barplot(x='importance', y='feature', data=feat_imp.head(20))\nplt.title(\"Top 20 Feature Importances (Random Forest)\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:32:16.912423Z","iopub.execute_input":"2025-10-31T16:32:16.912644Z","iopub.status.idle":"2025-10-31T16:32:17.307910Z","shell.execute_reply.started":"2025-10-31T16:32:16.912628Z","shell.execute_reply":"2025-10-31T16:32:17.307146Z"}},"outputs":[],"execution_count":null}]}