{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-16T14:05:35.682414Z","iopub.execute_input":"2025-11-16T14:05:35.683105Z","iopub.status.idle":"2025-11-16T14:05:35.687938Z","shell.execute_reply.started":"2025-11-16T14:05:35.683081Z","shell.execute_reply":"2025-11-16T14:05:35.687034Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load data\nbase_path = \"/kaggle/input/csiro-biomass/\"\ndf = pd.read_csv(base_path + \"train.csv\")\n\n# Create directory for plots\nos.makedirs(\"/kaggle/working/plots\", exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T13:34:14.070768Z","iopub.execute_input":"2025-11-16T13:34:14.071130Z","iopub.status.idle":"2025-11-16T13:34:14.096685Z","shell.execute_reply.started":"2025-11-16T13:34:14.071101Z","shell.execute_reply":"2025-11-16T13:34:14.095769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 1. BASIC DATA SUMMARY\n# ============================================================\nprint(\"\\n===== HEAD =====\")\nprint(df.head())\n\nprint(\"\\n===== SHAPE =====\")\nprint(df.shape)\n\nprint(\"\\n===== DATA TYPES =====\")\nprint(df.dtypes)\n\nprint(\"\\n===== MISSING VALUES =====\")\nprint(df.isna().sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T13:34:37.566872Z","iopub.execute_input":"2025-11-16T13:34:37.567765Z","iopub.status.idle":"2025-11-16T13:34:37.586984Z","shell.execute_reply.started":"2025-11-16T13:34:37.567734Z","shell.execute_reply":"2025-11-16T13:34:37.586239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 2. NUMERIC STATISTICS\n# ============================================================\nnumeric_cols = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\", \"target\"]\n\nprint(\"\\n===== NUMERIC SUMMARY =====\")\nprint(df[numeric_cols].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T13:35:25.087720Z","iopub.execute_input":"2025-11-16T13:35:25.088049Z","iopub.status.idle":"2025-11-16T13:35:25.115371Z","shell.execute_reply.started":"2025-11-16T13:35:25.088023Z","shell.execute_reply":"2025-11-16T13:35:25.114309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 3. CATEGORICAL SUMMARY\n# ============================================================\ncategorical_cols = [\"State\", \"Species\", \"target_name\"]\n\nfor c in categorical_cols:\n    print(f\"\\n===== {c} — Unique Values =====\")\n    print(df[c].value_counts().head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T13:38:05.080764Z","iopub.execute_input":"2025-11-16T13:38:05.081129Z","iopub.status.idle":"2025-11-16T13:38:05.093570Z","shell.execute_reply.started":"2025-11-16T13:38:05.081105Z","shell.execute_reply":"2025-11-16T13:38:05.092698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 4. PIVOT TARGET TABLE (WIDE FORMAT)\n# ============================================================\ndf_wide = df.pivot_table(\n    index=[\"image_path\", \"Sampling_Date\", \"State\", \"Species\",\n           \"Pre_GSHH_NDVI\", \"Height_Ave_cm\"],\n    columns=\"target_name\",\n    values=\"target\",\n    aggfunc=\"first\"\n).reset_index()\n\nprint(\"\\n===== WIDE FORMAT SHAPE =====\")\nprint(df_wide.shape)\n\nprint(\"\\n===== WIDE FORMAT HEAD =====\")\nprint(df_wide.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T13:48:48.153779Z","iopub.execute_input":"2025-11-16T13:48:48.154168Z","iopub.status.idle":"2025-11-16T13:48:48.178202Z","shell.execute_reply.started":"2025-11-16T13:48:48.154139Z","shell.execute_reply":"2025-11-16T13:48:48.177352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n===== MISSING VALUES =====\")\nprint(df_wide.isna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T13:56:42.291343Z","iopub.execute_input":"2025-11-16T13:56:42.292149Z","iopub.status.idle":"2025-11-16T13:56:42.298805Z","shell.execute_reply.started":"2025-11-16T13:56:42.292120Z","shell.execute_reply":"2025-11-16T13:56:42.297619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 5. TARGET DISTRIBUTIONS\n# ============================================================\ntarget_cols = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\n# Create a 2x3 grid of subplots\nfig, axes = plt.subplots(2, 3, figsize=(18, 8))\naxes = axes.flatten()  # Flatten to easily index\n\nfor i, t in enumerate(target_cols):\n    sns.histplot(df_wide[t], kde=True, ax=axes[i])\n    axes[i].set_title(f\"Distribution of {t}\")\n\n# Hide the unused subplot (6th one)\nif len(target_cols) < len(axes):\n    for j in range(len(target_cols), len(axes)):\n        fig.delaxes(axes[j])\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T14:06:04.891505Z","iopub.execute_input":"2025-11-16T14:06:04.891812Z","iopub.status.idle":"2025-11-16T14:06:06.537723Z","shell.execute_reply.started":"2025-11-16T14:06:04.891792Z","shell.execute_reply":"2025-11-16T14:06:06.536786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 6. BOX PLOTS FOR TARGETS\n# ============================================================\nplt.figure(figsize=(10,5))\nsns.boxplot(data=df_wide[target_cols])\nplt.title(\"Boxplot of All Biomass Targets\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T13:58:51.703361Z","iopub.execute_input":"2025-11-16T13:58:51.703730Z","iopub.status.idle":"2025-11-16T13:58:51.903076Z","shell.execute_reply.started":"2025-11-16T13:58:51.703706Z","shell.execute_reply":"2025-11-16T13:58:51.901975Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 7. CORRELATION ANALYSIS\n# ============================================================\ncorr = df_wide[target_cols + [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\"]].corr()\n\nplt.figure(figsize=(10,8))\nsns.heatmap(corr, annot=True, cmap=\"coolwarm\", fmt=\".2f\")\nplt.title(\"Correlation Heatmap — Targets and Metadata\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T14:00:25.968905Z","iopub.execute_input":"2025-11-16T14:00:25.969249Z","iopub.status.idle":"2025-11-16T14:00:26.324687Z","shell.execute_reply.started":"2025-11-16T14:00:25.969227Z","shell.execute_reply":"2025-11-16T14:00:26.323777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 8. NDVI & HEIGHT RELATIONSHIPS\n# ============================================================\ntarget_cols = [\"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\", \"Dry_Total_g\"]\n\n# Create a 5-row, 2-column grid of subplots\nfig, axes = plt.subplots(nrows=len(target_cols), ncols=2, figsize=(14, 20))\nfig.suptitle(\"Scatter Plots: NDVI & Height vs Target Variables\", fontsize=16, y=1.02)\n\nfor i, t in enumerate(target_cols):\n    # NDVI vs target\n    sns.scatterplot(x=df_wide[\"Pre_GSHH_NDVI\"], y=df_wide[t], alpha=0.6, ax=axes[i, 0])\n    axes[i, 0].set_title(f\"NDVI vs {t}\")\n    axes[i, 0].set_xlabel(\"Pre_GSHH_NDVI\")\n    axes[i, 0].set_ylabel(t)\n\n    # Height vs target\n    sns.scatterplot(x=df_wide[\"Height_Ave_cm\"], y=df_wide[t], alpha=0.6, ax=axes[i, 1])\n    axes[i, 1].set_title(f\"Height vs {t}\")\n    axes[i, 1].set_xlabel(\"Height_Ave_cm\")\n    axes[i, 1].set_ylabel(t)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T14:02:53.815000Z","iopub.execute_input":"2025-11-16T14:02:53.815405Z","iopub.status.idle":"2025-11-16T14:02:55.843561Z","shell.execute_reply.started":"2025-11-16T14:02:53.815377Z","shell.execute_reply":"2025-11-16T14:02:55.842338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 9. TARGET RELATIONSHIPS (PAIRPLOT)\n# ============================================================\nsns.pairplot(df_wide[target_cols], diag_kind=\"kde\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T14:05:47.021219Z","iopub.execute_input":"2025-11-16T14:05:47.021567Z","iopub.status.idle":"2025-11-16T14:05:52.302769Z","shell.execute_reply.started":"2025-11-16T14:05:47.021545Z","shell.execute_reply":"2025-11-16T14:05:52.301817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 10. CATEGORY DISTRIBUTIONS\n# ============================================================\nfor col in [\"State\", \"Species\"]:\n    plt.figure(figsize=(8,4))\n    df_wide[col].value_counts().plot(kind=\"bar\")\n    plt.title(f\"Distribution of {col}\")\n    plt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T14:16:39.484837Z","iopub.execute_input":"2025-11-16T14:16:39.485258Z","iopub.status.idle":"2025-11-16T14:16:39.898843Z","shell.execute_reply.started":"2025-11-16T14:16:39.485229Z","shell.execute_reply":"2025-11-16T14:16:39.897938Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 11. OUTLIER ANALYSIS\n# ============================================================\nnumeric_cols = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\"]\nplt.figure(figsize=(10,4))\nsns.boxplot(data=df_wide[numeric_cols])\nplt.title(\"Outlier Detection — Numeric Columns\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T14:15:14.345547Z","iopub.execute_input":"2025-11-16T14:15:14.345899Z","iopub.status.idle":"2025-11-16T14:15:14.514642Z","shell.execute_reply.started":"2025-11-16T14:15:14.345877Z","shell.execute_reply":"2025-11-16T14:15:14.513494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n===== HIGH TARGET OUTLIERS =====\")\nfor target in target_cols:\n    q3 = df_wide[target].quantile(0.75)\n    iqr = df_wide[target].quantile(0.75) - df_wide[target].quantile(0.25)\n    cutoff = q3 + 1.5 * iqr\n    outliers = df_wide[df_wide[target] > cutoff]\n    print(f\"{target}: {len(outliers)} outliers\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-16T14:15:46.129099Z","iopub.execute_input":"2025-11-16T14:15:46.129815Z","iopub.status.idle":"2025-11-16T14:15:46.149028Z","shell.execute_reply.started":"2025-11-16T14:15:46.129775Z","shell.execute_reply":"2025-11-16T14:15:46.147929Z"}},"outputs":[],"execution_count":null}]}