{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"ddba2680","cell_type":"code","source":"# ======================================================\n# CSIRO Biomass – EDA Notebook\n# ======================================================\n\n# ===============================\n# Imports\n# ===============================\nimport os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nimport random\n\n# ===============================\n# 1️⃣ Paths\n# ===============================\nif \"KAGGLE_URL_BASE\" in os.environ:\n    data_dir = \"/kaggle/input/csiro-biomass\"\nelse:\n    data_dir = \"/Users/sannaka/Documents/Projects/Kaggle/csiro-biomass/data\"\n\ntrain_csv = os.path.join(data_dir, \"train.csv\")\ntest_csv = os.path.join(data_dir, \"test.csv\")\ntrain_images_dir = data_dir\ntest_images_dir = data_dir\n\n# ===============================\n# 2️⃣ Load Data\n# ===============================\ntrain_df = pd.read_csv(train_csv)\ntest_df = pd.read_csv(test_csv)\n\nprint(\"Train shape:\", train_df.shape)\nprint(\"Test shape:\", test_df.shape)\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:49.447818Z","iopub.execute_input":"2025-10-31T08:50:49.448160Z","iopub.status.idle":"2025-10-31T08:50:49.492600Z","shell.execute_reply.started":"2025-10-31T08:50:49.448137Z","shell.execute_reply":"2025-10-31T08:50:49.491219Z"}},"outputs":[],"execution_count":null},{"id":"737b0b29","cell_type":"code","source":"# ===============================\n# 3️⃣ Check Missing Values\n# ===============================\nprint(\"Missing values in train:\")\nprint(train_df.isna().sum())\n\nprint(\"\\nMissing values in test:\")\nprint(test_df.isna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:49.494639Z","iopub.execute_input":"2025-10-31T08:50:49.494967Z","iopub.status.idle":"2025-10-31T08:50:49.507621Z","shell.execute_reply.started":"2025-10-31T08:50:49.494944Z","shell.execute_reply":"2025-10-31T08:50:49.506059Z"}},"outputs":[],"execution_count":null},{"id":"55014829","cell_type":"code","source":"# ===============================\n# 4️⃣ Numeric Features Summary\n# ===============================\nnumeric_cols = ['Height_Ave_cm', 'Pre_GSHH_NDVI']  \ntrain_df[numeric_cols].describe()\n\n# Histograms\ntrain_df[numeric_cols].hist(bins=20, figsize=(10,4))\nplt.suptitle(\"Numeric Feature Distributions\")\nplt.show()\n\n# Boxplots\nplt.figure(figsize=(10,4))\nfor i, col in enumerate(numeric_cols):\n    plt.subplot(1,len(numeric_cols),i+1)\n    sns.boxplot(y=train_df[col])\n    plt.title(col)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:49.509624Z","iopub.execute_input":"2025-10-31T08:50:49.510072Z","iopub.status.idle":"2025-10-31T08:50:50.384178Z","shell.execute_reply.started":"2025-10-31T08:50:49.510034Z","shell.execute_reply":"2025-10-31T08:50:50.382865Z"}},"outputs":[],"execution_count":null},{"id":"44980546","cell_type":"code","source":"# ===============================\n# 5️⃣ Categorical Features Summary (Readable)\n# ===============================\ncategorical_cols = ['Species', 'State', 'Sampling_Date']  \n\nfor col in categorical_cols:\n    print(f\"\\nColumn: {col}\")\n    counts = train_df[col].value_counts()\n    print(counts)\n    \n    # Horizontal bar plot for better readability\n    plt.figure(figsize=(8,4))\n    counts.plot(kind='barh', color='skyblue')\n    plt.title(f\"Counts of {col}\")\n    plt.xlabel(\"Number of samples\")\n    plt.ylabel(col)\n    plt.gca().invert_yaxis()  # largest bars on top\n    plt.tight_layout()\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:50.385286Z","iopub.execute_input":"2025-10-31T08:50:50.385622Z","iopub.status.idle":"2025-10-31T08:50:51.331376Z","shell.execute_reply.started":"2025-10-31T08:50:50.385596Z","shell.execute_reply":"2025-10-31T08:50:51.330127Z"}},"outputs":[],"execution_count":null},{"id":"49a309c7","cell_type":"code","source":"# ===============================\n# 6️⃣ Target Variable Analysis\n# ===============================\ntarget_col = \"target\"\nsns.histplot(train_df[target_col], bins=30, kde=True)\nplt.title(\"Target Distribution\")\nplt.show()\n\nprint(\"Target statistics:\")\nprint(train_df[target_col].describe())\n\n# Optional: log-transform if skewed\ntrain_df[target_col + \"_log\"] = np.log1p(train_df[target_col])\nsns.histplot(train_df[target_col + \"_log\"], bins=30, kde=True)\nplt.title(\"Log-transformed Target Distribution\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:51.334142Z","iopub.execute_input":"2025-10-31T08:50:51.334486Z","iopub.status.idle":"2025-10-31T08:50:51.997544Z","shell.execute_reply.started":"2025-10-31T08:50:51.334462Z","shell.execute_reply":"2025-10-31T08:50:51.996152Z"}},"outputs":[],"execution_count":null},{"id":"7b70df60","cell_type":"code","source":"# ===============================\n# 7️⃣ Correlation Analysis\n# ===============================\ncorr = train_df[numeric_cols + [target_col]].corr()\nplt.figure(figsize=(6,5))\nsns.heatmap(corr, annot=True, cmap='coolwarm')\nplt.title(\"Correlation Matrix\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:51.998548Z","iopub.execute_input":"2025-10-31T08:50:51.998863Z","iopub.status.idle":"2025-10-31T08:50:52.252890Z","shell.execute_reply.started":"2025-10-31T08:50:51.998838Z","shell.execute_reply":"2025-10-31T08:50:52.251716Z"}},"outputs":[],"execution_count":null},{"id":"8f6f5fcb","cell_type":"code","source":"# ===============================\n# 8️⃣ Numeric vs Target\n# ===============================\nplt.figure(figsize=(10,4))\nfor i, col in enumerate(numeric_cols):\n    plt.subplot(1,len(numeric_cols),i+1)\n    sns.scatterplot(x=train_df[col], y=train_df[target_col])\n    plt.title(f\"{col} vs Target\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:52.254110Z","iopub.execute_input":"2025-10-31T08:50:52.254411Z","iopub.status.idle":"2025-10-31T08:50:52.683099Z","shell.execute_reply.started":"2025-10-31T08:50:52.254385Z","shell.execute_reply":"2025-10-31T08:50:52.681778Z"}},"outputs":[],"execution_count":null},{"id":"eb59be04","cell_type":"code","source":"# ===============================\n# 9️⃣ Categorical vs Target\n# ===============================\nfor col in categorical_cols:\n    plt.figure(figsize=(6,4))\n    sns.boxplot(x=train_df[col], y=train_df[target_col])\n    plt.title(f\"{col} vs Target\")\n    plt.xticks(rotation=45)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:52.684181Z","iopub.execute_input":"2025-10-31T08:50:52.684472Z","iopub.status.idle":"2025-10-31T08:50:53.900457Z","shell.execute_reply.started":"2025-10-31T08:50:52.684449Z","shell.execute_reply":"2025-10-31T08:50:53.899388Z"}},"outputs":[],"execution_count":null},{"id":"ec76b04f","cell_type":"code","source":"# ===============================\n# 🔟 Feature Interactions\n# ===============================\nif 'Height_Ave_cm' in train_df.columns and 'Pre_GSHH_NDVI' in train_df.columns:\n    train_df['Height_x_NDVI'] = train_df['Height_Ave_cm'] * train_df['Pre_GSHH_NDVI']\n    sns.scatterplot(x='Height_x_NDVI', y=target_col, data=train_df)\n    plt.title(\"Height × NDVI vs Target\")\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:53.901478Z","iopub.execute_input":"2025-10-31T08:50:53.901769Z","iopub.status.idle":"2025-10-31T08:50:54.132684Z","shell.execute_reply.started":"2025-10-31T08:50:53.901748Z","shell.execute_reply":"2025-10-31T08:50:54.131657Z"}},"outputs":[],"execution_count":null},{"id":"aa85f0aa","cell_type":"code","source":"# ===============================\n# 11️⃣ Sample Image Visualization\n# ===============================\nsample_images = train_df['image_path'].sample(6).values\nplt.figure(figsize=(12,6))\nfor i, img_file in enumerate(sample_images):\n    img_path = os.path.join(train_images_dir, img_file)\n    img = Image.open(img_path)\n    plt.subplot(2,3,i+1)\n    plt.imshow(img)\n    plt.axis('off')\n    plt.title(img_file)\nplt.suptitle(\"Sample Training Images\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T08:50:54.133721Z","iopub.execute_input":"2025-10-31T08:50:54.134090Z","iopub.status.idle":"2025-10-31T08:50:56.767171Z","shell.execute_reply.started":"2025-10-31T08:50:54.134060Z","shell.execute_reply":"2025-10-31T08:50:56.765867Z"}},"outputs":[],"execution_count":null}]}