{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nfrom PIL import Image\nimport plotly.express as px","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:55:47.731939Z","iopub.execute_input":"2025-10-31T16:55:47.732192Z","iopub.status.idle":"2025-10-31T16:55:53.461811Z","shell.execute_reply.started":"2025-10-31T16:55:47.732159Z","shell.execute_reply":"2025-10-31T16:55:53.460994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.set_style('whitegrid')\npd.set_option('display.max_columns', 50)\npd.set_option('display.max_rows', 100)\n\nprint(\"Libraries imported successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:55:53.462672Z","iopub.execute_input":"2025-10-31T16:55:53.463233Z","iopub.status.idle":"2025-10-31T16:55:53.469923Z","shell.execute_reply.started":"2025-10-31T16:55:53.463199Z","shell.execute_reply":"2025-10-31T16:55:53.468946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/csiro-biomass/'\nTRAIN_CSV_PATH = os.path.join(BASE_PATH, 'train.csv')\nTEST_CSV_PATH = os.path.join(BASE_PATH, 'test.csv')\nSUBMISSION_CSV_PATH = os.path.join(BASE_PATH, 'sample_submission.csv')\nTRAIN_IMG_DIR = os.path.join(BASE_PATH, 'train/')\nTEST_IMG_DIR = os.path.join(BASE_PATH, 'test/')\n\ntry:\n    train_df = pd.read_csv(TRAIN_CSV_PATH)\n    test_df = pd.read_csv(TEST_CSV_PATH)\n    submission_df = pd.read_csv(SUBMISSION_CSV_PATH)\n    \n    print(\"CSV files loaded successfully!\")\n    print(f\"Train data shape: {train_df.shape}\")\n    print(f\"Test data shape: {test_df.shape}\")\n    print(f\"Submission data shape: {submission_df.shape}\")\n\nexcept FileNotFoundError as e:\n    print(f\"Error loading files: {e}\")\n    print(\"Please ensure you are running this in a Kaggle environment with the competition data attached.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:55:57.385086Z","iopub.execute_input":"2025-10-31T16:55:57.385402Z","iopub.status.idle":"2025-10-31T16:55:57.428036Z","shell.execute_reply.started":"2025-10-31T16:55:57.385380Z","shell.execute_reply":"2025-10-31T16:55:57.427041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*50)\nprint(\"--- Train DataFrame Head ---\")\nprint(\"=\"*50)\ndisplay(train_df.head())\n\nprint(\"\\n\" + \"=\"*50)\nprint(\"--- Test DataFrame Head ---\")\nprint(\"=\"*50)\ndisplay(test_df.head())\n\nprint(\"\\n\" + \"=\"*50)\nprint(\"--- Sample Submission DataFrame Head ---\")\nprint(\"=\"*50)\ndisplay(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:56:15.177882Z","iopub.execute_input":"2025-10-31T16:56:15.178939Z","iopub.status.idle":"2025-10-31T16:56:15.223550Z","shell.execute_reply.started":"2025-10-31T16:56:15.178905Z","shell.execute_reply":"2025-10-31T16:56:15.222807Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Deeper Dive into the Training Data","metadata":{}},{"cell_type":"code","source":"# --- Data Info and Null Check ---\nprint(\"=\"*50)\nprint(\"--- DataFrame Info ---\")\nprint(\"=\"*50)\ntrain_df.info()\n\n# --- Descriptive Statistics ---\nprint(\"\\n\" + \"=\"*50)\nprint(\"--- Descriptive Statistics for Numerical Columns (Long)---\")\nprint(\"=\"*50)\ndisplay(train_df.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:59:07.699420Z","iopub.execute_input":"2025-10-31T16:59:07.700213Z","iopub.status.idle":"2025-10-31T16:59:07.750344Z","shell.execute_reply.started":"2025-10-31T16:59:07.700176Z","shell.execute_reply":"2025-10-31T16:59:07.748887Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Reshaping the Data for Analysis","metadata":{}},{"cell_type":"code","source":"# --- Pivot the DataFrame ---\ntrain_wide_df = train_df.pivot_table(\n    index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'],\n    columns='target_name',\n    values='target'\n).reset_index()\n\n# Extract the unique image ID for easier access\ntrain_wide_df['image_id'] = train_wide_df['image_path'].apply(lambda x: os.path.splitext(os.path.basename(x))[0])\n\nprint(\"\\n\" + \"=\"*50)\nprint(\"--- Reshaped (Wide) DataFrame Head ---\")\nprint(\"=\"*50)\ndisplay(train_wide_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T16:59:41.425491Z","iopub.execute_input":"2025-10-31T16:59:41.425828Z","iopub.status.idle":"2025-10-31T16:59:41.478383Z","shell.execute_reply.started":"2025-10-31T16:59:41.425805Z","shell.execute_reply":"2025-10-31T16:59:41.477593Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\\nShape of the wide DataFrame: {train_wide_df.shape}\")\nprint(f\"Number of unique images: {train_df['image_path'].nunique()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:00:03.471553Z","iopub.execute_input":"2025-10-31T17:00:03.471919Z","iopub.status.idle":"2025-10-31T17:00:03.478049Z","shell.execute_reply.started":"2025-10-31T17:00:03.471891Z","shell.execute_reply":"2025-10-31T17:00:03.477268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Descriptive Statistics ---\nprint(\"\\n\" + \"=\"*50)\nprint(\"--- Descriptive Statistics for Wide DataFrame ---\")\nprint(\"=\"*50)\ndisplay(train_wide_df.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:00:09.672579Z","iopub.execute_input":"2025-10-31T17:00:09.672934Z","iopub.status.idle":"2025-10-31T17:00:09.700528Z","shell.execute_reply.started":"2025-10-31T17:00:09.672910Z","shell.execute_reply":"2025-10-31T17:00:09.699699Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Correlation Analysis","metadata":{}},{"cell_type":"code","source":"numeric_cols = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'GDM_g', 'Dry_Total_g']\n\n# --- Compute Correlation Matrix ---\ncorr_matrix = train_wide_df[numeric_cols].corr()\n\n# --- Visualize the Correlation Matrix with a Heatmap ---\nplt.figure(figsize=(12, 10))\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', linewidths=.5)\nplt.title('Correlation Matrix of Features and Biomass Targets', fontsize=16)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:04:04.407134Z","iopub.execute_input":"2025-10-31T17:04:04.407959Z","iopub.status.idle":"2025-10-31T17:04:04.978818Z","shell.execute_reply.started":"2025-10-31T17:04:04.407929Z","shell.execute_reply":"2025-10-31T17:04:04.977779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display the correlation matrix as a table\nprint(\"\\n\" + \"=\"*50)\nprint(\"--- Correlation Matrix ---\")\nprint(\"=\"*50)\ndisplay(corr_matrix)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:04:52.735143Z","iopub.execute_input":"2025-10-31T17:04:52.735479Z","iopub.status.idle":"2025-10-31T17:04:52.750528Z","shell.execute_reply.started":"2025-10-31T17:04:52.735455Z","shell.execute_reply":"2025-10-31T17:04:52.749624Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Categorical Feature Distribution","metadata":{}},{"cell_type":"code","source":"# --- State Distribution ---\nfig_state = px.bar(\n    train_wide_df['State'].value_counts().reset_index(),\n    x='State',\n    y='count',\n    title='Number of Samples per State',\n    labels={'State': 'State', 'count': 'Number of Images'},\n    color='State',\n    text_auto=True\n)\nfig_state.show(renderer='iframe')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:39:33.557528Z","iopub.execute_input":"2025-10-31T17:39:33.558280Z","iopub.status.idle":"2025-10-31T17:39:33.674117Z","shell.execute_reply.started":"2025-10-31T17:39:33.558256Z","shell.execute_reply":"2025-10-31T17:39:33.673346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Species Distribution ---\n# Let's look at the top 15 most common species combinations\nspecies_counts = train_wide_df['Species'].value_counts().nlargest(15).reset_index()\nfig_species = px.bar(\n    species_counts,\n    x='Species',\n    y='count',\n    title='Top 15 Most Common Pasture Species',\n    labels={'Species': 'Species Combination', 'count': 'Number of Images'},\n    color='count',\n    color_continuous_scale=px.colors.sequential.Viridis,\n    text_auto=True\n)\nfig_species.update_xaxes(tickangle=45)\nfig_species.show(renderer='iframe')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:39:45.990466Z","iopub.execute_input":"2025-10-31T17:39:45.990790Z","iopub.status.idle":"2025-10-31T17:39:46.077448Z","shell.execute_reply.started":"2025-10-31T17:39:45.990769Z","shell.execute_reply":"2025-10-31T17:39:46.076676Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Temporal Analysis: Exploring Sampling Dates","metadata":{}},{"cell_type":"code","source":"# --- Convert Sampling_Date to datetime objects ---\ntrain_wide_df['Sampling_Date'] = pd.to_datetime(train_wide_df['Sampling_Date'])\n\n# --- Extract Year and Month ---\ntrain_wide_df['Year'] = train_wide_df['Sampling_Date'].dt.year\ntrain_wide_df['Month'] = train_wide_df['Sampling_Date'].dt.month\n\n# --- Plot Distribution by Year ---\nfig_year = px.histogram(\n    train_wide_df,\n    x='Year',\n    title='Distribution of Samples by Year',\n    labels={'Year': 'Year'},\n    text_auto=True\n)\nfig_year.show(renderer='iframe')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:40:19.192131Z","iopub.execute_input":"2025-10-31T17:40:19.192446Z","iopub.status.idle":"2025-10-31T17:40:19.260869Z","shell.execute_reply.started":"2025-10-31T17:40:19.192423Z","shell.execute_reply":"2025-10-31T17:40:19.260049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Plot Distribution by Month ---\nfig_month = px.histogram(\n    train_wide_df,\n    x='Month',\n    title='Distribution of Samples by Month',\n    labels={'Month': 'Month'},\n    text_auto=True\n)\nfig_month.update_xaxes(dtick=1)\nfig_month.show(renderer='iframe')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:40:26.059016Z","iopub.execute_input":"2025-10-31T17:40:26.059566Z","iopub.status.idle":"2025-10-31T17:40:26.135135Z","shell.execute_reply.started":"2025-10-31T17:40:26.059539Z","shell.execute_reply":"2025-10-31T17:40:26.134344Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visual Exploration of Image Data","metadata":{}},{"cell_type":"code","source":"# --- Function to Display Images with Metadata ---\ndef display_sample_images(df, n_samples=5, random_seed=42):\n    \"\"\"\n    Displays a random selection of images along with their metadata and biomass targets.\n    \"\"\"\n    samples = df.sample(n_samples, random_state=random_seed)\n    \n    plt.figure(figsize=(20, 5 * n_samples))\n    \n    for i, (idx, row) in enumerate(samples.iterrows()):\n        img_path = os.path.join(TRAIN_IMG_DIR, os.path.basename(row['image_path']))\n        \n        try:\n            img = Image.open(img_path)\n        except FileNotFoundError:\n            print(f\"Image not found at {img_path}. Skipping.\")\n            continue\n            \n        plt.subplot(n_samples, 1, i + 1)\n        plt.imshow(img)\n        plt.axis('off')\n        \n        # --- Create Title with Metadata and Targets ---\n        title = (\n            f\"Image ID: {row['image_id']} | State: {row['State']} | Species: {row['Species']}\\n\"\n            f\"NDVI: {row['Pre_GSHH_NDVI']:.2f} | Height: {row['Height_Ave_cm']:.2f} cm\\n\"\n            f\"--------------------------------------------------------------------------------\\n\"\n            f\"Dry_Total_g: {row['Dry_Total_g']:.2f} | GDM_g: {row['GDM_g']:.2f} | Dry_Green_g: {row['Dry_Green_g']:.2f} | \"\n            f\"Dry_Dead_g: {row['Dry_Dead_g']:.2f} | Dry_Clover_g: {row['Dry_Clover_g']:.2f}\"\n        )\n        plt.title(title, loc='left', fontsize=12, family='monospace')\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:14:10.076383Z","iopub.execute_input":"2025-10-31T17:14:10.077284Z","iopub.status.idle":"2025-10-31T17:14:10.085508Z","shell.execute_reply.started":"2025-10-31T17:14:10.077256Z","shell.execute_reply":"2025-10-31T17:14:10.084522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display_sample_images(train_wide_df, n_samples=5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:14:19.679661Z","iopub.execute_input":"2025-10-31T17:14:19.680008Z","iopub.status.idle":"2025-10-31T17:14:23.342123Z","shell.execute_reply.started":"2025-10-31T17:14:19.679986Z","shell.execute_reply":"2025-10-31T17:14:23.340518Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Analyzing Image Dimensions","metadata":{}},{"cell_type":"code","source":"# --- Get Image Dimensions ---\nimage_dims = []\nfor image_file in os.listdir(TRAIN_IMG_DIR):\n    with Image.open(os.path.join(TRAIN_IMG_DIR, image_file)) as img:\n        image_dims.append(img.size)\n\n# --- Create a DataFrame of Dimensions ---\ndims_df = pd.DataFrame(image_dims, columns=['Width', 'Height'])\ndims_counts = dims_df.value_counts().reset_index(name='count')\n\nprint(\"=\"*50)\nprint(\"--- Image Dimension Analysis ---\")\nprint(\"=\"*50)\nprint(f\"Total images analyzed: {len(dims_df)}\")\nprint(f\"Number of unique dimensions found: {len(dims_counts)}\")\n\nif len(dims_counts) == 1:\n    print(f\"\\nAll images have the same dimensions: {dims_counts.iloc[0]['Width']}x{dims_counts.iloc[0]['Height']}\")\nelse:\n    print(\"\\nMultiple image dimensions found:\")\n    display(dims_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-31T17:20:30.235705Z","iopub.execute_input":"2025-10-31T17:20:30.236160Z","iopub.status.idle":"2025-10-31T17:20:34.789192Z","shell.execute_reply.started":"2025-10-31T17:20:30.236133Z","shell.execute_reply":"2025-10-31T17:20:34.788281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\nThis exploratory data analysis has provided a deep understanding of the CSIRO pasture biomass dataset, laying a strong foundation for the modeling phases.\n\n### 1. Data Structure and Integrity\n*   **Complete Data**: The training dataset contains 357 unique images, each with 5 corresponding biomass target values. Crucially, there are **no missing values**, which simplifies preprocessing.\n*   **Wide vs. Long Format**: We successfully reshaped the data from its initial 'long' format to a 'wide' format, which proved essential for effective correlation and feature analysis.\n\n### 2. Key Feature Insights\n*   **`Height_Ave_cm` is a Powerhouse**: Pasture height shows a strong positive correlation with green biomass (`Dry_Green_g`: 0.65) and total green matter (`GDM_g`: 0.58). This is likely one of the most predictive features available.\n*   **`Pre_GSHH_NDVI` is a Reliable Indicator**: As expected, the NDVI (a measure of greenness) is moderately correlated with `GDM_g` (0.47) and `Dry_Green_g` (0.35). It's a valuable input for any model.\n*   **Biomass Inter-correlation**: The target variables themselves are highly correlated. `Dry_Total_g` is strongly predicted by `GDM_g` (0.90) and `Dry_Green_g` (0.83). This suggests that models that accurately predict green matter will likely do well in predicting total biomass, which has the highest weight (0.5) in the evaluation metric.\n\n### 3. Data Distribution and Bias\n*   **Geographic Skew**: The data is predominantly from Tasmania (138 samples) and Victoria (112 samples). Any model will inherently be biased towards the pasture types and conditions found in these states.\n*   **Species Imbalance**: The dataset is dominated by `Ryegrass_Clover` and `Ryegrass`. There is a long tail of rare species, which may be challenging to model accurately.\n*   **Temporal Concentration**: All samples were collected in **2015**, primarily during the autumn and spring growing seasons (May, June, September). The model will not have seen data from other years or seasons.\n\n### 4. Image Characteristics\n*   **Consistent Dimensions**: All training images share the same dimensions: **2000x1000 pixels**. This is a significant advantage, as it standardizes the input for computer vision models and simplifies the preprocessing pipeline.\n*   **Visual-Data Link**: The sample images clearly show a visual correspondence between the pasture's appearance (color, density, texture) and its biomass composition. Lush, green fields correspond to high `Dry_Green_g`, while patchy, brown fields have high `Dry_Dead_g`. This confirms that a computer vision approach is highly viable.\n\n---","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}