{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Outline\n\nThis notebook is to go through basic info about data include image and excel.\n\n1. Load and check data\n2. Visualize data and analyze\n3. Conclusion","metadata":{}},{"cell_type":"markdown","source":"# Load and check data","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import tqdm\nimport torch\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T15:08:48.049057Z","iopub.execute_input":"2025-12-30T15:08:48.049402Z","iopub.status.idle":"2025-12-30T15:08:48.744957Z","shell.execute_reply.started":"2025-12-30T15:08:48.049379Z","shell.execute_reply":"2025-12-30T15:08:48.744161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# conifg data for further use\nclass CFG:\n    train_csv = \"/kaggle/input/csiro-biomass/train.csv\"\n    test_csv  = \"/kaggle/input/csiro-biomass/test.csv\"\n\n    image_root = \"/kaggle/input/csiro-biomass\"\n\n    batch_size = 4\n    seed = 42\n    epochs = 10\n\n    target_cols = [\n        \"Dry_Green_g\",\n        \"Dry_Dead_g\",\n        \"Dry_Clover_g\",\n        \"GDM_g\",\n        \"Dry_Total_g\"\n    ]\n\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.466257Z","iopub.execute_input":"2025-12-30T14:53:43.466839Z","iopub.status.idle":"2025-12-30T14:53:43.483490Z","shell.execute_reply.started":"2025-12-30T14:53:43.466808Z","shell.execute_reply":"2025-12-30T14:53:43.482623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set seed for reproducibility\ndef set_seed(seed):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nset_seed(CFG.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.484382Z","iopub.execute_input":"2025-12-30T14:53:43.484701Z","iopub.status.idle":"2025-12-30T14:53:43.501374Z","shell.execute_reply.started":"2025-12-30T14:53:43.484665Z","shell.execute_reply":"2025-12-30T14:53:43.500404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_in = pd.read_csv(CFG.train_csv)\ndf_out = pd.read_csv(CFG.test_csv)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.503190Z","iopub.execute_input":"2025-12-30T14:53:43.503529Z","iopub.status.idle":"2025-12-30T14:53:43.525011Z","shell.execute_reply.started":"2025-12-30T14:53:43.503507Z","shell.execute_reply":"2025-12-30T14:53:43.524161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_in.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.525914Z","iopub.execute_input":"2025-12-30T14:53:43.526935Z","iopub.status.idle":"2025-12-30T14:53:43.540756Z","shell.execute_reply.started":"2025-12-30T14:53:43.526910Z","shell.execute_reply":"2025-12-30T14:53:43.540116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.542406Z","iopub.execute_input":"2025-12-30T14:53:43.542816Z","iopub.status.idle":"2025-12-30T14:53:43.558012Z","shell.execute_reply.started":"2025-12-30T14:53:43.542794Z","shell.execute_reply":"2025-12-30T14:53:43.557338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_in.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.558867Z","iopub.execute_input":"2025-12-30T14:53:43.559169Z","iopub.status.idle":"2025-12-30T14:53:43.577497Z","shell.execute_reply.started":"2025-12-30T14:53:43.559143Z","shell.execute_reply":"2025-12-30T14:53:43.576778Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The data above shows that each image have 5 target to predict and there is no missing data in it.\n\nAnd train data have sample_id, image_path, Sampling_Date, State, Species, Pre_GSHH_NDVI, Height_Ave_cm, target_name, target information in it.\n\n1. Dry_Clover_g\n2. Dry_Dead_g\n3. Dry_Green_g\n4. Dry_Total_g\n5. GDM_g","metadata":{}},{"cell_type":"markdown","source":"# Visualize data","metadata":{}},{"cell_type":"code","source":"# Pivot the table so each target_name becomes a column\ndf_pivot = df_in.pivot_table(\n    index=[\"image_path\", \"Sampling_Date\", \"State\", \"Species\"],\n    columns=\"target_name\",\n    values=\"target\"\n).reset_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:59:08.099890Z","iopub.execute_input":"2025-12-30T14:59:08.100224Z","iopub.status.idle":"2025-12-30T14:59:08.114945Z","shell.execute_reply.started":"2025-12-30T14:59:08.100200Z","shell.execute_reply":"2025-12-30T14:59:08.113986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_pivot.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:59:08.361820Z","iopub.execute_input":"2025-12-30T14:59:08.362171Z","iopub.status.idle":"2025-12-30T14:59:08.375438Z","shell.execute_reply.started":"2025-12-30T14:59:08.362143Z","shell.execute_reply":"2025-12-30T14:59:08.374381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_cols = df_pivot.columns[df_pivot.dtypes.apply(lambda x: np.issubdtype(x, np.number))].tolist()\nnumeric_cols = [c for c in numeric_cols if c not in ['target_name', 'image_path','State','Species']]\n\nfig, axs = plt.subplots(1, 5, figsize=(15, 5)) # Adjust grid size as needed\naxs = axs.flatten() # Flattens the 2D grid to 1D\n\nfor i, col in enumerate(numeric_cols):\n    if i < len(axs):\n        axs[i].hist(df_pivot[col], bins=30, color='skyblue', edgecolor='black')\n        axs[i].set_title(col)\n    else:\n        axs[i].axis('off') # Hide unused subplots\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T15:05:43.028316Z","iopub.execute_input":"2025-12-30T15:05:43.028651Z","iopub.status.idle":"2025-12-30T15:05:43.909865Z","shell.execute_reply.started":"2025-12-30T15:05:43.028615Z","shell.execute_reply":"2025-12-30T15:05:43.909101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_pivot[\"State\"].unique())\nprint(df_pivot[\"Species\"].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T15:09:50.631423Z","iopub.execute_input":"2025-12-30T15:09:50.631754Z","iopub.status.idle":"2025-12-30T15:09:50.637766Z","shell.execute_reply.started":"2025-12-30T15:09:50.631728Z","shell.execute_reply":"2025-12-30T15:09:50.636874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# we use \"Label Encodeing\" because its simple to set up\n# State, Species\n\nencoder = LabelEncoder()\nState_encoded = encoder.fit_transform(df_pivot[\"State\"])\nSpecies_encoded = encoder.fit_transform(df_pivot[\"Species\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T15:09:02.339931Z","iopub.execute_input":"2025-12-30T15:09:02.340880Z","iopub.status.idle":"2025-12-30T15:09:02.345673Z","shell.execute_reply.started":"2025-12-30T15:09:02.340851Z","shell.execute_reply":"2025-12-30T15:09:02.344873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create subplots\nfig, axs = plt.subplots(1, 2, figsize=(10, 4))  # 1 row, 2 columns\naxs[0].hist(State_encoded, color='orange', edgecolor='black')\naxs[0].set_title('State')\naxs[1].hist(Species_encoded, color='orange', edgecolor='black')\naxs[1].set_title('Species')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T15:09:03.328939Z","iopub.execute_input":"2025-12-30T15:09:03.329828Z","iopub.status.idle":"2025-12-30T15:09:03.725788Z","shell.execute_reply.started":"2025-12-30T15:09:03.329798Z","shell.execute_reply":"2025-12-30T15:09:03.725095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to read a JPG file\ndef load_jpg(path):\n    try:\n        with Image.open(path) as img:\n            return path, img.copy()  # return a copy of the image, not just its size\n    except Exception as e:\n        return path, f\"Error: {e}\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.578310Z","iopub.execute_input":"2025-12-30T14:53:43.578557Z","iopub.status.idle":"2025-12-30T14:53:43.592912Z","shell.execute_reply.started":"2025-12-30T14:53:43.578533Z","shell.execute_reply":"2025-12-30T14:53:43.592077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Collect all jpg files in a directory\njpg_files = [os.path.join(CFG.image_root+\"/train\", f) for f in os.listdir(CFG.image_root+\"/train\") if f.endswith(\".jpg\")]\n\n# Using ThreadPoolExecutor with tqdm\nresults = []\nwith ThreadPoolExecutor(max_workers=4) as executor:\n    futures = {executor.submit(load_jpg, path): path for path in jpg_files[:9]}\n    for future in tqdm(as_completed(futures), total=len(futures), desc=\"Reading JPGs\"):\n        results.append(future.result())\n\n# Print results\nfor path, img in results:\n    print(f\"{path}: {img}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.593741Z","iopub.execute_input":"2025-12-30T14:53:43.593951Z","iopub.status.idle":"2025-12-30T14:53:43.810163Z","shell.execute_reply.started":"2025-12-30T14:53:43.593930Z","shell.execute_reply":"2025-12-30T14:53:43.809283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Filter out successful loads\nimages = [(path, img) for path, img in results if not isinstance(img, str)]\nimages = images[:9]\n# Display images in a grid ---\ncols = 3\nrows = 3\nfig, axes = plt.subplots(rows, cols, figsize=(12, 4 * rows))\naxes = axes.flatten()\n\nfor i, (path, img) in enumerate(images):\n    axes[i].imshow(img)\n    axes[i].set_title(os.path.basename(path))\n    axes[i].axis(\"off\")\n\n# Hide unused subplots\nfor j in range(i + 1, len(axes)):\n    axes[j].axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T14:53:43.812180Z","iopub.execute_input":"2025-12-30T14:53:43.812414Z","iopub.status.idle":"2025-12-30T14:53:46.831058Z","shell.execute_reply.started":"2025-12-30T14:53:43.812396Z","shell.execute_reply":"2025-12-30T14:53:46.830050Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Conclusion","metadata":{}},{"cell_type":"markdown","source":"1. From the training data we can find out that Dry_Clover_g, Dry_Dead_g,Dry_green_g is mostly range around 0. **Therefore overfitting might cause predition value around 0.**\n2. It seems like GDM_g = Dry_Green_g + Dry_Clover_g and Dry_Total_g = GDM_g + Dry_Dead_g. We need to take this into consider.**Because we can simply sum of the prediction of previous value(Dry_Clover_g, Dry_Dead_g,Dry_green_g) as our prediction rather then seperate predict it.**\n4. State and species seems have a huge difference in their total number. This might cause the model to overfitting on some state and species. **Therefore we might need to consider this as a input.**\n5. From the 9 images above we can tell that each picture have a different lighting and image size=2000x1000. **So if we need to use this as input we can do some basic image augmentation and resize the image. Because DINO_V2 model have a specific image input size.**","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}