{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CSIRO Image2Biomass Prediction with Lightning ⚡\n\nCompetition: https://www.kaggle.com/competitions/csiro-biomass\n\nAuthor: Based on https://github.com/Borda/kaggle_image-classify","metadata":{}},{"cell_type":"code","source":"!pip install -q pytorch-lightning torchmetrics timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-18T12:31:32.039189Z","iopub.execute_input":"2025-11-18T12:31:32.039425Z","iopub.status.idle":"2025-11-18T12:48:12.476085Z","shell.execute_reply.started":"2025-11-18T12:31:32.039400Z","shell.execute_reply":"2025-11-18T12:48:12.475254Z"},"_kg_hide-output":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport random\nfrom typing import Optional\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport timm\nfrom timm.data import resolve_data_config\nfrom timm.data.transforms_factory import create_transform\n\nimport pytorch_lightning as pl\nimport torchmetrics\nimport warnings\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nprint(f\"PyTorch: {torch.__version__}\")\nprint(f\"Lightning: {pl.__version__}\")\nprint(f\"TIMM: {timm.__version__}\")\nprint(f\"Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}\")\n\npl.seed_everything(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T15:19:14.770371Z","iopub.execute_input":"2025-11-18T15:19:14.770967Z","iopub.status.idle":"2025-11-18T15:19:30.730231Z","shell.execute_reply.started":"2025-11-18T15:19:14.770941Z","shell.execute_reply":"2025-11-18T15:19:30.729613Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Explore Data","metadata":{}},{"cell_type":"code","source":"PATH_DATA = '/kaggle/input/csiro-biomass'\nPATH_TRAIN_CSV = os.path.join(PATH_DATA, 'train.csv')\nPATH_TRAIN_IMG = os.path.join(PATH_DATA, 'train')\nPATH_TEST_IMG = os.path.join(PATH_DATA, 'test')\n\ndf = pd.read_csv(PATH_TRAIN_CSV)\nprint(f\"Dataset size: {df.shape}\")\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T15:19:36.009083Z","iopub.execute_input":"2025-11-18T15:19:36.009358Z","iopub.status.idle":"2025-11-18T15:19:36.046930Z","shell.execute_reply.started":"2025-11-18T15:19:36.009338Z","shell.execute_reply":"2025-11-18T15:19:36.046176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TARGET_COLS = [c for c in df.columns if c not in ['image_id', 'Image']]\nprint(f\"Target columns: {TARGET_COLS}\")\nprint(f\"Number of targets: {len(TARGET_COLS)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T15:19:39.109246Z","iopub.execute_input":"2025-11-18T15:19:39.109885Z","iopub.status.idle":"2025-11-18T15:19:39.114649Z","shell.execute_reply.started":"2025-11-18T15:19:39.109856Z","shell.execute_reply":"2025-11-18T15:19:39.113885Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plot target distribution","metadata":{}},{"cell_type":"code","source":"# Exclude non-numeric or identifier columns from histogram plotting\ncols_to_plot = [col for col in TARGET_COLS if col not in ['sample_id', 'image_path', 'State', 'target_name']]\n\nfor col in cols_to_plot:\n    plt.figure(figsize=(8, 3)) # Create a new figure for each histogram\n    plt.hist(df[col].dropna(), bins=50, edgecolor='black', alpha=0.7)\n    plt.xlabel(col, fontsize=12)\n    plt.ylabel('Count', fontsize=12)\n    plt.title(f'{col} Distribution', fontsize=14, fontweight='bold')\n    plt.grid(alpha=0.3)\n    plt.xticks(rotation=45, ha=\"right\") # Rotate x-axis labels\n    plt.tight_layout() # Adjust layout to prevent overlap\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T15:19:42.179989Z","iopub.execute_input":"2025-11-18T15:19:42.180593Z","iopub.status.idle":"2025-11-18T15:19:43.476840Z","shell.execute_reply.started":"2025-11-18T15:19:42.180518Z","shell.execute_reply":"2025-11-18T15:19:43.475667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_to_plot = ['State', 'target_name']\nn_rows, n_cols = 1, len(cols_to_plot)\nfig, axes = plt.subplots(n_rows, n_cols, figsize=(4 * n_cols, 4 * n_rows))\n\n# Ensure axes is an array even for a single subplot\naxes = axes.flatten()\n\nfor ax, col in zip(axes, cols_to_plot):\n    counts = df[col].value_counts()\n    ax.pie(counts, labels=counts.index, autopct='%1.1f%%', startangle=140)\n    ax.set_title(f'Distribution of {col}', fontsize=14, fontweight='bold')\n    ax.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T12:51:11.864064Z","iopub.execute_input":"2025-11-18T12:51:11.864943Z","iopub.status.idle":"2025-11-18T12:51:12.048792Z","shell.execute_reply.started":"2025-11-18T12:51:11.864908Z","shell.execute_reply":"2025-11-18T12:51:12.048033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert 'Sampling_Date' to datetime objects\ndf['Sampling_Date'] = pd.to_datetime(df['Sampling_Date'])\n\n# Extract the day of the year\ndf['Day_of_Year'] = df['Sampling_Date'].dt.dayofyear\n\n# Calculate the correlation between 'target' and 'Day_of_Year'\ncorrelation = df['target'].corr(df['Day_of_Year'])\n\nprint(f\"The correlation between 'target' and 'Day_of_Year' is: {correlation}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T12:51:24.938358Z","iopub.execute_input":"2025-11-18T12:51:24.938851Z","iopub.status.idle":"2025-11-18T12:51:24.955573Z","shell.execute_reply.started":"2025-11-18T12:51:24.938829Z","shell.execute_reply":"2025-11-18T12:51:24.954941Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Show sample images","metadata":{}},{"cell_type":"code","source":"def show_images(df_sample, n=12, path_img=PATH_DATA):\n    \"\"\"Displays a linear sampling of images sorted by target value.\"\"\"\n\n    # Sort the DataFrame by the 'target' column\n    df_sorted = df_sample.sort_values(by='target').reset_index(drop=True)\n\n    # Perform linear sampling\n    indices_to_show = np.linspace(0, len(df_sorted) - 1, n, dtype=int)\n    df_to_show = df_sorted.iloc[indices_to_show]\n\n    # Determine the number of rows and columns for subplots\n    n_cols = 3  # You can adjust this number\n    n_rows = (n + n_cols - 1) // n_cols\n\n    fig, axes = plt.subplots(n_rows, n_cols, figsize=(16, 4 * n_rows))\n    axes = axes.flatten()\n\n    # Remove unused subplots if any\n    for i in range(n, len(axes)):\n        fig.delaxes(axes[i])\n\n    for i, (idx, row) in enumerate(df_to_show.iterrows()):\n        # Use image_path directly (includes train/ID....jpg)\n        img_path = os.path.join(path_img, row['image_path'])\n\n        if os.path.exists(img_path):\n            img = Image.open(img_path).convert('RGB')\n            axes[i].imshow(img)\n            # Include the target value in the title\n            title = f\"ID: {row['sample_id']}\\nTarget: {row['target']:.2f}\"\n            axes[i].set_title(title, fontsize=10)\n        axes[i].axis('off')\n\n    plt.tight_layout()\n    plt.show()\n\n# Example usage: Show 12 images linearly sampled based on target value\nshow_images(df, n=12)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T15:20:52.904663Z","iopub.execute_input":"2025-11-18T15:20:52.905215Z","iopub.status.idle":"2025-11-18T15:20:57.429105Z","shell.execute_reply.started":"2025-11-18T15:20:52.905191Z","shell.execute_reply":"2025-11-18T15:20:57.427800Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset & DataModule","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n\n# --- Bước 1: Chuyển đổi dữ liệu để tính Correlation ---\n# Hiện tại 1 ảnh có nhiều dòng (mỗi dòng 1 loại target).\n# Chúng ta cần pivot để mỗi ảnh là 1 dòng, các loại target là các cột.\n\n# Lấy các cột feature cơ bản (giả sử các feature này giống nhau cho cùng 1 sample_id)\nfeature_cols = ['sample_id', 'Pre_GSHH_NDVI', 'Height_Ave_cm', 'Day_of_Year']\ndf_features = df[feature_cols].drop_duplicates(subset=['sample_id'])\n\n# Pivot cột target\ndf_pivot_targets = df.pivot(index='sample_id', columns='target_name', values='target')\n\n# Gộp lại thành 1 dataframe hoàn chỉnh cho phân tích\ndf_analysis = pd.merge(df_features, df_pivot_targets, on='sample_id')\n\n# Tính ma trận tương quan\ncorr_matrix = df_analysis.drop(columns=['sample_id']).corr()\n\n# Vẽ Heatmap\nplt.figure(figsize=(12, 10))\nsns.heatmap(corr_matrix, annot=True, fmt=\".2f\", cmap='coolwarm', center=0)\nplt.title('Correlation Matrix: Features vs Biomass Targets', fontsize=16, fontweight='bold')\nplt.show()\n# --- ĐOẠN CODE ĐÃ SỬA LỖI ---\n\n# In ra top tương quan với từng loại bạn quan tâm\ntarget_interests = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g']\nprint(\"\\n--- Top Correlations ---\")\n\n# Tính lại correlation matrix một lần duy nhất với chế độ chỉ lấy số (để tránh lỗi string)\nsafe_corr_matrix = df_analysis.corr(numeric_only=True)\n\nfor t in target_interests:\n    if t in safe_corr_matrix.columns:\n        print(f\"\\nCorrelations with {t}:\")\n        # Lấy dữ liệu từ matrix đã tính sẵn, không tính lại từ df_analysis nữa\n        top_corr = safe_corr_matrix[t].sort_values(ascending=False)\n        print(top_corr[1:6]) # Bỏ qua chính nó (vị trí 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T12:57:43.206510Z","iopub.execute_input":"2025-11-18T12:57:43.207112Z","iopub.status.idle":"2025-11-18T12:57:43.534860Z","shell.execute_reply.started":"2025-11-18T12:57:43.207090Z","shell.execute_reply":"2025-11-18T12:57:43.534017Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom PIL import Image\n\ndef show_top_images(df_full, target_name, n=5, path_img=PATH_DATA):\n    \"\"\"\n    Hiển thị n ảnh có giá trị target cao nhất cho một loại target_name cụ thể.\n    \"\"\"\n    print(f\"\\n>>> Hiển thị Top {n} ảnh có lượng '{target_name}' CAO NHẤT <<<\")\n    \n    # 1. Lọc rows chỉ chứa loại target cần tìm (ví dụ chỉ lấy dòng Dry_Green_g)\n    df_subset = df_full[df_full['target_name'] == target_name]\n    \n    if df_subset.empty:\n        print(f\"Không tìm thấy dữ liệu cho {target_name}\")\n        return\n\n    # 2. Sắp xếp giảm dần theo giá trị target\n    df_sorted = df_subset.sort_values(by='target', ascending=False).head(n)\n    \n    # 3. Vẽ hình\n    fig, axes = plt.subplots(1, n, figsize=(4 * n, 5))\n    if n == 1: axes = [axes] # Xử lý lỗi nếu chỉ vẽ 1 ảnh\n    \n    for i, (idx, row) in enumerate(df_sorted.iterrows()):\n        # Xử lý đường dẫn ảnh\n        img_path = os.path.join(path_img, row['image_path'])\n        \n        ax = axes[i]\n        if os.path.exists(img_path):\n            img = Image.open(img_path).convert('RGB')\n            ax.imshow(img)\n            \n            # Tạo tiêu đề chi tiết\n            ndvi_val = row['Pre_GSHH_NDVI'] if 'Pre_GSHH_NDVI' in row else 'N/A'\n            title = (f\"ID: {row['sample_id']}\\n\"\n                     f\"Val: {row['target']:.2f} g\\n\"\n                     f\"NDVI: {ndvi_val}\")\n            ax.set_title(title, fontsize=11, fontweight='bold', color='darkgreen')\n        else:\n            ax.text(0.5, 0.5, \"Img Not Found\", ha='center')\n            \n        ax.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\n# --- Gọi hàm hiển thị ---\n\n# 1. Xem ảnh nhiều cỏ xanh nhất (Dry_Green_g) -> Thường ảnh sẽ rất xanh, NDVI cao\nshow_top_images(df, 'Dry_Green_g', n=5)\n\n# 2. Xem ảnh nhiều cỏ chết/khô nhất (Dry_Dead_g) -> Thường ảnh màu vàng/nâu\nshow_top_images(df, 'Dry_Dead_g', n=5)\n\n# 3. Xem ảnh nhiều cỏ ba lá nhất (Dry_Clover_g) -> Xem hình dạng lá có khác biệt không\nshow_top_images(df, 'Dry_Clover_g', n=5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T13:08:38.851033Z","iopub.execute_input":"2025-11-18T13:08:38.851361Z","iopub.status.idle":"2025-11-18T13:08:44.306803Z","shell.execute_reply.started":"2025-11-18T13:08:38.851340Z","shell.execute_reply":"2025-11-18T13:08:44.305993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_feature_extremes(df_full, feature_col, n=5, ascending=False, path_img=PATH_DATA):\n    \"\"\"\n    Hiển thị ảnh dựa trên việc sort theo feature (Height, NDVI).\n    Đã sửa lỗi lặp ID: Đảm bảo mỗi ảnh chỉ hiện 1 lần.\n    \"\"\"\n    # Xác định chế độ hiển thị\n    mode = \"THẤP NHẤT (Low)\" if ascending else \"CAO NHẤT (High)\"\n    print(f\"\\n>>> Top {n} ảnh có '{feature_col}' {mode} <<<\")\n\n    # --- BƯỚC QUAN TRỌNG: XỬ LÝ LẶP ID ---\n    # Cách 1: Chỉ lấy các dòng là 'Dry_Green_g' để đại diện cho ảnh đó (giúp tiêu đề hiển thị khối lượng cỏ xanh)\n    df_unique = df_full[df_full['target_name'] == 'Dry_Green_g'].copy()\n    \n    # Fallback: Nếu lọc ra rỗng (do data không có Dry_Green_g), thì dùng drop_duplicates thô\n    if df_unique.empty:\n        df_unique = df_full.drop_duplicates(subset=['sample_id'])\n\n    # Loại bỏ các giá trị NaN ở cột feature đang xét\n    df_unique = df_unique.dropna(subset=[feature_col])\n    \n    # 2. Sắp xếp dữ liệu đã clean\n    df_sorted = df_unique.sort_values(by=feature_col, ascending=ascending).head(n)\n    \n    # 3. Vẽ hình\n    fig, axes = plt.subplots(1, n, figsize=(4 * n, 5))\n    if n == 1: axes = [axes] # Xử lý trường hợp n=1\n    \n    for i, (idx, row) in enumerate(df_sorted.iterrows()):\n        img_path = os.path.join(path_img, row['image_path'])\n        ax = axes[i]\n        \n        if os.path.exists(img_path):\n            img = Image.open(img_path).convert('RGB')\n            ax.imshow(img)\n            \n            # Lấy thông tin hiển thị\n            val = row[feature_col]\n            target_val = row['target'] # Đây sẽ là Dry_Green_g (nếu có)\n            \n            title = (f\"ID: {row['sample_id']}\\n\"\n                     f\"{feature_col}: {val:.2f}\\n\"\n                     f\"Green Mass: {target_val:.2f}g\\n\"\n                     f\"Species: {row['Species']}\")\n            \n            # Tô màu tiêu đề: Xanh nếu High, Đỏ nâu nếu Low\n            color = 'darkgreen' if not ascending else 'brown'\n            ax.set_title(title, fontsize=11, fontweight='bold', color=color)\n        else:\n            ax.text(0.5, 0.5, \"Img Not Found\", ha='center')\n            ax.set_title(f\"ID: {row['sample_id']}\")\n        \n        ax.axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\n# --- CHẠY THỬ NGHIỆM ---\n\n# 1. KIỂM TRA CHIỀU CAO (HEIGHT)\n# Kỳ vọng: Cây cao sẽ có Green Mass lớn\nshow_feature_extremes(df, 'Height_Ave_cm', n=5, ascending=False)\n\n# Kỳ vọng: Cây thấp sát đất\nshow_feature_extremes(df, 'Height_Ave_cm', n=5, ascending=True)\n\n# 2. KIỂM TRA ĐỘ XANH (NDVI)\n# Kỳ vọng: Cây rất xanh, lá tốt\nshow_feature_extremes(df, 'Pre_GSHH_NDVI', n=5, ascending=False)\n\n# Kỳ vọng: Đất trọc hoặc cỏ khô vàng\nshow_feature_extremes(df, 'Pre_GSHH_NDVI', n=5, ascending=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-18T15:35:19.406090Z","iopub.execute_input":"2025-11-18T15:35:19.406826Z","iopub.status.idle":"2025-11-18T15:35:26.595274Z","shell.execute_reply.started":"2025-11-18T15:35:19.406802Z","shell.execute_reply":"2025-11-18T15:35:26.594531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nclass BiomassDataset(Dataset):\n    \"\"\"Simple dataset for biomass regression with metafeatures.\"\"\"\n\n    def __init__(self, df, path_img, transforms=None, mode='train', species_classes=None, target_name_classes=None):\n        self.df = df.reset_index(drop=True)\n        self.path_img = path_img\n        self.transforms = transforms\n        self.mode = mode\n        self.target_col = 'target' if mode == 'train' else None\n        self._len = len(self.df) * 2\n\n        # Initialize LabelEncoders and fit\n        self.target_name_encoder = LabelEncoder()\n        # Use provided classes or create from DataFrame and Expose the classes as properties\n        self.species_classes = species_classes or self.df['Species'].unique()\n        self.target_name_classes = target_name_classes or self.df['target_name'].unique()\n        # Prepare encoding\n        self.target_name_encoder.fit(self.target_name_classes)\n        self.df['target_name_encoded'] = self.target_name_encoder.transform(self.df['target_name'])\n\n        # Encode categorical features within the dataset\n        if self.mode != 'test':\n            # Initialize LabelEncoders and fit\n            self.species_encoder = LabelEncoder()\n            self.species_encoder.fit(self.species_classes)\n            self.df['Species_encoded'] = self.species_encoder.transform(self.df['Species'])\n\n    def __len__(self):\n        return self._len\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx // 2]\n        img_relative_path = row['image_path']\n        img_path = os.path.join(self.path_img, img_relative_path)\n        img = Image.open(img_path).convert('RGB')\n\n        half = idx % 2\n        width, height = img.size\n        img_cropped = img.crop((0, 0, width // 2, height)) if half == 0 else img.crop((width // 2, 0, width, height))\n\n        if self.transforms:\n            img_cropped = self.transforms(img_cropped)\n\n        target_name_encoded = torch.tensor(row['target_name_encoded'], dtype=torch.long) # Use long for categorical\n            \n        if self.mode == 'test':\n            return img_cropped, target_name_encoded, img_relative_path\n\n        # Extract and convert metafeatures to tensors (assuming they are already encoded in the dataframe)\n        species_encoded = torch.tensor(row['Species_encoded'], dtype=torch.long) # Use long for categorical\n        ndvi = torch.tensor(row['Pre_GSHH_NDVI'], dtype=torch.float32)\n        height = torch.tensor(row['Height_Ave_cm'], dtype=torch.float32)\n        target = torch.tensor(row[self.target_col], dtype=torch.float32)\n        return img_cropped, target_name_encoded, target, species_encoded, ndvi, height","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T09:09:32.580762Z","iopub.execute_input":"2025-11-04T09:09:32.581151Z","iopub.status.idle":"2025-11-04T09:09:32.593275Z","shell.execute_reply.started":"2025-11-04T09:09:32.581122Z","shell.execute_reply":"2025-11-04T09:09:32.592445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize the dataset (assuming you have a DataFrame 'df' and image path 'PATH_DATA')\ndataset = BiomassDataset(df, PATH_DATA)\n# Get three random indices\nrandom_indices = random.sample(range(len(dataset)), 3)\n\n# Display the random samples\nfig, axes = plt.subplots(1, 3, figsize=(15, 5))\n\nfor i, idx in enumerate(random_indices):\n    img, target_name_encoded, target, species_encoded, ndvi, height = dataset[idx]\n    # Convert the PyTorch tensor image back to PIL Image for displaying\n    # This assumes the default tensor format from PILToTensor or similar\n    if isinstance(img, torch.Tensor):\n        img = img.permute(1, 2, 0).numpy() # Assuming CxHxW format, convert to HxWxD\n\n    axes[i].imshow(img)\n    # Display targets and other data in a multi-line title\n    title = f\"Target: {target:.2f}\\n\"\n    title += f\"Species: {dataset.species_classes[species_encoded]}\\n\"\n    title += f\"NDVI: {ndvi:.2f}\\n\"\n    title += f\"Height: {height:.2f}\\n\"\n    title += f\"Target Name: {dataset.target_name_classes[target_name_encoded]}\"\n\n    axes[i].set_title(title, fontsize=10)\n    axes[i].axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T09:09:32.594543Z","iopub.execute_input":"2025-11-04T09:09:32.594868Z","iopub.status.idle":"2025-11-04T09:09:33.472211Z","shell.execute_reply.started":"2025-11-04T09:09:32.59485Z","shell.execute_reply":"2025-11-04T09:09:33.470955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torchvision import transforms\nfrom pathlib import Path\n\n\nclass BiomassDataModule(pl.LightningDataModule):\n    \"\"\"Simple DataModule for biomass regression with metafeatures.\"\"\"\n\n    def __init__(self, data_path, batch_size=32, img_size=(456, 456), val_split=0.2):\n        super().__init__()\n        self.save_hyperparameters()\n        self.data_path = data_path\n        self.batch_size = batch_size\n        self.img_size = img_size\n        self.val_split = val_split\n        self.train_df: Optional[pd.DataFrame] = None\n        self.val_df: Optional[pd.DataFrame] = None\n        self.test_df: Optional[pd.DataFrame] = None\n        self._color_mean = [0.485, 0.456, 0.406]\n        self._color_std = [0.229, 0.224, 0.225]\n        inimg_size = int(img_size[0] * 1.5)\n        self.transforms = transforms.Compose([\n            transforms.Resize((inimg_size, inimg_size)),\n            transforms.RandomResizedCrop(self.img_size),\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomVerticalFlip(),\n            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n            transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)),\n            transforms.GaussianBlur(kernel_size=3),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=self._color_mean, std=self._color_std),\n        ])\n        self.test_transforms = transforms.Compose([\n            transforms.Resize((inimg_size, inimg_size)),\n            transforms.CenterCrop(self.img_size),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=self._color_mean, std=self._color_std),\n        ])\n        self._num_workers = os.cpu_count() if os.cpu_count() is not None else 0\n\n        # Load train and test data here to fit encoders on combined data\n        self.df = pd.read_csv(os.path.join(self.data_path, 'train.csv'))\n\n        # Get unique classes from the training data for consistent encoding\n        self.species_classes = sorted(self.df['Species'].unique())\n        self.target_name_classes = sorted(self.df['target_name'].unique())\n\n    def setup(self, stage: Optional[str] = None):\n        if stage == 'fit' or stage is None:\n            shuffled_df = self.df.sample(frac=1, random_state=42).reset_index(drop=True)\n            val_size = int(len(shuffled_df) * self.val_split)\n            self.train_df = shuffled_df[:-val_size].copy() # Use .copy() to avoid SettingWithCopyWarning\n            self.val_df = shuffled_df[-val_size:].copy() # Use .copy() to avoid SettingWithCopyWarning\n\n        if stage == 'test' or stage is None:\n            test_image_dir = os.path.join(self.data_path, 'test')\n            assert os.path.isdir(test_image_dir)\n            self.test_df = pd.read_csv(os.path.join(self.data_path, 'test.csv'))\n\n    def train_dataloader(self):\n        train_dataset = BiomassDataset(\n            self.train_df,\n            self.data_path,\n            transforms=self.transforms,\n            mode='train',\n            species_classes=self.species_classes,\n            target_name_classes=self.target_name_classes\n        )\n        return DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=self._num_workers)\n\n    def val_dataloader(self):\n        val_dataset = BiomassDataset(\n            self.val_df,\n            self.data_path,\n            transforms=self.test_transforms,\n            mode='train',\n            species_classes=self.species_classes,\n            target_name_classes=self.target_name_classes\n        )\n        return DataLoader(val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=self._num_workers)\n\n    def test_dataloader(self):\n        if self.test_df is None:\n            self.setup(stage='test')\n\n        test_dataset = BiomassDataset(\n            self.test_df,\n            self.data_path,\n            transforms=self.test_transforms,\n            mode='test',\n            species_classes=self.species_classes,\n            target_name_classes=self.target_name_classes\n        )\n        return DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=0)\n\n    def teardown(self, stage: Optional[str] = None):\n        # Clean up resources if needed\n        pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T09:09:33.473006Z","iopub.execute_input":"2025-11-04T09:09:33.473321Z","iopub.status.idle":"2025-11-04T09:09:33.486666Z","shell.execute_reply.started":"2025-11-04T09:09:33.473303Z","shell.execute_reply":"2025-11-04T09:09:33.485946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example Usage (assuming df, PATH_DATA and TARGET_COLS are defined)\ndata_module = BiomassDataModule(PATH_DATA, batch_size=16, img_size=(448, 448))\ndata_module.setup()\n\n# You can now access the dataloaders\ntrain_loader = data_module.train_dataloader()\nval_loader = data_module.val_dataloader()\n\nprint(f\"Number of training batches: {len(train_loader)}\")\nprint(f\"Number of validation batches: {len(val_loader)}\")\n\n# Example of getting a batch (optional)\ntrain_images, train_targets, train_species_encoded, train_ndvi, train_height, train_target_name_encoded = next(iter(train_loader))\nprint(f\"Shape of training images batch: {train_images.shape}\")\nprint(f\"Shape of training targets batch: {train_targets.shape}\")\nprint(f\"Shape of training species encoded batch: {train_species_encoded.shape}\")\nprint(f\"Shape of training NDVI batch: {train_ndvi.shape}\")\nprint(f\"Shape of training height batch: {train_height.shape}\")\nprint(f\"Shape of training target name encoded batch: {train_target_name_encoded.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T09:09:33.488212Z","iopub.execute_input":"2025-11-04T09:09:33.488532Z","iopub.status.idle":"2025-11-04T09:09:38.30442Z","shell.execute_reply.started":"2025-11-04T09:09:33.488514Z","shell.execute_reply":"2025-11-04T09:09:38.302989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the first batch from the training dataloader\ntrain_images, train_target_name_encoded, train_targets, train_species_encoded, train_ndvi, train_height = next(iter(train_loader))\n\n# Determine how many images to show (e.g., the first 4 from the batch)\nn_images_to_show = min(4, train_images.shape[0])\n\nfig, axes = plt.subplots(1, n_images_to_show, figsize=(4 * n_images_to_show, 5))\n\n# Ensure axes is an array even for a single image\nif n_images_to_show == 1:\n    axes = [axes]\n\n# Assuming data_module is available in the environment to access class names\n# If not, you might need to pass them from the data_module\n# For now, assuming data_module is accessible\nspecies_classes = data_module.species_classes\ntarget_name_classes = data_module.target_name_classes\n\nfor i in range(n_images_to_show):\n    img = train_images[i].permute(1, 2, 0).numpy() # Convert from CxHxW to HxWxD for displaying\n    # Denormalize the image for better visualization (using ImageNet standards)\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    img = std * img + mean\n    img = np.clip(img, 0, 1) # Clip values to be between 0 and 1\n\n    axes[i].imshow(img)\n    # Display targets and other data in a multi-line title\n    title = f\"Target: {train_targets[i].item():.2f} [{train_targets.dtype}]\\n\"\n    title += f\"Species: {train_species_encoded[i]} [{train_species_encoded.dtype}]\\n\"\n    title += f\"NDVI: {train_ndvi[i].item():.2f} [{train_ndvi.dtype}]\\n\"\n    title += f\"Height: {train_height[i].item():.2f} [{train_height.dtype}]\\n\"\n    title += f\"Target Name: {train_target_name_encoded[i]} [{train_target_name_encoded.dtype}]\"\n\n    axes[i].set_title(title, fontsize=10)\n    # axes[i].axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-04T09:09:38.305834Z","iopub.execute_input":"2025-11-04T09:09:38.306197Z","iopub.status.idle":"2025-11-04T09:09:45.267973Z","shell.execute_reply.started":"2025-11-04T09:09:38.30616Z","shell.execute_reply":"2025-11-04T09:09:45.266335Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# LightningModule & training\n\nto select backbones: https://github.com/huggingface/pytorch-image-models/blob/main/results/results-imagenet-a-clean.csv","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingLR\nimport torchmetrics\n\n\nclass SimplifiedBiomassModel(pl.LightningModule):\n    \"\"\"Simplified regression model predicting NDVI, Height, and Target from image.\"\"\"\n\n    def __init__(\n        self,\n        model_name=\"resnet18\",\n        pretrained=True,\n        learning_rate=5e-4,\n        loss_weight_smooth_l1=0.5,\n        loss_weight_ndvi=1.0,  # Add weight for NDVI loss\n        loss_weight_height=1.0, # Add weight for Height loss\n        loss_weight_target=2.0, # Add weight for main target loss (higher by default)\n        T_max=10, # Number of epochs for CosineAnnealingLR\n        num_target_names=5, # Add number of unique target names\n        hidden_dims=[256, 128] # Change hidden_dim to be a list of dimensions\n    ):\n        super().__init__()\n        self.save_hyperparameters()\n        # Ensure hidden_dims is a list even if a single value is passed\n        if not isinstance(hidden_dims, list):\n            hidden_dims = [hidden_dims]\n        self.hparams.hidden_dims = hidden_dims # Save the list in hparams\n\n        self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='avg')\n        in_features = self.backbone.num_features\n\n        # Dynamically build the regression head based on hidden_dims list\n        layers = []\n        current_dim = in_features + num_target_names\n        for h_dim in self.hparams.hidden_dims:\n            layers.append(nn.Linear(current_dim, h_dim))\n            layers.append(nn.ReLU()) # Use ReLU activation\n            current_dim = h_dim\n        # Add the final output layer\n        layers.append(nn.Linear(current_dim, 3)) # Output 3 values: NDVI, Height, Target\n        self.regression_head = nn.Sequential(*layers)\n\n        # Loss functions\n        self.smooth_l1_criterion = nn.SmoothL1Loss()\n        self.mse_criterion = nn.MSELoss()\n\n        # Use MetricCollection for train and validation metrics\n        metrics = torchmetrics.MetricCollection({\n            'ndvi_mae': torchmetrics.MeanAbsoluteError(),\n            'height_mae': torchmetrics.MeanAbsoluteError(),\n            'target_mae': torchmetrics.MeanAbsoluteError(),\n            'target_mse': torchmetrics.MeanSquaredError(),\n        })\n\n        self.train_metrics = metrics.clone(prefix='train_')\n        self.val_metrics = metrics.clone(prefix='val_')\n\n\n    def forward(self, x, target_name_encoded):\n        # Pass the input image through the backbone\n        features = self.backbone(x)\n        # One-hot encode the target_name_encoded\n        target_name_one_hot = F.one_hot(target_name_encoded, num_classes=self.hparams.num_target_names).float()\n        # Concatenate image features and one-hot encoded target name\n        combined_features = torch.cat([features, target_name_one_hot], dim=1)\n        # Pass the combined features through the regression head\n        predictions = self.regression_head(combined_features) # Output is [batch_size, 3]\n        return predictions\n\n    def _compute_loss(self, predictions, targets):\n        \"\"\"Computes the combined loss for NDVI, Height, and Target.\"\"\"\n        # Unpack predictions and targets\n        ndvi_pred, height_pred, target_pred = predictions[:, 0], predictions[:, 1], predictions[:, 2]\n        ndvi_target, height_target, target_target = targets # Targets tuple contains ndvi, height, target\n\n        # Calculate individual losses with weights\n        # Normalize weights to be in the range 0-1 while maintaining relative proportions\n        total_weight = self.hparams.loss_weight_ndvi + self.hparams.loss_weight_height + self.hparams.loss_weight_target\n        normalized_ndvi_weight = self.hparams.loss_weight_ndvi / total_weight\n        normalized_height_weight = self.hparams.loss_weight_height / total_weight\n        normalized_target_weight = self.hparams.loss_weight_target / total_weight\n\n        ndvi_loss = self.mse_criterion(ndvi_pred.squeeze(), ndvi_target.squeeze()) * normalized_ndvi_weight\n        height_loss = self.mse_criterion(height_pred.squeeze(), height_target.squeeze()) * normalized_height_weight\n\n        # Main target loss with weighted SmoothL1 and MSE\n        smooth_l1_loss = self.smooth_l1_criterion(target_pred.squeeze(), target_target.squeeze())\n        mse_loss = self.mse_criterion(target_pred.squeeze(), target_target.squeeze())\n        main_target_loss = (self.hparams.loss_weight_smooth_l1 * smooth_l1_loss + (1 - self.hparams.loss_weight_smooth_l1) * mse_loss) * normalized_target_weight\n\n        # Combine losses\n        total_loss = ndvi_loss + height_loss + main_target_loss\n        return total_loss, ndvi_loss, height_loss, main_target_loss\n\n    def training_step(self, batch, batch_idx):\n        # The batch now contains images, target_name_encoded, target, species, ndvi, height\n        images, target_name_encoded, target, species, ndvi, height = batch\n        predictions = self(images, target_name_encoded) # Forward pass with images and target_name_encoded\n\n        # Pack the relevant targets\n        targets = (ndvi, height, target)\n        total_loss, ndvi_loss, height_loss, main_target_loss = self._compute_loss(predictions, targets)\n\n        # Log losses\n        self.log('train_total_loss', total_loss, on_step=True, prog_bar=True)\n        self.log('train_ndvi_loss', ndvi_loss, on_step=True, prog_bar=False)\n        self.log('train_height_loss', height_loss, on_step=True, prog_bar=False)\n        self.log('train_main_target_loss', main_target_loss, on_step=True, prog_bar=False)\n\n        # Log metrics using MetricCollection\n        metrics_preds = torch.stack([predictions[:, 0], predictions[:, 1], predictions[:, 2]], dim=1)\n        metrics_targets = torch.stack([ndvi, height, target], dim=1)\n\n        # Update the MetricCollection directly\n        self.train_metrics.update(metrics_preds, metrics_targets)\n\n        self.log_dict(self.train_metrics, on_step=True, on_epoch=True, prog_bar=False)\n        return total_loss\n\n    def validation_step(self, batch, batch_idx):\n        # The batch structure from BiomassDataset in 'train' mode is:\n        # img_cropped, target_name_encoded, target, species_encoded, ndvi, height\n        images, target_name_encoded, target, species_target, ndvi_target, height_target = batch\n        predictions = self(images, target_name_encoded) # Forward pass with images and target_name_encoded\n\n        # Pack the relevant targets\n        targets = (ndvi_target, height_target, target)\n        total_loss, ndvi_loss, height_loss, main_target_loss = self._compute_loss(predictions, targets)\n\n        # Log losses\n        self.log('val_total_loss', total_loss, on_step=True, prog_bar=False)\n        self.log('val_ndvi_loss', ndvi_loss, on_step=True, prog_bar=False)\n        self.log('val_height_loss', height_loss, on_step=True, prog_bar=False)\n        self.log('val_main_target_loss', main_target_loss, on_step=True, prog_bar=False)\n\n        # Log metrics using MetricCollection\n        metrics_preds = torch.stack([predictions[:, 0], predictions[:, 1], predictions[:, 2]], dim=1)\n        metrics_targets = torch.stack([ndvi_target, height_target, target], dim=1)\n\n        # Update the MetricCollection directly\n        self.val_metrics.update(metrics_preds, metrics_targets)\n        self.log_dict(self.val_metrics, on_step=True, prog_bar=False)\n\n\n    def predict_step(self, batch, batch_idx):\n        \"\"\"Prediction step for the test set.\"\"\"\n        # The batch structure from BiomassDataset in 'test' mode is:\n        # img_cropped, target_name_encoded, img_relative_path\n        images, target_name_encoded, img_relative_path = batch\n        predictions = self(images, target_name_encoded) # Forward pass with images and target_name_encoded\n        # Unpack predictions\n        ndvi_pred, height_pred, target_pred = predictions[:, 0], predictions[:, 1], predictions[:, 2]\n        # Return all predictions and the original image path\n        return target_pred.squeeze(), img_relative_path\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.AdamW(self.parameters(), lr=self.hparams.learning_rate)\n        scheduler = {\n            'scheduler': CosineAnnealingLR(optimizer, T_max=self.hparams.T_max),\n            'interval': 'epoch',\n            'frequency': 1,\n        }\n        return [optimizer], [scheduler]\n\n# Initialize the simplified model\n# You need to get the number of unique target names from your data_module\n# Assuming data_module is already initialized and setup has been called\nnum_target_names = len(data_module.target_name_classes)\n\nmodel = SimplifiedBiomassModel(\n    model_name=\"efficientnet_b5.sw_in12k_ft_in1k\",\n    pretrained=False,\n    learning_rate=1e-2,\n    num_target_names=num_target_names, # Pass the number of target names\n    hidden_dims=[256, 128, 64], # Example: three hidden layers with dimensions 512, 256, 128\n    T_max=10, # Set T_max to the number of epochs for CosineAnnealingLR\n    loss_weight_ndvi=0.5, # Example weight for NDVI loss\n    loss_weight_height=0.5, # Example weight for Height loss\n    loss_weight_target=2.0 # Example weight for main target loss\n)","metadata":{"trusted":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-11-04T09:09:45.269539Z","iopub.execute_input":"2025-11-04T09:09:45.269894Z","iopub.status.idle":"2025-11-04T09:09:45.773466Z","shell.execute_reply.started":"2025-11-04T09:09:45.269862Z","shell.execute_reply":"2025-11-04T09:09:45.7728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pytorch_lightning.loggers import CSVLogger\n\n# Initialize the CSVLogger\nlogger = CSVLogger(\"logs\", name=\"biomass_regression\")\n\n# Initialize the Trainer\ntrainer = pl.Trainer(\n    max_epochs=75, # You can adjust the number of epochs\n    logger=logger,\n    accelerator='auto', # Use auto to automatically select accelerator (GPU/CPU)\n    devices='auto', # Use auto to automatically select devices\n    precision='16-mixed', # Use Automatic Mixed Precision (AMP)\n    log_every_n_steps=5, # Update progress bar every 5 steps\n    # gradient_clip_val=1.0, # Add gradient clipping to prevent NaN\n    accumulate_grad_batches=6,\n)\n\n# Fit the model\ntrainer.fit(model, data_module)","metadata":{"trusted":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-11-04T09:09:45.774223Z","iopub.execute_input":"2025-11-04T09:09:45.774453Z","execution_failed":"2025-11-04T09:11:07.774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the path to save the model\nmodel_save_path = \"biomass_regression_model.pth\"\n\n# Save the model's state dictionary\ntorch.save(model.state_dict(), model_save_path)\nprint(f\"Model saved to {model_save_path}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-04T09:11:07.774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()\n\n# Read the metrics.csv using the trainer's logger directory\nmetrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n\n# Remove the step column and set epoch as index\n# metrics.set_index(\"step\", inplace=True)\ndisplay(metrics.dropna(axis=1, how=\"all\").head())\n\n# Melt the DataFrame to long-form for plotting\nmetrics_melted = metrics.reset_index().melt(id_vars='epoch', var_name='metric', value_name='value')\n\n# Define metric groups\nmetric_groups = {\n    'Loss': [c for c in metrics.columns if \"_loss\" in c],\n    'MAE': [c for c in metrics.columns if \"_mae\" in c and \"loss\" not in c],\n    'MSE': [c for c in metrics.columns if \"_mse\" in c and \"loss\" not in c],\n}\n\n# Plot metrics for each group in a separate chart\nfor title, metric_list in metric_groups.items():\n    # Filter melted DataFrame for the current group\n    group_metrics = metrics_melted[metrics_melted['metric'].isin(metric_list)]\n\n    plt.figure(figsize=(10, 5))\n    sns.lineplot(data=group_metrics, x='epoch', y='value', hue='metric')\n    plt.title(f'{title} over Epochs', fontsize=14, fontweight='bold')\n    plt.xlabel('Epoch', fontsize=12)\n    plt.ylabel(title, fontsize=12)\n    plt.grid(True, alpha=0.3)\n    plt.yscale('log')  # Set y-axis to logarithmic scale\n    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left') # Move legend outside and to the right\n    plt.tight_layout() # Adjust layout to prevent legend overlap\n    plt.show()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-04T09:11:07.774Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prediction & submission","metadata":{}},{"cell_type":"code","source":"# Assuming data_module is already initialized and setup has been called for the test stage\n# If not, uncomment and run the DataModule initialization and setup cells first\n# data_module = BiomassDataModule(PATH_DATA, batch_size=64)\n# data_module.setup(stage='test')\n\n# Get the test dataloader\ntest_loader = data_module.test_dataloader()\n\n# Get the first batch from the test dataloader\ntest_images, test_target_names, test_img_paths = next(iter(test_loader))\n\n# Determine how many images to show (e.g., the first 4 from the batch)\nn_images_to_show = min(4, test_images.shape[0])\n\nfig, axes = plt.subplots(1, n_images_to_show, figsize=(4 * n_images_to_show, 5))\n\n# Ensure axes is an array even for a single image\nif n_images_to_show == 1:\n    axes = [axes]\n\nfor i in range(n_images_to_show):\n    img = test_images[i].permute(1, 2, 0).numpy() # Convert from CxHxW to HxWxD for displaying\n    # Denormalize the image for better visualization (using ImageNet standards)\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    img = std * img + mean\n    img = np.clip(img, 0, 1) # Clip values to be between 0 and 1\n\n    axes[i].imshow(img)\n    # Display the sample_id for test images\n    axes[i].set_title(f\"path: {test_img_paths[i]}\\n target name: {test_target_names[i]}\")\n    axes[i].axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-04T09:11:07.774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Need to re-initialize the test dataloader to get the sample_ids in order\ntest_loader = data_module.test_dataloader()\n\n# Debugging: Print the type and content of test_loader\nprint(f\"Type of test_loader: {type(test_loader)}\")\nprint(f\"Content of test_loader: {test_loader}\")\n\n\n# Generate predictions on the test set by manually iterating through the dataloader\nall_predictions = []\nall_image_paths = []\n\nmodel.eval() # Set the model to evaluation mode\nwith torch.no_grad(): # Disable gradient calculation\n    for batch in test_loader:\n        # Assuming predict_step returns (target_pred, species_pred, ndvi_pred, height_pred, img_relative_path)\n        target_pred, img_relative_path = model.predict_step(batch, batch_idx=0) # Pass batch and dummy batch_idx\n\n        # Convert target predictions to list and extend the all_predictions list\n        all_predictions.extend(target_pred.tolist())\n        all_image_paths.extend(img_relative_path)\n\n\n# Create a DataFrame with predictions and image paths\npredictions_raw_df = pd.DataFrame({'image_path': all_image_paths, 'target': all_predictions})\n\n# Display the first few rows of the submiss\nprint(\"Submission DataFrame head:\")\ndisplay(predictions_raw_df.head())\n\n# You can save the submission_df to a CSV file in the required format\n# submission_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-04T09:11:07.774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# prevent any negative values\npredictions_raw_df[predictions_raw_df[\"target\"] < 0][\"target\"] = 0\n\n# Group by image_path and take the mean of the predictions\nprediction_df = predictions_raw_df.groupby('image_path')['target'].mean().reset_index()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-04T09:11:07.774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the path to the sample submission file\ntest_csv_path = os.path.join(PATH_DATA, 'test.csv')\n\n# Load the sample submission file\ntest_csv = pd.read_csv(test_csv_path)\n# display(test_csv.head())\n\n# del sample_submission_df['target']\ntest_csv = test_csv.merge(prediction_df, on='image_path', how='left')\ndisplay(test_csv.head())\n\n# dump prediction into CSV file\ntest_csv[[\"sample_id\", \"target\"]].to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-04T09:11:07.775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"! head submission.csv","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-04T09:11:07.775Z"}},"outputs":[],"execution_count":null}]}