{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\ntrain_data = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:16:17.841222Z","iopub.execute_input":"2025-11-20T17:16:17.841722Z","iopub.status.idle":"2025-11-20T17:16:18.132975Z","shell.execute_reply.started":"2025-11-20T17:16:17.841697Z","shell.execute_reply":"2025-11-20T17:16:18.132320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:16:20.391877Z","iopub.execute_input":"2025-11-20T17:16:20.392151Z","iopub.status.idle":"2025-11-20T17:16:20.426288Z","shell.execute_reply.started":"2025-11-20T17:16:20.392128Z","shell.execute_reply":"2025-11-20T17:16:20.425610Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T14:28:41.002127Z","iopub.execute_input":"2025-11-10T14:28:41.002442Z","iopub.status.idle":"2025-11-10T14:28:41.009773Z","shell.execute_reply.started":"2025-11-10T14:28:41.002419Z","shell.execute_reply":"2025-11-10T14:28:41.008622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"1785/5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T14:41:22.366000Z","iopub.execute_input":"2025-11-10T14:41:22.366344Z","iopub.status.idle":"2025-11-10T14:41:22.372762Z","shell.execute_reply.started":"2025-11-10T14:41:22.366314Z","shell.execute_reply":"2025-11-10T14:41:22.371638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimage_files = os.listdir(f\"/kaggle/input/csiro-biomass/train/\")\nprint(len(image_files))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T14:43:35.911456Z","iopub.execute_input":"2025-11-10T14:43:35.911817Z","iopub.status.idle":"2025-11-10T14:43:35.931136Z","shell.execute_reply.started":"2025-11-10T14:43:35.911791Z","shell.execute_reply":"2025-11-10T14:43:35.929881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_data['State'].unique())\nprint(train_data['Species'].unique())\n#print(train_data['Pre_GSHH_NDVI'].unique())\nprint(train_data['target_name'].unique())\n#print(train_data['Height_Ave_cm'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T16:56:45.808651Z","iopub.execute_input":"2025-11-20T16:56:45.809009Z","iopub.status.idle":"2025-11-20T16:56:45.815370Z","shell.execute_reply.started":"2025-11-20T16:56:45.808985Z","shell.execute_reply":"2025-11-20T16:56:45.814632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Columns to plot\ncolumns_to_plot = ['State', 'Species', 'target_name']\n\n# Loop through each column\nfor col in columns_to_plot:\n    counts = train_data[col].value_counts() / 5  # divide by 5 since 5 samples per image\n    \n    plt.figure(figsize=(8, 4))\n    bars = plt.bar(counts.index, counts.values, color='skyblue', edgecolor='black')\n    plt.title(f'Number of images (adjusted) for each category in \"{col}\"', fontsize=14)\n    plt.xlabel(col, fontsize=12)\n    plt.ylabel('Image Count (count/5)', fontsize=12)\n    plt.xticks(rotation=45, ha='right')\n\n    # Add count labels on top of each bar\n    for bar in bars:\n        height = bar.get_height()\n        plt.text(bar.get_x() + bar.get_width()/2, height, f'{int(height)}',\n                 ha='center', va='bottom', fontsize=10)\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T15:16:18.367487Z","iopub.execute_input":"2025-11-10T15:16:18.367858Z","iopub.status.idle":"2025-11-10T15:16:19.055744Z","shell.execute_reply.started":"2025-11-10T15:16:18.367834Z","shell.execute_reply":"2025-11-10T15:16:19.054749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --quiet --no-deps timm==0.9.2 tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:16:52.476906Z","iopub.execute_input":"2025-11-20T17:16:52.477522Z","iopub.status.idle":"2025-11-20T17:16:56.022936Z","shell.execute_reply.started":"2025-11-20T17:16:52.477493Z","shell.execute_reply":"2025-11-20T17:16:56.022131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2) Feature extraction script (run in a notebook cell)\nimport os\nfrom pathlib import Path\nfrom PIL import Image\nimport numpy as np\nfrom tqdm import tqdm\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport timm  # timm provides many pretrained models\n\n# --------- USER CONFIG ----------\n# Path to folder that contains images (change to your folder)\nIMAGE_DIR = \"/kaggle/input/csiro-biomass/train\"  # <-- set this to your images folder\n# Where to write output (Kaggle working dir)\nOUTPUT_DIR = \"/kaggle/working/\"\nBATCH_SIZE = 2\nNUM_WORKERS = 2\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n# --------------------------------\n\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\n# Accept common image file extensions\nIMG_EXTS = {\".jpg\", \".jpeg\", \".png\", \".bmp\", \".tiff\", \".webp\"}\n\n# 1) Build dataset that yields PIL images and filename\nclass ImageFolderList(Dataset):\n    def __init__(self, root_dir, transform=None):\n        self.root_dir = Path(root_dir)\n        # gather files (non-recursive). If you want recursive set rglob('*')\n        files = sorted([p for p in self.root_dir.rglob(\"*\") if p.suffix.lower() in IMG_EXTS])\n        self.files = files\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.files)\n    \n    def __getitem__(self, idx):\n        p = self.files[idx]\n        img = Image.open(p).convert(\"RGB\")\n        if self.transform:\n            img = self.transform(img)\n        return img, str(p)\n\n# 2) Preprocessing: efficientnet_b0 expects 224x224; use ImageNet mean/std\ntransform = transforms.Compose([\n    #transforms.Resize(256),\n    #transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std =[0.229, 0.224, 0.225])\n])\n\n# 3) Create dataset & loader\ndataset = ImageFolderList(IMAGE_DIR, transform=transform)\nif len(dataset) == 0:\n    raise RuntimeError(f\"No images found in {IMAGE_DIR}. Check the path and extensions.\")\n\nloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True)\n\n# 4) Build model: timm efficientnet_b0 pretrained on ImageNet\n#    We'll create a model that outputs global pooled features.\nmodel = timm.create_model('efficientnet_b0', pretrained=True, num_classes=0, global_pool='avg')  \n# num_classes=0 with global_pool='avg' makes the model return feature vectors from the global pooling layer\n# (timm behavior). If timm version behaves differently, see fallback below.\nmodel.to(DEVICE)\nmodel.eval()\n\n# 5) Iterate and collect features\nall_features = []\nall_filenames = []\n\nwith torch.no_grad():\n    for batch_imgs, batch_paths in tqdm(loader, desc=\"Extracting features\"):\n        batch_imgs = batch_imgs.to(DEVICE)\n        # model returns features with shape (batch, feat_dim)\n        feats = model(batch_imgs)\n        # If result is a tuple (some timm versions), try to handle it:\n        if isinstance(feats, (list, tuple)):\n            feats = feats[0]\n        feats = feats.cpu().numpy()\n        all_features.append(feats)\n        all_filenames.extend(batch_paths)\n\n# Stack into arrays\nfeatures_array = np.vstack(all_features)  # shape: (N_images, feat_dim)\nfilenames_array = np.array(all_filenames)\n\nprint(\"Extracted features shape:\", features_array.shape)\nprint(\"Number of files:\", filenames_array.shape[0])\n\n# 6) Save to .npy files\nfeatures_out = os.path.join(OUTPUT_DIR, \"features.npy\")\nfilenames_out = os.path.join(OUTPUT_DIR, \"filenames.npy\")\nnp.save(features_out, features_array)\nnp.save(filenames_out, filenames_array)\n\nprint(f\"Saved features -> {features_out}\")\nprint(f\"Saved filenames -> {filenames_out}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:21:57.446794Z","iopub.execute_input":"2025-11-20T17:21:57.447741Z","iopub.status.idle":"2025-11-20T17:22:23.438043Z","shell.execute_reply.started":"2025-11-20T17:21:57.447703Z","shell.execute_reply":"2025-11-20T17:22:23.437278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\ndata = np.load(\"/kaggle/working/features.npy\")\nprint(data.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:23:24.259264Z","iopub.execute_input":"2025-11-20T17:23:24.259931Z","iopub.status.idle":"2025-11-20T17:23:24.266154Z","shell.execute_reply.started":"2025-11-20T17:23:24.259900Z","shell.execute_reply":"2025-11-20T17:23:24.265485Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\ndata = np.load(\"/kaggle/working/filenames.npy\")\nprint(data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:23:46.134530Z","iopub.execute_input":"2025-11-20T17:23:46.135390Z","iopub.status.idle":"2025-11-20T17:23:46.141148Z","shell.execute_reply.started":"2025-11-20T17:23:46.135361Z","shell.execute_reply":"2025-11-20T17:23:46.140349Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 2) Feature extraction script (run in a notebook cell)\nimport os\nfrom pathlib import Path\nfrom PIL import Image\nimport numpy as np\nfrom tqdm import tqdm\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nimport timm  # timm provides many pretrained models\n\n# --------- USER CONFIG ----------\n# Path to folder that contains images (change to your folder)\nIMAGE_DIR = \"/kaggle/input/csiro-biomass/test\"  # <-- set this to your images folder\n# Where to write output (Kaggle working dir)\nOUTPUT_DIR = \"/kaggle/working/\"\nBATCH_SIZE = 1\nNUM_WORKERS = 0\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n# --------------------------------\n\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\n# Accept common image file extensions\nIMG_EXTS = {\".jpg\", \".jpeg\", \".png\", \".bmp\", \".tiff\", \".webp\"}\n\n# 1) Build dataset that yields PIL images and filename\nclass ImageFolderList(Dataset):\n    def __init__(self, root_dir, transform=None):\n        self.root_dir = Path(root_dir)\n        # gather files (non-recursive). If you want recursive set rglob('*')\n        files = sorted([p for p in self.root_dir.rglob(\"*\") if p.suffix.lower() in IMG_EXTS])\n        self.files = files\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.files)\n    \n    def __getitem__(self, idx):\n        p = self.files[idx]\n        img = Image.open(p).convert(\"RGB\")\n        if self.transform:\n            img = self.transform(img)\n        return img, str(p)\n\n# 2) Preprocessing: efficientnet_b0 expects 224x224; use ImageNet mean/std\ntransform = transforms.Compose([\n    #transforms.Resize(256),\n    #transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std =[0.229, 0.224, 0.225])\n])\n\n# 3) Create dataset & loader\ndataset = ImageFolderList(IMAGE_DIR, transform=transform)\nif len(dataset) == 0:\n    raise RuntimeError(f\"No images found in {IMAGE_DIR}. Check the path and extensions.\")\n\nloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True)\n\n# 4) Build model: timm efficientnet_b0 pretrained on ImageNet\n#    We'll create a model that outputs global pooled features.\nmodel = timm.create_model('efficientnet_b0', pretrained=True, num_classes=0, global_pool='avg')  \n# num_classes=0 with global_pool='avg' makes the model return feature vectors from the global pooling layer\n# (timm behavior). If timm version behaves differently, see fallback below.\nmodel.to(DEVICE)\nmodel.eval()\n\n# 5) Iterate and collect features\nall_features = []\nall_filenames = []\n\nwith torch.no_grad():\n    for batch_imgs, batch_paths in tqdm(loader, desc=\"Extracting features\"):\n        batch_imgs = batch_imgs.to(DEVICE)\n        # model returns features with shape (batch, feat_dim)\n        feats = model(batch_imgs)\n        # If result is a tuple (some timm versions), try to handle it:\n        if isinstance(feats, (list, tuple)):\n            feats = feats[0]\n        feats = feats.cpu().numpy()\n        all_features.append(feats)\n        all_filenames.extend(batch_paths)\n\n# Stack into arrays\nfeatures_array = np.vstack(all_features)  # shape: (N_images, feat_dim)\nfilenames_array = np.array(all_filenames)\n\nprint(\"Extracted features shape:\", features_array.shape)\nprint(\"Number of files:\", filenames_array.shape[0])\n\n# 6) Save to .npy files\nfeatures_out = os.path.join(OUTPUT_DIR, \"features_test.npy\")\nfilenames_out = os.path.join(OUTPUT_DIR, \"filenames_test.npy\")\nnp.save(features_out, features_array)\nnp.save(filenames_out, filenames_array)\n\nprint(f\"Saved features -> {features_out}\")\nprint(f\"Saved filenames -> {filenames_out}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:25:08.919275Z","iopub.execute_input":"2025-11-20T17:25:08.920044Z","iopub.status.idle":"2025-11-20T17:25:09.390896Z","shell.execute_reply.started":"2025-11-20T17:25:08.920017Z","shell.execute_reply":"2025-11-20T17:25:09.390129Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\ndata = np.load(\"/kaggle/working/features_test.npy\")\nprint(data.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:26:40.612985Z","iopub.execute_input":"2025-11-20T17:26:40.613694Z","iopub.status.idle":"2025-11-20T17:26:40.619020Z","shell.execute_reply.started":"2025-11-20T17:26:40.613670Z","shell.execute_reply":"2025-11-20T17:26:40.618158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load features, compute Euclidean distances, plot distances + KDE, print top-10 matches\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.neighbors import KernelDensity\n\n# ---------- USER CONFIG ----------\nTRAIN_FEATURES_PATH = \"/kaggle/working/features.npy\"   # path to train features (N, D)\nTEST_FEATURES_PATH  = \"/kaggle/working/features_test.npy\"  # path to test features (1, D) or (D,)\nFILENAMES_PATH = \"/kaggle/working/filenames.npy\"       # path to filenames corresponding to train features\nOUTPUT_DIR = \"/kaggle/working\"\nTOP_K = 10\n# ----------------------------------\n\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\n# 1) Load arrays\ntrain_feats = np.load(TRAIN_FEATURES_PATH)\ntest_feats = np.load(TEST_FEATURES_PATH)\n# filenames might be saved as object dtype; allow_pickle when loading if needed\ntry:\n    filenames = np.load(FILENAMES_PATH)\nexcept Exception:\n    filenames = np.load(FILENAMES_PATH, allow_pickle=True)\n\n# Ensure shapes are expected\nif test_feats.ndim == 2 and test_feats.shape[0] == 1:\n    query = test_feats[0]\nelif test_feats.ndim == 1:\n    query = test_feats\nelse:\n    raise ValueError(f\"Unexpected shape for test features: {test_feats.shape}\")\n\nif train_feats.ndim != 2:\n    raise ValueError(f\"Unexpected shape for train features: {train_feats.shape}\")\n\nif train_feats.shape[1] != query.shape[0]:\n    raise ValueError(f\"Feature dimension mismatch: train {train_feats.shape}, test {query.shape}\")\n\nN = train_feats.shape[0]\nprint(f\"Loaded train features: {train_feats.shape}, test feature: {query.shape}, N_train = {N}\")\nprint(f\"Filenames shape: {filenames.shape}\")\n\n# 2) Compute Euclidean distances\n# Efficient vectorized computation: ||x - q||_2\ndiffs = train_feats - query[np.newaxis, :]    # shape (N, D)\ndists = np.linalg.norm(diffs, axis=1)          # shape (N,)\n\n# 3) Sort distances to get nearest neighbors\nsorted_idx = np.argsort(dists)\ntopk_idx = sorted_idx[:TOP_K]\ntopk_dists = dists[topk_idx]\ntopk_files = filenames[topk_idx]\n\n# 4) Print top-K\nprint(f\"\\nTop {TOP_K} closest train images (Euclidean distance):\")\nfor rank, (idx, dist, fname) in enumerate(zip(topk_idx, topk_dists, topk_files), start=1):\n    print(f\"{rank:2d}. idx={int(idx):4d}  dist={dist:.6f}  file={fname}\")\n\n# 5) Plot distances (scatter) and KDE\nplt.figure(figsize=(12,5))\n\n# Scatter / stem plot of distances (unsorted order)\nplt.subplot(1,2,1)\nplt.scatter(range(N), dists, s=12)\nplt.scatter(topk_idx, dists[topk_idx], color='red', s=30, label=f\"Top {TOP_K} closest\")\nplt.xlabel(\"Train image index\")\nplt.ylabel(\"Euclidean distance to test\")\nplt.title(\"Distances from test embedding to all train embeddings\")\nplt.legend()\nplt.grid(alpha=0.25)\n\n# KDE of distances using sklearn's KernelDensity (1D)\nplt.subplot(1,2,2)\nX = dists[:, np.newaxis]\nkde = KernelDensity(kernel=\"gaussian\", bandwidth=0.1).fit(X)  # bandwidth may be tuned\nx_plot = np.linspace(dists.min() - 1e-6, dists.max() + 1e-6, 512)[:, np.newaxis]\nlog_dens = kde.score_samples(x_plot)\ndens = np.exp(log_dens)\n\nplt.plot(x_plot[:,0], dens)\nplt.fill_between(x_plot[:,0], dens, alpha=0.3)\nplt.xlabel(\"Euclidean distance\")\nplt.ylabel(\"Density (KDE)\")\nplt.title(\"KDE of distances\")\nplt.grid(alpha=0.25)\n\nplt.tight_layout()\nplot_path = os.path.join(OUTPUT_DIR, \"distances_and_kde.png\")\nplt.savefig(plot_path, dpi=150, bbox_inches=\"tight\")\nprint(f\"\\nSaved plot to: {plot_path}\")\nplt.show()\n\n# 6) (Optional) Save top-K results to CSV for convenience\nimport csv\ncsv_out = os.path.join(OUTPUT_DIR, \"topk_matches.csv\")\nwith open(csv_out, \"w\", newline=\"\", encoding=\"utf-8\") as f:\n    writer = csv.writer(f)\n    writer.writerow([\"rank\", \"train_index\", \"distance\", \"filename\"])\n    for rank, (idx, dist, fname) in enumerate(zip(topk_idx, topk_dists, topk_files), start=1):\n        writer.writerow([rank, int(idx), float(dist), str(fname)])\nprint(\"Saved top-k matches to:\", csv_out)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:29:48.097495Z","iopub.execute_input":"2025-11-20T17:29:48.098113Z","iopub.status.idle":"2025-11-20T17:29:49.857633Z","shell.execute_reply.started":"2025-11-20T17:29:48.098088Z","shell.execute_reply":"2025-11-20T17:29:49.857034Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_10_matches = pd.read_csv('/kaggle/working/topk_matches.csv')\ntop_10_matches","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:32:07.665175Z","iopub.execute_input":"2025-11-20T17:32:07.665947Z","iopub.status.idle":"2025-11-20T17:32:07.678429Z","shell.execute_reply.started":"2025-11-20T17:32:07.665921Z","shell.execute_reply":"2025-11-20T17:32:07.677478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================== CONFIG ==================\nTOPK_CSV = \"/kaggle/working/topk_matches.csv\"   # CSV with columns: rank, train_index, distance, filename\nTRAIN_CSV = \"/kaggle/input/csiro-biomass/train.csv\"  # path to train_data csv (the one you showed)\nOUTPUT_DIR = \"/kaggle/working\"\n# Provide the test sample prefix (the numeric id part). \n# Final sample_id will be e.g. \"ID1001187975__Dry_Clover_g\" etc.\nTEST_SAMPLE_PREFIX = \"ID1001187975\"\n# The five target names (adjust capitalization to match your train.csv 'target_name' values if needed)\nTARGET_NAMES = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\nTOP_K = 10\n# ============================================\n\nimport os\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\n\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\n# 1) Load the top-k CSV and the train dataframe\ntopk_df = pd.read_csv(TOPK_CSV)\ntrain_df = pd.read_csv(TRAIN_CSV)\n\n# Print quick checks\nprint(\"Top-k CSV columns:\", topk_df.columns.tolist())\nprint(\"Train CSV columns:\", train_df.columns.tolist())\n\n# We expect topk CSV to contain a filename column (string paths to train images)\n# Possible header names: 'filename' or 'file' or 'filename' depending on previous code.\n# Try to find a column with 'file' or 'filename' in its name:\nfilename_col = None\nfor c in topk_df.columns:\n    if \"file\" in c.lower():\n        filename_col = c\n        break\nif filename_col is None:\n    raise RuntimeError(\"Could not find a filename column in topk CSV. Columns: \" + \", \".join(topk_df.columns))\n\nprint(\"Using filename column:\", filename_col)\n\n# Take top-K rows (in case CSV contains more)\ntopk = topk_df.head(TOP_K).copy()\nprint(f\"Using top {len(topk)} rows from {TOPK_CSV}\")\n\n# 2) Extract image id from filename paths.\n# Example filename: \"train/ID1011485656.jpg\" => image_id \"ID1011485656\"\ndef extract_image_id(path_str):\n    p = Path(path_str)\n    return p.stem\n\ntopk[\"image_id\"] = topk[filename_col].astype(str).apply(extract_image_id)\nprint(\"Top-k image ids:\\n\", topk[\"image_id\"].tolist())\n\n# 3) For each target, collect values for each top-k image and average\nresults = []\nmissing_images = []\nfor tname in TARGET_NAMES:\n    vals = []\n    for img_id in topk[\"image_id\"]:\n        # The train dataframe row for this image and target is where image_path ends with f\"{img_id}.jpg\"\n        # OR where sample_id starts with img_id (if you prefer). We'll try both robustly.\n        # 1) Match image_path (exact match or endswith)\n        match1 = train_df[train_df[\"image_path\"].astype(str).str.endswith(f\"{img_id}.jpg\")]\n        # 2) Match sample_id starting with image id (sample_id looks like ID1011485656__Dry_Clover_g)\n        match2 = train_df[train_df[\"sample_id\"].astype(str).str.startswith(f\"{img_id}__\")]\n        # Combine potential matches\n        cand = pd.concat([match1, match2]).drop_duplicates()\n        # Now filter by the target_name column matching tname (exact)\n        cand = cand[cand[\"target_name\"].astype(str) == tname]\n        if len(cand) == 0:\n            # Try case-insensitive match on target_name as a fallback\n            cand = pd.concat([match1, match2]).drop_duplicates()\n            cand = cand[cand[\"target_name\"].astype(str).str.lower() == tname.lower()]\n        if len(cand) == 0:\n            # couldn't find the target for this image\n            missing_images.append((img_id, tname))\n            continue\n        # If multiple rows found (should be 1), take the mean just to be robust\n        val = cand[\"target\"].astype(float).mean()\n        vals.append(val)\n    if len(vals) == 0:\n        avg_val = np.nan\n    else:\n        avg_val = float(np.mean(vals))\n    results.append((tname, avg_val, len(vals)))\n\n# Report missing lookups if any\nif len(missing_images) > 0:\n    print(f\"\\nWarning: could not find {len(missing_images)} (image, target) pairs in train.csv. Examples:\")\n    print(missing_images[:10])\n\n# 4) Build test.csv rows: sample_id = f\"{TEST_SAMPLE_PREFIX}__{target_name}\"\nout_rows = []\nfor tname, avg_val, count_found in results:\n    sample_id = f\"{TEST_SAMPLE_PREFIX}__{tname}\"\n    out_rows.append({\"sample_id\": sample_id, \"target\": avg_val})\n\nout_df = pd.DataFrame(out_rows)\n\n# 5) Save test.csv\nout_path = os.path.join(OUTPUT_DIR, \"test.csv\")\nout_df.to_csv(out_path, index=False)\nprint(f\"\\nSaved predictions to {out_path}\\n\")\n\n# 6) Print results\nprint(\"Predicted averages for test image (averaged over top-K = {} train images):\".format(TOP_K))\nprint(out_df.to_string(index=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:46:41.321186Z","iopub.execute_input":"2025-11-20T17:46:41.321732Z","iopub.status.idle":"2025-11-20T17:46:41.530225Z","shell.execute_reply.started":"2025-11-20T17:46:41.321707Z","shell.execute_reply":"2025-11-20T17:46:41.529467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/working/test.csv')\ntest","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-20T17:47:31.356189Z","iopub.execute_input":"2025-11-20T17:47:31.356449Z","iopub.status.idle":"2025-11-20T17:47:31.366276Z","shell.execute_reply.started":"2025-11-20T17:47:31.356431Z","shell.execute_reply":"2025-11-20T17:47:31.365535Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null}]}