{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":127282,"databundleVersionId":15520105,"sourceType":"competition"}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install wildlife-datasets wildlife-tools timm scikit-learn --quiet --upgrade","metadata":{"_uuid":"d6b21a67-b575-4935-9a02-c40beaba0cf1","_cell_guid":"fcbbb870-2177-4a34-9e7b-18845ae560e8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:13:00.746130Z","iopub.execute_input":"2026-02-16T03:13:00.746369Z","iopub.status.idle":"2026-02-16T03:13:13.948252Z","shell.execute_reply.started":"2026-02-16T03:13:00.746331Z","shell.execute_reply":"2026-02-16T03:13:13.947564Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Imports","metadata":{"_uuid":"46a937df-8bee-4e57-9b12-84fe15659292","_cell_guid":"be360a5b-a06e-4ddb-88e8-0b3cb0627407","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import os\nimport gc\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.transforms as T\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm.notebook import tqdm\nfrom sklearn.cluster import AgglomerativeClustering, DBSCAN\nfrom sklearn.preprocessing import normalize\nfrom sklearn.metrics import silhouette_score\nimport timm\nfrom transformers import AutoModel\nfrom wildlife_datasets.datasets import AnimalCLEF2026\nfrom PIL import Image\nfrom collections import defaultdict","metadata":{"_uuid":"a1d06eca-1342-449a-8afb-2e998eeaea40","_cell_guid":"829346e2-ab1b-4e0a-b727-f95967cf6f7d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:13:13.949393Z","iopub.execute_input":"2026-02-16T03:13:13.949693Z","iopub.status.idle":"2026-02-16T03:13:32.135181Z","shell.execute_reply.started":"2026-02-16T03:13:13.949655Z","shell.execute_reply":"2026-02-16T03:13:32.134598Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Config","metadata":{"_uuid":"9789cc6f-495d-431d-84c2-6e1c3027ee91","_cell_guid":"c7c5b674-8a86-4c77-80cb-f689e942d23a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"warnings.filterwarnings(\"ignore\")\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nBATCH_SIZE = 48 # Slightly lower to allow multi-crop without OOM\nNUM_WORKERS = 4\nROOT_DIR = \"/kaggle/input/animal-clef-2026\"\n\nSPECIES_CONFIG = {\n \"LynxID2025\": {\n \"models\": [\"miewid\", \"megadescriptor\"],\n \"model_weights\": [0.6, 0.4],\n \"threshold_range\": (0.28, 0.45),\n \"image_size\": {\"miewid\": 512, \"megadescriptor\": 384},\n \"qe_k\": 5,\n \"qe_alpha\": 0.6,\n \"tta\": True,\n },\n \"SalamanderID2025\": {\n \"models\": [\"megadescriptor\", \"miewid\"],\n \"model_weights\": [0.65, 0.35],\n \"threshold_range\": (0.18, 0.35),\n \"image_size\": {\"miewid\": 512, \"megadescriptor\": 384},\n \"qe_k\": 3,\n \"qe_alpha\": 0.4,\n \"tta\": True,\n },\n \"SeaTurtleID2022\": {\n \"models\": [\"megadescriptor\", \"miewid\"],\n \"model_weights\": [0.55, 0.45],\n \"threshold_range\": (0.35, 0.55),\n \"image_size\": {\"miewid\": 512, \"megadescriptor\": 384},\n \"qe_k\": 8,\n \"qe_alpha\": 0.5,\n \"tta\": True,\n },\n \"TexasHornedLizards\": {\n \"models\": [\"miewid\", \"megadescriptor\"],\n \"model_weights\": [0.55, 0.45],\n \"threshold_range\": (0.22, 0.40),\n \"image_size\": {\"miewid\": 512, \"megadescriptor\": 384},\n \"qe_k\": 5,\n \"qe_alpha\": 0.5,\n \"tta\": True,\n },\n}\n\nprint(f\"šŸ–„ļø Device: {DEVICE}\")\nif torch.cuda.is_available():\n print(f\"šŸ–„ļø GPU: {torch.cuda.get_device_name(0)}\")\n print(f\"šŸ–„ļø VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB\")\nif torch.cuda.device_count() > 1:\n print(f\"šŸš€ Using {torch.cuda.device_count()} GPUs\")","metadata":{"_uuid":"3e7d95fd-8f05-43f2-bf94-1788854040be","_cell_guid":"65732e4c-d675-4d6c-8055-810e674c4211","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:14:52.598467Z","iopub.execute_input":"2026-02-16T03:14:52.599179Z","iopub.status.idle":"2026-02-16T03:14:52.606777Z","shell.execute_reply.started":"2026-02-16T03:14:52.599153Z","shell.execute_reply":"2026-02-16T03:14:52.605997Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Models","metadata":{"_uuid":"a8522a8f-0b3e-4c5b-8684-c2b5b862579a","_cell_guid":"da6169e2-3341-4ec0-ab63-96db373ada1d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"_model_cache = {}\n\n\nclass MiewIDWrapper(nn.Module):\n def __init__(self):\n super().__init__()\n self.backbone = AutoModel.from_pretrained(\n \"conservationxlabs/miewid-msv3\", trust_remote_code=True\n )\n\n def forward(self, x):\n return self.backbone(x)\n\n\nclass MegaDescriptorWrapper(nn.Module):\n def __init__(self):\n super().__init__()\n self.backbone = timm.create_model(\n \"hf-hub:BVRA/MegaDescriptor-L-384\", pretrained=True\n )\n self.backbone.reset_classifier(0)\n\n def forward(self, x):\n return self.backbone(x)\n\n\ndef get_model(model_type):\n \"\"\"Get model with caching to avoid reloading.\"\"\"\n if model_type in _model_cache:\n return _model_cache[model_type]\n\n if model_type == \"miewid\":\n model = MiewIDWrapper()\n else:\n model = MegaDescriptorWrapper()\n\n model.to(DEVICE)\n if torch.cuda.device_count() > 1:\n model = nn.DataParallel(model)\n model.eval()\n _model_cache[model_type] = model\n return model","metadata":{"_uuid":"6466ad6f-877d-40e4-bc6c-8545508544fe","_cell_guid":"c454f476-388d-49a0-a6c0-c59855f16cf6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:14:59.282580Z","iopub.execute_input":"2026-02-16T03:14:59.282884Z","iopub.status.idle":"2026-02-16T03:14:59.290025Z","shell.execute_reply.started":"2026-02-16T03:14:59.282858Z","shell.execute_reply":"2026-02-16T03:14:59.289212Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test-Time Augmentation (TTA)","metadata":{"_uuid":"8dd87ed1-95e5-4655-87d2-8cbe115d7782","_cell_guid":"ff562bd9-046b-48ea-8327-90a0e70876bc","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def get_tta_transforms(image_size):\n \"\"\"Return a list of TTA transform pipelines.\"\"\"\n base_norm = T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n\n transforms = [\n # Original\n T.Compose([\n T.Resize((image_size, image_size)),\n T.ToTensor(),\n base_norm,\n ]),\n # Horizontal flip\n T.Compose([\n T.Resize((image_size, image_size)),\n T.RandomHorizontalFlip(p=1.0),\n T.ToTensor(),\n base_norm,\n ]),\n # Slight crop center\n T.Compose([\n T.Resize((int(image_size * 1.1), int(image_size * 1.1))),\n T.CenterCrop(image_size),\n T.ToTensor(),\n base_norm,\n ]),\n # Color jitter (mild)\n T.Compose([\n T.Resize((image_size, image_size)),\n T.ColorJitter(brightness=0.1, contrast=0.1),\n T.ToTensor(),\n base_norm,\n ]),\n # Slight rotation via affine\n T.Compose([\n T.Resize((image_size, image_size)),\n T.RandomAffine(degrees=5, translate=(0.02, 0.02)),\n T.ToTensor(),\n base_norm,\n ]),\n ]\n return transforms","metadata":{"_uuid":"13d63c68-21d5-462f-80e0-6654bb7c4acb","_cell_guid":"3e9434f6-2ec9-4393-8f28-c3438eafbe92","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:15:03.362137Z","iopub.execute_input":"2026-02-16T03:15:03.362881Z","iopub.status.idle":"2026-02-16T03:15:03.368498Z","shell.execute_reply.started":"2026-02-16T03:15:03.362855Z","shell.execute_reply":"2026-02-16T03:15:03.367780Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature Extraction","metadata":{"_uuid":"3386f2d4-5b24-4171-bb66-1720b3b1ff3e","_cell_guid":"dfd51fba-279b-459e-aeaf-a20012245ece","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def extract_features_single(model, dataset, image_size):\n \"\"\"Extract features with a single transform.\"\"\"\n dataset.transform = T.Compose([\n T.Resize((image_size, image_size)),\n T.ToTensor(),\n T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n ])\n loader = DataLoader(\n dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS,\n shuffle=False, pin_memory=True\n )\n all_features = []\n\n with torch.no_grad(), torch.cuda.amp.autocast():\n for batch in tqdm(loader, leave=False, desc=\"Extracting\"):\n images = batch[0].to(DEVICE, non_blocking=True)\n\n # Forward + horizontal flip (basic TTA always on)\n feats_orig = model(images)\n feats_flip = model(torch.flip(images, dims=[3]))\n feats = feats_orig + feats_flip\n\n feats = F.normalize(feats.float(), p=2, dim=1)\n all_features.append(feats.cpu().numpy())\n\n return np.concatenate(all_features)\n\n\ndef extract_features_tta(model, dataset, image_size):\n \"\"\"Extract features with full TTA (multiple augmentations averaged).\"\"\"\n tta_transforms = get_tta_transforms(image_size)\n all_feats = []\n\n for i, tfm in enumerate(tta_transforms):\n dataset.transform = tfm\n loader = DataLoader(\n dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS,\n shuffle=False, pin_memory=True\n )\n feats_list = []\n\n with torch.no_grad(), torch.cuda.amp.autocast():\n for batch in tqdm(loader, leave=False, desc=f\"TTA {i+1}/{len(tta_transforms)}\"):\n images = batch[0].to(DEVICE, non_blocking=True)\n feats = model(images)\n feats = F.normalize(feats.float(), p=2, dim=1)\n feats_list.append(feats.cpu().numpy())\n\n all_feats.append(np.concatenate(feats_list))\n\n # Average all TTA features\n stacked = np.stack(all_feats, axis=0)\n averaged = np.mean(stacked, axis=0)\n return normalize(averaged, axis=1, norm=\"l2\")\n\n\ndef extract_features(model, dataset, image_size, use_tta=False):\n \"\"\"Main entry point for feature extraction.\"\"\"\n if use_tta:\n return extract_features_tta(model, dataset, image_size)\n else:\n return extract_features_single(model, dataset, image_size)","metadata":{"_uuid":"92c227ea-ca33-4ee9-aba3-122d523b2356","_cell_guid":"63d94f81-d29b-4154-bf0f-cd26a672044d","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:15:05.433653Z","iopub.execute_input":"2026-02-16T03:15:05.433923Z","iopub.status.idle":"2026-02-16T03:15:05.443395Z","shell.execute_reply.started":"2026-02-16T03:15:05.433900Z","shell.execute_reply":"2026-02-16T03:15:05.442825Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Multi-Model Ensemble","metadata":{"_uuid":"068c853b-d21d-4379-9250-efc3ac4848c6","_cell_guid":"aa39e847-f085-4b78-b734-3a9846bb4cf5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def extract_ensemble_features(sp_dataset, cfg):\n \"\"\"Extract and fuse features from multiple models.\"\"\"\n all_model_feats = []\n\n for model_type in cfg[\"models\"]:\n print(f\" šŸ“ Extracting with {model_type}...\")\n model = get_model(model_type)\n img_size = cfg[\"image_size\"][model_type]\n feats = extract_features(model, sp_dataset, img_size, use_tta=cfg.get(\"tta\", False))\n all_model_feats.append(feats)\n\n # Weighted fusion\n weights = cfg.get(\"model_weights\", [1.0 / len(cfg[\"models\"])] * len(cfg[\"models\"]))\n fused = np.zeros_like(all_model_feats[0])\n for w, feats in zip(weights, all_model_feats):\n # Project to common dimensionality if needed\n fused[:, :feats.shape[1]] += w * feats[:, :feats.shape[1]] if feats.shape[1] <= fused.shape[1] else 0\n\n # If dimensions differ, concatenate instead\n dims = [f.shape[1] for f in all_model_feats]\n if len(set(dims)) > 1:\n # Concatenate and re-normalize\n weighted_feats = [w * f for w, f in zip(weights, all_model_feats)]\n fused = np.concatenate(weighted_feats, axis=1)\n\n return normalize(fused, axis=1, norm=\"l2\")","metadata":{"_uuid":"893bc068-54b6-426a-b642-5a8a3d27edbb","_cell_guid":"a9b6d6cd-0577-4052-88dc-583be96f9752","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:15:09.474169Z","iopub.execute_input":"2026-02-16T03:15:09.474503Z","iopub.status.idle":"2026-02-16T03:15:09.481007Z","shell.execute_reply.started":"2026-02-16T03:15:09.474479Z","shell.execute_reply":"2026-02-16T03:15:09.480350Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Advanced Post-Processing","metadata":{"_uuid":"071db5b9-78e7-431f-a7d3-06ec3de42f45","_cell_guid":"a8975ee3-c743-4a76-a006-2e96d4ad873a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def query_expansion(features, k=5, alpha=0.5):\n \"\"\"Average Query Expansion (AQE).\"\"\"\n sim_matrix = np.dot(features, features.T)\n knn_indices = np.argsort(sim_matrix, axis=1)[:, -k:]\n expanded = np.zeros_like(features)\n for i in range(features.shape[0]):\n neighbor_feats = features[knn_indices[i]]\n neighbor_sims = sim_matrix[i, knn_indices[i]]\n # Weighted by similarity\n weights = np.expand_dims(neighbor_sims, 1)\n expanded[i] = features[i] + alpha * np.sum(weights * neighbor_feats, axis=0) / np.sum(weights)\n return normalize(expanded, axis=1, norm=\"l2\")\n\n\ndef iterative_query_expansion(features, k=5, alpha=0.5, iterations=2):\n \"\"\"Apply query expansion multiple times for refinement.\"\"\"\n feats = features.copy()\n for it in range(iterations):\n feats = query_expansion(feats, k=k, alpha=alpha * (0.8 ** it))\n return feats\n\n\ndef diffusion_ranking(features, k=10, alpha=0.99, n_iter=20):\n \"\"\"\n Feature diffusion on the k-NN graph for more robust similarities.\n \"\"\"\n n = features.shape[0]\n sim = np.dot(features, features.T)\n\n # Build sparse k-NN affinity\n W = np.zeros((n, n), dtype=np.float32)\n for i in range(n):\n idx = np.argsort(sim[i])[-k:]\n W[i, idx] = sim[i, idx]\n W[idx, i] = sim[i, idx]\n\n # Symmetric normalization\n D_inv_sqrt = np.diag(1.0 / np.sqrt(np.maximum(W.sum(axis=1), 1e-10)))\n S = D_inv_sqrt @ W @ D_inv_sqrt\n\n # Iterative diffusion\n F_mat = np.eye(n, dtype=np.float32)\n for _ in range(n_iter):\n F_mat = alpha * S @ F_mat + (1 - alpha) * np.eye(n)\n\n # Convert back to features\n U, s, Vt = np.linalg.svd(F_mat, full_matrices=False)\n dim = min(features.shape[1], n)\n diffused = U[:, :dim] * np.sqrt(s[:dim])\n return normalize(diffused, axis=1, norm=\"l2\")\n\n\ndef find_optimal_threshold(features, threshold_range, n_steps=20):\n \"\"\"\n Find optimal clustering threshold using silhouette score.\n \"\"\"\n dist_matrix = np.clip(1.0 - np.dot(features, features.T), 0, 2)\n low, high = threshold_range\n thresholds = np.linspace(low, high, n_steps)\n\n best_score = -1\n best_threshold = (low + high) / 2\n best_labels = None\n\n for thresh in thresholds:\n try:\n labels = AgglomerativeClustering(\n n_clusters=None,\n metric=\"precomputed\",\n linkage=\"average\",\n distance_threshold=thresh,\n ).fit_predict(dist_matrix)\n\n n_clusters = len(np.unique(labels))\n if n_clusters < 2 or n_clusters >= len(features) - 1:\n continue\n\n score = silhouette_score(dist_matrix, labels, metric=\"precomputed\")\n\n if score > best_score:\n best_score = score\n best_threshold = thresh\n best_labels = labels\n except Exception:\n continue\n\n print(f\" šŸŽÆ Optimal threshold: {best_threshold:.3f} (silhouette: {best_score:.3f})\")\n return best_threshold, best_labels\n\n\ndef merge_small_clusters(labels, features, min_size=2, merge_threshold=0.6):\n \"\"\"Merge very small clusters into their nearest larger cluster.\"\"\"\n unique, counts = np.unique(labels, return_counts=True)\n small_clusters = unique[counts < min_size]\n\n if len(small_clusters) == 0:\n return labels\n\n labels = labels.copy()\n large_clusters = unique[counts >= min_size]\n\n if len(large_clusters) == 0:\n return labels\n\n # Compute centroids of large clusters\n centroids = {}\n for c in large_clusters:\n mask = labels == c\n centroids[c] = normalize(features[mask].mean(axis=0, keepdims=True))[0]\n\n centroid_matrix = np.stack([centroids[c] for c in large_clusters])\n\n for sc in small_clusters:\n mask = labels == sc\n sc_centroid = normalize(features[mask].mean(axis=0, keepdims=True))[0]\n sims = np.dot(centroid_matrix, sc_centroid)\n best_idx = np.argmax(sims)\n\n if sims[best_idx] > merge_threshold:\n labels[mask] = large_clusters[best_idx]\n\n return labels","metadata":{"_uuid":"adb76164-09da-4bf9-ac87-7612a4c2687a","_cell_guid":"b33e6dd1-7f18-4e75-b30c-e5c99e936ee4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:15:11.473901Z","iopub.execute_input":"2026-02-16T03:15:11.474602Z","iopub.status.idle":"2026-02-16T03:15:11.489687Z","shell.execute_reply.started":"2026-02-16T03:15:11.474570Z","shell.execute_reply":"2026-02-16T03:15:11.488907Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Clustering Pipeline","metadata":{"_uuid":"030687f0-4699-4e46-bc64-28ec3cef4284","_cell_guid":"89675bf3-0d5c-4f9b-8243-4c5488d043a1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def cluster_and_format(features, image_ids, species, cfg):\n \"\"\"Full clustering pipeline with optimization.\"\"\"\n print(f\" -> Clustering {species}...\")\n\n # Step 1: Iterative Query Expansion\n feats = iterative_query_expansion(\n features,\n k=cfg[\"qe_k\"],\n alpha=cfg.get(\"qe_alpha\", 0.5),\n iterations=2\n )\n\n # Step 2: Diffusion (only for larger datasets to keep runtime manageable)\n if len(feats) <= 5000:\n print(f\" šŸŒŠ Applying diffusion ranking...\")\n feats = diffusion_ranking(feats, k=min(20, len(feats) - 1))\n\n # Step 3: Find optimal threshold\n threshold_range = cfg.get(\"threshold_range\", (0.25, 0.50))\n best_threshold, best_labels = find_optimal_threshold(feats, threshold_range, n_steps=30)\n\n if best_labels is None:\n # Fallback\n dist_matrix = np.clip(1.0 - np.dot(feats, feats.T), 0, 2)\n best_labels = AgglomerativeClustering(\n n_clusters=None,\n metric=\"precomputed\",\n linkage=\"average\",\n distance_threshold=np.mean(threshold_range),\n ).fit_predict(dist_matrix)\n\n # Step 4: Merge small clusters\n labels = merge_small_clusters(best_labels, feats, min_size=2, merge_threshold=0.55)\n\n n_individuals = len(np.unique(labels))\n cluster_sizes = np.bincount(labels)\n print(f\" āœ… Found {n_individuals} individuals \"\n f\"(min size: {cluster_sizes.min()}, max: {cluster_sizes.max()}, \"\n f\"median: {np.median(cluster_sizes):.0f})\")\n\n return pd.DataFrame({\n \"image_id\": image_ids,\n \"cluster\": [f\"cluster_{species}_{lbl}\" for lbl in labels],\n })","metadata":{"_uuid":"33f79ae1-961e-42b2-96b0-e3751de85f9c","_cell_guid":"2c4438a9-7002-4b34-ad34-6e8696c45216","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:15:15.175601Z","iopub.execute_input":"2026-02-16T03:15:15.176337Z","iopub.status.idle":"2026-02-16T03:15:15.182793Z","shell.execute_reply.started":"2026-02-16T03:15:15.176299Z","shell.execute_reply":"2026-02-16T03:15:15.182069Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Validation on Training Data (Optional)","metadata":{"_uuid":"73553d70-8860-4e04-a510-471cb2dda4a0","_cell_guid":"efc62bab-255f-4c27-b640-18f1a8ec74f7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def validate_on_train(full_dataset, species, cfg, max_samples=500):\n \"\"\"\n Quick validation on training split to sanity-check parameters.\n \"\"\"\n train_meta = full_dataset.metadata[\n (full_dataset.metadata[\"split\"] == \"train\") &\n (full_dataset.metadata[\"dataset\"] == species)\n ]\n\n if len(train_meta) == 0 or \"identity\" not in train_meta.columns:\n return\n\n # Sample if too large\n if len(train_meta) > max_samples:\n train_meta = train_meta.sample(max_samples, random_state=42)\n\n train_dataset = full_dataset.get_subset(train_meta.index.values)\n feats = extract_ensemble_features(train_dataset, cfg)\n feats = iterative_query_expansion(feats, k=cfg[\"qe_k\"], alpha=cfg.get(\"qe_alpha\", 0.5))\n\n dist_matrix = np.clip(1.0 - np.dot(feats, feats.T), 0, 2)\n true_labels = train_meta[\"identity\"].values\n\n # Try different thresholds\n best_ari = -1\n best_thresh = 0\n from sklearn.metrics import adjusted_rand_score\n\n for thresh in np.linspace(*cfg[\"threshold_range\"], 20):\n try:\n pred_labels = AgglomerativeClustering(\n n_clusters=None, metric=\"precomputed\",\n linkage=\"average\", distance_threshold=thresh,\n ).fit_predict(dist_matrix)\n ari = adjusted_rand_score(true_labels, pred_labels)\n if ari > best_ari:\n best_ari = ari\n best_thresh = thresh\n except Exception:\n continue\n\n print(f\" šŸ“Š Validation ARI={best_ari:.3f} at threshold={best_thresh:.3f}\")\n return best_thresh","metadata":{"_uuid":"77066a6a-db35-4442-93f6-f3cccb836a2a","_cell_guid":"d50024b9-6db5-4f8b-9040-6e1b4cb1f3d7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:15:17.730570Z","iopub.execute_input":"2026-02-16T03:15:17.731131Z","iopub.status.idle":"2026-02-16T03:15:17.737804Z","shell.execute_reply.started":"2026-02-16T03:15:17.731105Z","shell.execute_reply":"2026-02-16T03:15:17.737157Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Main Execution","metadata":{"_uuid":"e7ec295c-685c-4ed4-bcf2-d678fe4a51ad","_cell_guid":"38749495-0287-4efb-9ac1-cec9ae344fc2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"print(\"šŸ“‚ Loading Data...\")\nfull_dataset = AnimalCLEF2026(root=ROOT_DIR, transform=None, load_label=True)\ntest_meta = full_dataset.metadata[full_dataset.metadata[\"split\"] == \"test\"]\nresults = []\n\nprint(f\"šŸ“Š Test set: {len(test_meta)} images across {test_meta['dataset'].nunique()} species\\n\")\n\nfor species in test_meta[\"dataset\"].unique():\n print(f\"{'='*60}\")\n print(f\"šŸ¦ Processing: {species}\")\n cfg = SPECIES_CONFIG.get(species)\n\n if cfg is None:\n print(f\" āš ļø No config for {species}, using defaults\")\n cfg = {\n \"models\": [\"megadescriptor\"],\n \"model_weights\": [1.0],\n \"threshold_range\": (0.30, 0.50),\n \"image_size\": {\"megadescriptor\": 384, \"miewid\": 512},\n \"qe_k\": 5,\n \"qe_alpha\": 0.5,\n \"tta\": False,\n }\n\n # Subset\n sp_meta = test_meta[test_meta[\"dataset\"] == species]\n sp_dataset = full_dataset.get_subset(sp_meta.index.values)\n print(f\" šŸ“· {len(sp_meta)} test images\")\n\n # Optional: validate on training data\n # validate_on_train(full_dataset, species, cfg)\n\n # Extract ensemble features\n feats = extract_ensemble_features(sp_dataset, cfg)\n print(f\" šŸ“ Feature dim: {feats.shape[1]}\")\n\n # Cluster\n result = cluster_and_format(feats, sp_meta[\"image_id\"].values, species, cfg)\n results.append(result)\n\n del feats, sp_dataset\n gc.collect()\n torch.cuda.empty_cache()\n print()\n\n# Clear model cache\nfor model in _model_cache.values():\n del model\n_model_cache.clear()\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"_uuid":"708a4158-ca46-43cb-bf57-26cd4ec37a88","_cell_guid":"e60dbabd-923b-40f7-84b5-723f8207542c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:15:20.035015Z","iopub.execute_input":"2026-02-16T03:15:20.035325Z","iopub.status.idle":"2026-02-16T03:34:55.386665Z","shell.execute_reply.started":"2026-02-16T03:15:20.035299Z","shell.execute_reply":"2026-02-16T03:34:55.385770Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission","metadata":{"_uuid":"ff873b8e-3252-4b28-b494-863bc35f877d","_cell_guid":"e1acc06d-e246-430f-a0b2-37ef765d2fd6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"preds = pd.concat(results).reset_index(drop=True)\nsample_sub = pd.read_csv(os.path.join(ROOT_DIR, \"sample_submission.csv\"))\n\n# Map predictions\npred_map = dict(zip(preds[\"image_id\"], preds[\"cluster\"]))\nsample_sub[\"cluster\"] = sample_sub[\"image_id\"].map(pred_map).fillna(\"cluster_unknown_0\")\n\n# Sanity checks\nn_missing = sample_sub[\"cluster\"].str.contains(\"unknown\").sum()\nif n_missing > 0:\n print(f\"āš ļø {n_missing} images without predictions!\")\n\nn_clusters_total = sample_sub[\"cluster\"].nunique()\nprint(f\"\\nšŸ“Š Summary:\")\nprint(f\" Total images: {len(sample_sub)}\")\nprint(f\" Total clusters: {n_clusters_total}\")\n\nfor species in test_meta[\"dataset\"].unique():\n sp_mask = sample_sub[\"image_id\"].isin(\n test_meta[test_meta[\"dataset\"] == species][\"image_id\"]\n )\n n_sp = sample_sub.loc[sp_mask, \"cluster\"].nunique()\n print(f\" {species}: {sp_mask.sum()} images, {n_sp} individuals\")\n\nsample_sub.to_csv(\"submission.csv\", index=False)\nprint(\"\\nšŸ† 'submission.csv' generated!\")","metadata":{"_uuid":"eef62e87-9100-45f6-b46b-a1eaf7f20cda","_cell_guid":"47b0cef4-5a00-4aec-afc1-3a76bf8e53db","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:34:55.388394Z","iopub.execute_input":"2026-02-16T03:34:55.388999Z","iopub.status.idle":"2026-02-16T03:34:55.431731Z","shell.execute_reply.started":"2026-02-16T03:34:55.388972Z","shell.execute_reply":"2026-02-16T03:34:55.430967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv(\"/kaggle/working/submission.csv\")","metadata":{"_uuid":"964436ea-227c-40dd-abd8-3e871266f274","_cell_guid":"2916a7fa-1ee3-4a06-925d-34560fc172bb","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-02-16T03:34:55.432641Z","iopub.execute_input":"2026-02-16T03:34:55.432911Z","iopub.status.idle":"2026-02-16T03:34:55.459771Z","shell.execute_reply.started":"2026-02-16T03:34:55.432884Z","shell.execute_reply":"2026-02-16T03:34:55.459242Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}