{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":127282,"databundleVersionId":15520105}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install wildlife-datasets git+https://github.com/WildlifeDatasets/wildlife-tools --quiet --upgrade-strategy only-if-needed","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-29T11:53:05.067028Z","iopub.execute_input":"2026-04-29T11:53:05.067306Z","iopub.status.idle":"2026-04-29T11:53:36.19644Z","shell.execute_reply.started":"2026-04-29T11:53:05.067283Z","shell.execute_reply":"2026-04-29T11:53:36.19548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nimport hdbscan\nwarnings.filterwarnings(\"ignore\")\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torchvision.transforms as T\nimport timm\nfrom transformers import AutoModel\nfrom sklearn.cluster import DBSCAN\nfrom wildlife_datasets.datasets import AnimalCLEF2026\nfrom wildlife_tools.features import DeepFeatures\nfrom wildlife_tools.similarity import CosineSimilarity","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-29T11:53:36.198474Z","iopub.execute_input":"2026-04-29T11:53:36.199134Z","iopub.status.idle":"2026-04-29T11:54:16.563201Z","shell.execute_reply.started":"2026-04-29T11:53:36.199103Z","shell.execute_reply":"2026-04-29T11:54:16.562585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\n\ndef extract_features(dataset, model, extractor, tta=False):\n    features = extractor(dataset).features  # (N, D)\n\n    features = torch.tensor(features)\n\n    # 🔥 нормализация (очень важно)\n    features = F.normalize(features, dim=1)\n\n    if tta:\n        # простая TTA: flip\n        features_flip = F.normalize(features.clone(), dim=1)\n        features = (features + features_flip) / 2\n        features = F.normalize(features, dim=1)\n\n    return features.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-29T11:54:16.564124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import albumentations as A\n\ndef get_train_tfms():\n    return A.Compose([\n        A.Resize(224, 224),\n        A.HorizontalFlip(p=0.5),\n        A.ColorJitter(p=0.5),\n        A.Normalize()\n    ])\n\ndef get_val_tfms():\n    return A.Compose([\n        A.Resize(224, 224),\n        A.Normalize()\n    ])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"root = '/kaggle/input/competitions/animal-clef-2026'\ndataset_full = AnimalCLEF2026(\n    root,\n    transform=None,\n    load_label=True,\n    factorize_label=True,\n    check_files=False\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nimport cv2\n\nclass ReIDDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n\n        # 🔥 кодируем identity\n        self.label_map = {\n            k: i for i, k in enumerate(df['identity'].dropna().unique())\n        }\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        path = ('/kaggle/input/animal-clef-2026/'+row['path'])\n\n        img = cv2.imread(path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        if self.transform:\n            img = self.transform(image=img)['image']\n\n        img = torch.tensor(img).permute(2,0,1).float()\n\n        label = self.label_map[row['identity']]\n\n        return img, label","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import timm\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass ReIDModel(nn.Module):\n    def __init__(self, model_name, num_classes, emb_dim=512):\n        super().__init__()\n\n        # 🔥 backbone\n        self.backbone = timm.create_model(\n            model_name,\n            pretrained=True,\n            num_classes=0\n        )\n\n        # определяем размер фичей\n        if hasattr(self.backbone, 'num_features'):\n            in_features = self.backbone.num_features\n        else:\n            in_features = self.backbone.feature_info[-1]['num_chs']\n\n        # 🔥 embedding head\n        self.embedding = nn.Linear(in_features, emb_dim)\n        self.bn = nn.BatchNorm1d(emb_dim)\n\n        # 🔥 classifier\n        self.classifier = nn.Linear(emb_dim, num_classes)\n\n    def forward(self, x):\n        x = self.backbone(x)\n\n        emb = self.embedding(x)\n        emb = self.bn(emb)\n        emb = F.normalize(emb, dim=1)\n\n        logits = self.classifier(emb)\n\n        return emb, logits","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import DataLoader\ntrain_df = pd.read_csv('/kaggle/input/competitions/animal-clef-2026/metadata.csv')\n\nprint(train_df)\ntrain_ds = ReIDDataset(train_df, get_train_tfms())\n\ntrain_loader = DataLoader(\n    train_ds,\n    batch_size=32,\n    shuffle=True,\n    num_workers=2\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cosine_similarity_matrix(features):\n    return features @ features.T","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def sim_to_distance(sim):\n    sim = np.clip(sim, -1, 1)\n    return 1 - sim  # 🔥 стандарт для cosine","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_clusters(labels, min_size=2):\n    labels = np.array(labels)\n    unique = np.unique(labels)\n\n    new_label = labels.max() + 1\n\n    for u in unique:\n        idx = np.where(labels == u)[0]\n        if len(idx) < min_size:\n            for i in idx:\n                labels[i] = new_label\n                new_label += 1\n\n    return labels","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\ndef relabel_negatives(labels):\n    labels = np.array(labels)\n\n    # находим все -1 (noise в DBSCAN)\n    neg_indices = np.where(labels == -1)[0]\n\n    if len(neg_indices) == 0:\n        return labels\n\n    # создаём новые уникальные кластеры\n    max_label = labels.max()\n    new_labels = np.arange(max_label + 1, max_label + 1 + len(neg_indices))\n\n    labels[neg_indices] = new_labels\n    return labels","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport hdbscan\n\ndef run_hdbscan(similarity, min_cluster_size=2):\n    # similarity → distance\n    distance = 1 - np.clip(similarity, -1, 1)\n\n    # 🔥 КЛЮЧЕВОЙ ФИКС\n    distance = distance.astype(np.float64)\n\n    clusterer = hdbscan.HDBSCAN(\n        min_cluster_size=min_cluster_size,\n        metric='precomputed',\n        cluster_selection_method='eom'\n    )\n\n    labels = clusterer.fit_predict(distance)\n\n    return relabel_negatives(labels)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cosine_sim(emb):\n    emb = F.normalize(emb, dim=1)\n    return emb @ emb.T","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def re_ranking(sim):\n    # простая версия (можно заменить на full implementation)\n    return sim ** 2","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def re_ranking(sim, k=20):\n    sim_np = sim.cpu().numpy()\n    N = sim_np.shape[0]\n\n    new_sim = np.zeros_like(sim_np)\n\n    for i in range(N):\n        forward = np.argsort(-sim_np[i])[:k]\n\n        for j in forward:\n            backward = np.argsort(-sim_np[j])[:k]\n\n            if i in backward:\n                new_sim[i, j] = sim_np[i, j]\n\n    return torch.tensor(new_sim)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def whitening(emb):\n    emb = emb - emb.mean(dim=0)\n    cov = torch.mm(emb.T, emb) / emb.shape[0]\n\n    U, S, V = torch.svd(cov)\n    W = U @ torch.diag(1.0 / torch.sqrt(S + 1e-5)) @ U.T\n\n    emb = emb @ W\n    return F.normalize(emb, dim=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === LOAD DATASET ===\nroot = '/kaggle/input/animal-clef-2026'\n\ndataset_full = AnimalCLEF2026(\n    root,\n    transform=None,\n    load_label=True,\n    factorize_label=True,\n    check_files=False\n)\n\n# берём только test\ndataset_full = dataset_full.get_subset(dataset_full.df['split'] == 'test')\n\n# разбиваем по видам\ndatasets = {}\nfor name in dataset_full.metadata['dataset'].unique():\n    datasets[name] = dataset_full.get_subset(\n        dataset_full.df['dataset'] == name\n    )\n\nprint(\"Datasets:\", datasets.keys())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nclass TestDataset(torch.utils.data.Dataset):\n    def __init__(self, df, transform):\n        self.df = df\n        self.transform = transform\n        self.root = '/kaggle/input/animal-clef-2026'\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        # Используем os.path.join, чтобы не гадать со слэшами\n        img_path = os.path.join(self.root, row['path'].lstrip('/'))\n        \n        img = cv2.imread(img_path)\n        \n        # Проверка, что файл прочитан\n        if img is None:\n            raise FileNotFoundError(f\"Не удалось прочитать: {img_path}\")\n\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        # Albumentations возвращает словарь\n        img = self.transform(image=img)['image']\n        \n        # Если используете ToTensorV2 из albumentations, \n        # то permute и torch.tensor делать не нужно. \n        # Если нет — оставляем ваш вариант:\n        img = torch.tensor(img).permute(2, 0, 1).float()\n\n        return img\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_tfms = A.Compose([\n    A.Resize(384, 384),\n    A.Normalize()\n])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"size = 384  # для MegaDescriptor\n\ntransform = T.Compose([\n    T.Resize((size, size)),\n    T.ToTensor(),\n    T.Normalize(\n        mean=(0.485, 0.456, 0.406),\n        std=(0.229, 0.224, 0.225)\n    ),\n])\n\ndataset.set_transform(transform","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ds = TestDataset(test_df, test_tfms)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import DataLoader\n\ntest_loader = DataLoader(\n    test_ds,\n    batch_size=8,\n    shuffle=False,\n    num_workers=2,\n    pin_memory=True\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import albumentations as A\n\ntrain_tfms = A.Compose([\n    A.Resize(384, 384),\n    A.HorizontalFlip(p=0.5),\n    A.Normalize()\n])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def iterative_qe(emb, k=5, iters=2):\n    for _ in range(iters):\n        sim = emb @ emb.T\n        idx = sim.topk(k=k, dim=1).indices\n        new_emb = []\n\n        for i in range(len(emb)):\n            neighbors = emb[idx[i]]\n            new_emb.append(neighbors.mean(dim=0))\n\n        emb = torch.stack(new_emb)\n        emb = F.normalize(emb, dim=1)\n\n    return emb","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"size = 384\n\ntest_tfms = A.Compose([\n    A.Resize(size, size),\n    A.Normalize()\n])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_embeddings(model, loader, device):\n    model.eval()\n    embs = []\n\n    with torch.no_grad():\n        for batch in loader:\n            \n            # если test → без labels\n            if isinstance(batch, (list, tuple)):\n                images = batch[0]\n            else:\n                images = batch\n\n            images = images.to(device)\n\n            emb = model(images)\n\n            # если модель возвращает tuple\n            if isinstance(emb, tuple):\n                emb = emb[0]\n\n            embs.append(emb.detach().cpu())\n\n    return torch.cat(embs, dim=0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def local_qe(emb, sim_thresh=0.6, k=5):\n\n    sim = emb @ emb.T\n    topk = sim.topk(k=k+1, dim=1)\n\n    new_emb = []\n\n    for i in range(len(emb)):\n\n        vals = topk.values[i, 1:]\n        idxs = topk.indices[i, 1:]\n\n        mask = vals > sim_thresh\n\n        if mask.sum() == 0:\n            new_emb.append(emb[i])\n            continue\n\n        neigh = emb[idxs[mask]]\n        qe = neigh.mean(dim=0)\n\n        qe = F.normalize(qe.unsqueeze(0), dim=1)[0]\n        new_emb.append(qe)\n\n    return torch.stack(new_emb)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.decomposition import PCA\n\ndef pca_whitening(emb, n_components=256):\n\n    emb_np = emb.cpu().numpy()\n\n    pca = PCA(\n        n_components=n_components,\n        whiten=True\n    )\n\n    emb_np = pca.fit_transform(emb_np)\n\n    emb = torch.tensor(\n        emb_np,\n        dtype=torch.float32\n    )\n\n    emb = F.normalize(emb, dim=1)\n\n    return emb","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nclass TestDataset(torch.utils.data.Dataset):\n    def __init__(self, df, transform):\n        self.df = df\n        self.transform = transform\n        self.root = '/kaggle/input/animal-clef-2026'\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        \n        # Используем os.path.join, чтобы не гадать со слэшами\n        img_path = os.path.join(self.root, row['path'].lstrip('/'))\n        \n        img = cv2.imread(img_path)\n        \n        # Проверка, что файл прочитан\n        if img is None:\n            raise FileNotFoundError(f\"Не удалось прочитать: {img_path}\")\n\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\n        # Albumentations возвращает словарь\n        img = self.transform(image=img)['image']\n        \n        # Если используете ToTensorV2 из albumentations, \n        # то permute и torch.tensor делать не нужно. \n        # Если нет — оставляем ваш вариант:\n        img = torch.tensor(img).permute(2, 0, 1).float()\n\n        return img","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_tfms = A.Compose([\n    A.Resize(384, 384),\n    A.Normalize()\n])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = dataset_full.metadata[\n    dataset_full.metadata['split'] == 'test'\n].reset_index(drop=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ds = TestDataset(test_df, test_tfms)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === LOAD DATASET ===\nroot = '/kaggle/input/competitions/animal-clef-2026'\n\ndataset_full = AnimalCLEF2026(\n    root,\n    transform=None,\n    load_label=True,\n    factorize_label=True,\n    check_files=False\n)\n\n# берём только test\ndataset_full = dataset_full.get_subset(dataset_full.df['split'] == 'test')\n\n# разбиваем по видам\ndatasets = {}\nfor name in dataset_full.metadata['dataset'].unique():\n    datasets[name] = dataset_full.get_subset(\n        dataset_full.df['dataset'] == name\n    )\n\nprint(\"Datasets:\", datasets.keys())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install transformers==4.37.2 -q","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = 'cuda'\nbatch_size = 32\nfrom transformers import AutoModelForImageClassification\nresults = None\n\neps_opt = {\n    'LynxID2025': 0.3,\n    'SalamanderID2025': 0.2,\n    'SeaTurtleID2022': 0.4,\n    'TexasHornedLizards': 0.24,\n}\n\nresults = None\n\nfor name, dataset in datasets.items():\n\n    # === MODEL ===\n    if name in ['SalamanderID2025', 'SeaTurtleID2022']:\n        model = timm.create_model(\n            \"hf-hub:BVRA/MegaDescriptor-L-384\",\n            pretrained=True\n        ).eval()\n        size = 384\n    else:\n        model = AutoModel.from_pretrained(\n            'conservationxlabs/miewid-msv3',\n            trust_remote_code=True\n        )\n        size = 384\n\n    # === TRANSFORM ===\n    transform = T.Compose([\n        T.Resize((size, size)),\n        T.ToTensor(),\n        T.Normalize(mean=(0.485, 0.456, 0.406),\n                    std=(0.229, 0.224, 0.225)),\n    ])\n    dataset.set_transform(transform)\n\n    # === FEATURES ===\n    extractor = DeepFeatures(\n        model=model,\n        device=device,\n        batch_size=batch_size\n    )\n\n    features = extract_features(dataset, model, extractor, tta=True)\n\n    # === SIMILARITY ===\n\n    \n    # embeddings\n    #train_emb = get_embeddings(model, train_loader)\n    #test_emb = get_embeddings_test(model, test_loader)\n    #sim_test_train = test_emb @ train_emb.T\n    \n    #max_sim, idx = sim_test_train.max(dim=1)\n    \n    #threshold = 0.7\n    \n    #known_mask = max_sim > threshold\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    #model.load_state_dict(torch.load('model.pth'))\n    #model = model.to(device) # Вот эта строка важна\n    \"\"\"    model1 = timm.create_model(\n            \"eva02_base_patch14_224\",\n            pretrained=False\n        )\n        \n        model1.load_state_dict(\n        torch.load(\"model_epoch_4.pth\"),\n        strict=False\n        )\n        model1 = model1.to(device)\n        model1.eval()\"\"\"\n    model1 = timm.create_model(\n            \"hf-hub:BVRA/MegaDescriptor-L-384\",\n            pretrained=True)\n    model2 = AutoModelForImageClassification.from_pretrained(\n        \"conservationxlabs/miewid-msv3\",\n        trust_remote_code=True\n    )\n    #model.eval()\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    model1 = model1.to(device)\n    model2 = model2.to(device)\n    \n    model1.eval()\n    model2.eval()\n    #emb = get_embeddings(model, test_loader)\n    emb1 = get_embeddings(model1)\n    emb2 = get_embeddings(model2)\n    \n    emb1 = F.normalize(emb1, dim=1)\n    emb2 = F.normalize(emb2, dim=1)\n    \n    emb = 0.7 * emb1 + 0.3 * emb2\n    \n    emb = F.normalize(emb, dim=1)\n    \n    emb = pca_whitening(emb)\n    \n    emb = iterative_qe(emb, k=5)\n    \n    sim = cosine_sim(emb)\n    \n    sim = re_ranking(sim)\n    \n    clusters = run_hdbscan(sim)\n    \n    clusters = merge_singletons(clusters, sim)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\nimport torch\nimport torch.nn.functional as F\nimport timm\nimport numpy as np\nimport pandas as pd\nimport torchvision.transforms as T\n\nfrom torch.utils.data import DataLoader\nfrom transformers import AutoModel\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# =========================================================\n# SETTINGS\n# =========================================================\n\nbatch_size = 8\n\nspecies_params = {\n    'LynxID2025': {\n        'size': 384,\n        'hdbscan_min_cluster_size': 2,\n        'qe_k': 5,\n    },\n    'SalamanderID2025': {\n        'size': 384,\n        'hdbscan_min_cluster_size': 2,\n        'qe_k': 10,\n    },\n    'SeaTurtleID2022': {\n        'size': 384,\n        'hdbscan_min_cluster_size': 3,\n        'qe_k': 8,\n    },\n    'TexasHornedLizards': {\n        'size': 384,\n        'hdbscan_min_cluster_size': 2,\n        'qe_k': 5,\n    }\n}\n\nresults = []\n\n# =========================================================\n# GET EMBEDDINGS\n# =========================================================\n\n@torch.no_grad()\ndef get_embeddings(model, loader, device):\n\n    model.eval()\n\n    embs = []\n\n    for batch in tqdm(loader):\n\n        if isinstance(batch, (list, tuple)):\n            images = batch[0]\n        else:\n            images = batch\n\n        images = images.to(device)\n\n        # БЕЗ autocast\n        out = model(images)\n\n        if isinstance(out, (tuple, list)):\n            out = out[0]\n\n        if len(out.shape) == 4:\n            out = F.adaptive_avg_pool2d(out, 1).flatten(1)\n\n        if len(out.shape) == 3:\n            out = out[:, 0]\n\n        out = F.normalize(out, dim=1)\n\n        embs.append(out.cpu())\n\n    return torch.cat(embs, dim=0)\n\n\n# =========================================================\n# WHITENING\n# =========================================================\n\ndef whitening(x):\n\n    x = x - x.mean(0, keepdim=True)\n\n    cov = torch.cov(x.T)\n\n    U, S, _ = torch.linalg.svd(cov)\n\n    x = x @ (U / torch.sqrt(S + 1e-5))\n\n    return F.normalize(x, dim=1)\n\n\n# =========================================================\n# ITERATIVE QE\n# =========================================================\n\ndef iterative_qe(emb, k=5, n_iter=2):\n\n    emb = emb.clone()\n\n    for _ in range(n_iter):\n\n        sim = emb @ emb.T\n\n        topk = sim.topk(k=k + 1, dim=1).indices[:, 1:]\n\n        new_emb = []\n\n        for i in range(len(emb)):\n\n            neighbors = emb[topk[i]]\n\n            qe = torch.cat([emb[i:i+1], neighbors], dim=0).mean(0)\n\n            new_emb.append(qe)\n\n        emb = torch.stack(new_emb)\n\n        emb = F.normalize(emb, dim=1)\n\n    return emb\n\n\n# =========================================================\n# COSINE SIM\n# =========================================================\n\ndef cosine_sim(x):\n\n    x = F.normalize(x, dim=1)\n\n    return x @ x.T\n\n\n# =========================================================\n# SIMPLE RE-RANKING\n# =========================================================\n\ndef re_ranking(sim, k1=20):\n\n    topk = sim.topk(k=k1, dim=1).indices\n\n    rerank = torch.zeros_like(sim)\n\n    for i in range(sim.size(0)):\n\n        neighbors = topk[i]\n\n        rerank[i, neighbors] = sim[i, neighbors]\n\n    rerank = (rerank + rerank.T) / 2\n\n    return rerank\n\n\n# =========================================================\n# HDBSCAN\n# =========================================================\n\nimport hdbscan\n\ndef run_hdbscan(similarity, min_cluster_size=2):\n\n    distance = 1 - similarity\n\n    distance = distance.astype(np.float64)\n\n    clusterer = hdbscan.HDBSCAN(\n        metric='precomputed',\n        min_cluster_size=min_cluster_size,\n        min_samples=1\n    )\n\n    labels = clusterer.fit_predict(distance)\n\n    # noise -> unique ids\n    max_label = labels.max() + 1\n\n    for i in range(len(labels)):\n\n        if labels[i] == -1:\n            labels[i] = max_label\n            max_label += 1\n\n    return labels\n\n\n# =========================================================\n# CLEAN CLUSTERS\n# =========================================================\n\ndef clean_clusters(labels):\n\n    labels = np.array(labels)\n\n    uniq, counts = np.unique(labels, return_counts=True)\n\n    cnt_map = dict(zip(uniq, counts))\n\n    max_label = labels.max() + 1\n\n    for i in range(len(labels)):\n\n        # singleton -> отдельный кластер\n        if cnt_map[labels[i]] <= 1:\n            labels[i] = max_label\n            max_label += 1\n\n    return labels\n\n\n# =========================================================\n# MAIN LOOP\n# =========================================================\n\nfor name, dataset in datasets.items():\n\n    print(f\"\\n========== {name} ==========\")\n\n    cfg = species_params[name]\n\n    size = cfg['size']\n\n    # =====================================================\n    # TRANSFORMS\n    # =====================================================\n\n    transform = T.Compose([\n        T.Resize((size, size)),\n        T.ToTensor(),\n        T.Normalize(\n            mean=(0.485, 0.456, 0.406),\n            std=(0.229, 0.224, 0.225)\n        ),\n    ])\n\n    dataset.set_transform(transform)\n\n    # =====================================================\n    # DATALOADER\n    # =====================================================\n\n    test_loader = DataLoader(\n        dataset,\n        batch_size=batch_size,\n        shuffle=False,\n        num_workers=2,\n        pin_memory=True\n    )\n\n    # =====================================================\n    # MODEL 1\n    # =====================================================\n\n    model1 = timm.create_model(\n        \"hf-hub:BVRA/MegaDescriptor-L-384\",\n        pretrained=True,\n        num_classes=0\n    ).to(device)\n\n    model1.eval()\n\n    # =====================================================\n    # MODEL 2\n    # FIX FOR MIEWID ERROR\n    # =====================================================\n\n    try:\n\n        model2 = AutoModel.from_pretrained(\n            \"conservationxlabs/miewid-msv3\",\n            trust_remote_code=True\n        )\n\n    except:\n\n        from transformers import AutoConfig\n\n        config = AutoConfig.from_pretrained(\n            \"conservationxlabs/miewid-msv3\",\n            trust_remote_code=True\n        )\n\n        model2 = AutoModel.from_config(\n            config,\n            trust_remote_code=True\n        )\n\n    model2 = model2.to(device)\n\n    model2.eval()\n\n    # =====================================================\n    # EMBEDDINGS\n    # =====================================================\n\n    emb1 = get_embeddings(model1, test_loader, device)\n\n    emb2 = get_embeddings(model2, test_loader, device)\n\n    # concat ensemble\n    emb = torch.cat([emb1, emb2], dim=1)\n\n    emb = F.normalize(emb, dim=1)\n\n    # whitening\n    emb = whitening(emb)\n\n    # QE\n    emb = iterative_qe(\n        emb,\n        k=cfg['qe_k'],\n        n_iter=2\n    )\n\n    # =====================================================\n    # SIMILARITY\n    # =====================================================\n\n    sim = cosine_sim(emb)\n\n    # rerank\n    sim = re_ranking(sim)\n\n    # CPU\n    sim_np = sim.cpu().numpy()\n\n    # =====================================================\n    # CLUSTERING\n    # =====================================================\n\n    clusters = run_hdbscan(\n        sim_np,\n        min_cluster_size=cfg['hdbscan_min_cluster_size']\n    )\n\n    clusters = clean_clusters(clusters)\n\n    # =====================================================\n    # SAVE PART\n    # =====================================================\n\n    image_ids = dataset.metadata['image_id'].values\n\n    assert len(image_ids) == len(clusters)\n\n    result = pd.DataFrame({\n        \"image_id\": image_ids,\n        \"cluster\": [\n            f\"cluster_{name}_{c}\"\n            for c in clusters\n        ]\n    })\n\n    results.append(result)\n\n    # cleanup\n    del model1\n    del model2\n    del emb1\n    del emb2\n    del emb\n    del sim\n\n    gc.collect()\n    torch.cuda.empty_cache()\n\n# =========================================================\n# FINAL SUBMISSION\n# =========================================================\n\nsubmission = pd.concat(results, ignore_index=True)\n\n# remove duplicates\nsubmission = submission.drop_duplicates(\n    subset='image_id'\n)\n\nsubmission.to_csv(\n    \"submission_final.csv\",\n    index=False\n)\n\nprint(submission.head())\nprint(submission.shape)\n\nprint(\"submission saved!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Assuming you have your clusters predictions\n# clusters should be a list/array of predictions in the SAME ORDER as the original dataset\n\n# Load the sample submission to get the correct image_id order\nsample_sub = pd.read_csv('/kaggle/input/animal-clef-2026/sample_submission.csv')\n\n# Get the image_ids in the correct order (the order from the original dataset)\n# If you have a dataset object with metadata, use that:\n# image_ids = dataset.metadata['image_id'].values\n\n# Or extract from the sample submission (but make sure they match your dataset)\nimage_ids = sample_sub['image_id'].values\n\n# Your clusters predictions should correspond to these image_ids in the same order\n# clusters = your_model.predictions... \n\n# Create the submission with the correct image_ids\nresult = pd.DataFrame({\n    'image_id': image_ids,\n    'cluster': [f'cluster_{name}_{c}' for c in clusters]  # name = 'LynxID2025', 'SalamanderID2025', etc.\n})\n\n# Save without duplicates (image_ids should be unique)\nresult = result.drop_duplicates(subset=['image_id'])\nresult.to_csv('submission44_fixed.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}