{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15321952,"datasetId":9799849,"databundleVersionId":16228295},{"sourceType":"datasetVersion","sourceId":15324423,"datasetId":9801343,"databundleVersionId":16231129},{"sourceType":"datasetVersion","sourceId":15332860,"datasetId":9779459,"databundleVersionId":16240644},{"sourceType":"modelInstanceVersion","sourceId":781483,"databundleVersionId":16012874,"modelInstanceId":545530,"modelId":558539}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install gdown\n# !pip install sacrebleu","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:45:22.304696Z","iopub.execute_input":"2026-03-22T21:45:22.305281Z","iopub.status.idle":"2026-03-22T21:45:22.30867Z","shell.execute_reply.started":"2026-03-22T21:45:22.305251Z","shell.execute_reply":"2026-03-22T21:45:22.307836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import gdown\nimport pandas as pd\nimport numpy as np\nimport torch\n# import matplotlib.pyplot as plt\n# import seaborn as sns\nimport unicodedata\nimport re\n\n# import sacrebleu\nimport math\n\nfrom datasets import load_dataset\n\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer, Seq2SeqTrainer, Seq2SeqTrainingArguments\nfrom torch.cuda import temperature","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:45:22.309862Z","iopub.execute_input":"2026-03-22T21:45:22.310147Z","iopub.status.idle":"2026-03-22T21:45:35.003826Z","shell.execute_reply.started":"2026-03-22T21:45:22.310125Z","shell.execute_reply":"2026-03-22T21:45:35.003207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# file_id = '1qdSb3fLeI8FsXoZbiKpxKCFA_fE-89sG'\n# gdown.download(id=file_id, output='model.zip')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:45:35.004602Z","iopub.execute_input":"2026-03-22T21:45:35.00513Z","iopub.status.idle":"2026-03-22T21:45:35.008595Z","shell.execute_reply.started":"2026-03-22T21:45:35.005106Z","shell.execute_reply":"2026-03-22T21:45:35.007781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model_path = \"/kaggle/input/datasets/vitalykudelya/dpc26-models-2/LARGE_MIXES_WITH_OLD_NEW_DATA_v2_53.93/LARGE_MIXES_WITH_OLD_NEW_DATA_v2\"\ntest_set_path = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\ndevice = device = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:45:35.01083Z","iopub.execute_input":"2026-03-22T21:45:35.01116Z","iopub.status.idle":"2026-03-22T21:45:35.027734Z","shell.execute_reply.started":"2026-03-22T21:45:35.011133Z","shell.execute_reply":"2026-03-22T21:45:35.026963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scribal_characters = [\"?\", \"!\", \":\", \"/\"]\nparentheses_pattern = r'\\(.*?\\)'\nangle_bracket_pattern = r'<.*?>'\nsquare_bracket_pattern = r'\\[.*?\\]'\npartial_broken_signs_pattern = r'\\˹.*?\\˺'\nscribal_notation_patterns = [parentheses_pattern, angle_bracket_pattern, square_bracket_pattern, partial_broken_signs_pattern]\nsubscript_to_normal_map = {\n    '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n    '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9',\n    '⁰': '0', '¹': '1', '²': '2', '³': '3', '⁴': '4',\n    '⁵': '5', '⁶': '6', '⁷': '7', '⁸': '8', '⁹': '9'\n}\ntranslit_character_map = {'Ţ': 'Ṭ', 'Ḫ': 'H', 'ḫ': 'h', 'ḥ':\n                          'h', 'ṇ': 'n', 'ț': 'ṭ', 'ţ': 'ṭ',\n                          'ş': 'ṣ', 'ș': 'ṣ', 'Ş': 'Ṣ',\n                          'Ī': 'I', 'İ': 'I', 'Ü': 'Ù'}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:45:35.028618Z","iopub.execute_input":"2026-03-22T21:45:35.028875Z","iopub.status.idle":"2026-03-22T21:45:35.039902Z","shell.execute_reply.started":"2026-03-22T21:45:35.028839Z","shell.execute_reply":"2026-03-22T21:45:35.039357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nEnsemble MBR Pipeline\n=====================\nЭтапы:\n  1. Определение моделей (ModelConfig)\n  2. Генерация кандидатов от каждой модели последовательно\n  3. Сбор кандидатов в DataFrame\n  4. MBR decoding — выбор лучшего кандидата\n  5. (Опционально) Оценка на валидации\n\"\"\"\n\nimport gc\nimport math\nfrom dataclasses import dataclass, field\nfrom typing import List, Optional, Dict\n\nimport pandas as pd\nimport numpy as np\nimport sacrebleu\nimport torch\nfrom tqdm import tqdm\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n\n\n# ============================================================\n# Этап 0: Конфигурация моделей\n# ============================================================\n\n@dataclass\nclass ModelConfig:\n    \"\"\"Описание одной модели для ensemble\"\"\"\n    name: str                          # Человекочитаемое имя\n    model_path: str                    # Путь к весам\n    tokenizer_path: Optional[str] = None  # Путь к токенайзеру (если отличается)\n    \n    # Generation параметры\n    num_beams: int = 12\n    num_return: int = 5\n    max_input_length: int = 600\n    max_output_length: int = 690\n    length_penalty: float = 1.7\n    \n    # Batch\n    batch_size: int = 5\n    \n    # Доп. параметры загрузки\n    torch_dtype: Optional[torch.dtype] = None  # например torch.bfloat16\n    local_files_only: bool = True\n    \n    def __post_init__(self):\n        if self.tokenizer_path is None:\n            self.tokenizer_path = self.model_path\n\n\n# ============================================================\n# Этап 1: Генерация кандидатов от одной модели\n# ============================================================\n\ndef generate_candidates_single_model(\n    config: ModelConfig,\n    texts: List[str],\n    device: str = \"cuda\",\n) -> List[List[str]]:\n    \"\"\"\n    Загружает модель, генерирует кандидатов, освобождает память.\n    \n    Returns:\n        List[List[str]]: для каждого текста — список из num_return кандидатов\n    \"\"\"\n    print(f\"\\n{'='*60}\")\n    print(f\"[{config.name}] Loading model from {config.model_path}\")\n    print(f\"  beams={config.num_beams}, return={config.num_return}, \"\n          f\"lp={config.length_penalty}, batch={config.batch_size}\")\n    print(f\"{'='*60}\")\n    \n    # Загрузка\n    load_kwargs = {\"local_files_only\": config.local_files_only}\n    if config.torch_dtype is not None:\n        load_kwargs[\"torch_dtype\"] = config.torch_dtype\n    \n    model = AutoModelForSeq2SeqLM.from_pretrained(config.model_path, **load_kwargs)\n    model.to(device)\n    model.eval()\n    \n    tokenizer = AutoTokenizer.from_pretrained(\n        config.tokenizer_path, local_files_only=config.local_files_only\n    )\n    \n    # Генерация\n    all_candidates = []\n    n_batches = math.ceil(len(texts) / config.batch_size)\n    \n    for i in tqdm(range(0, len(texts), config.batch_size),\n                  total=n_batches, desc=f\"[{config.name}] Generating\"):\n        batch_texts = texts[i : i + config.batch_size]\n        \n        inputs = tokenizer(\n            batch_texts,\n            return_tensors=\"pt\",\n            max_length=config.max_input_length,\n            padding=True,\n            truncation=True,\n        ).to(device)\n        \n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                num_beams=config.num_beams,\n                num_return_sequences=config.num_return,\n                max_length=config.max_output_length,\n                length_penalty=config.length_penalty,\n                early_stopping=True,\n            )\n        \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        \n        # Разбить по примерам: [batch_size * num_return] → batch_size × num_return\n        for j in range(len(batch_texts)):\n            start = j * config.num_return\n            end = start + config.num_return\n            cands = [d.strip() for d in decoded[start:end]]\n            all_candidates.append(cands)\n    \n    # Освобождение памяти\n    del model, tokenizer\n    gc.collect()\n    torch.cuda.empty_cache()\n    \n    mem_free = torch.cuda.get_device_properties(0).total_memory - torch.cuda.memory_allocated()\n    print(f\"[{config.name}] Done. Generated {len(all_candidates)} × {config.num_return} candidates. \"\n          f\"VRAM free: {mem_free / 1e9:.1f} GB\")\n    \n    return all_candidates\n\n\n# ============================================================\n# Этап 2: Прогон всех моделей, сбор кандидатов в DataFrame\n# ============================================================\n\ndef generate_all_candidates(\n    model_configs: List[ModelConfig],\n    texts: List[str],\n    device: str = \"cuda\",\n) -> pd.DataFrame:\n    \"\"\"\n    Последовательно прогоняет все модели.\n    \n    Returns:\n        DataFrame с колонками:\n          - text_idx: индекс текста\n          - input_text: исходный текст\n          - {model_name}_0, {model_name}_1, ...: кандидаты от каждой модели\n    \"\"\"\n    n = len(texts)\n    \n    # Базовый DataFrame\n    df = pd.DataFrame({\n        \"text_idx\": range(n),\n        \"input_text\": texts,\n    })\n    \n    # Прогон каждой модели\n    for config in model_configs:\n        candidates = generate_candidates_single_model(config, texts, device)\n        \n        # Добавить колонки: model_name_0, model_name_1, ...\n        for k in range(config.num_return):\n            col_name = f\"{config.name}_{k}\"\n            df[col_name] = [candidates[i][k] if k < len(candidates[i]) else \"\" \n                           for i in range(n)]\n    \n    return df\n\n\ndef get_candidate_columns(df: pd.DataFrame) -> List[str]:\n    \"\"\"Вернуть имена колонок с кандидатами\"\"\"\n    return [c for c in df.columns if c not in (\"text_idx\", \"input_text\")]\n\n\ndef get_candidates_for_row(row: pd.Series, candidate_cols: List[str]) -> List[str]:\n    \"\"\"Извлечь всех кандидатов для одного примера\"\"\"\n    return [row[c] for c in candidate_cols if row[c] and str(row[c]).strip()]\n\n\n# ============================================================\n# Этап 3: MBR Decoding\n# ============================================================\n\ndef mbr_decode_chrf(candidates: List[str]) -> str:\n    \"\"\"MBR по chrF++ (быстрее, стабильнее)\"\"\"\n    if len(candidates) <= 1:\n        return candidates[0] if candidates else \"\"\n    \n    # Дедупликация\n    seen = set()\n    unique = []\n    for c in candidates:\n        if c not in seen:\n            seen.add(c)\n            unique.append(c)\n    candidates = unique\n    \n    if len(candidates) == 1:\n        return candidates[0]\n    \n    best_score = -1\n    best_text = candidates[0]\n    \n    for i, hyp in enumerate(candidates):\n        total = 0\n        for j, ref in enumerate(candidates):\n            if i != j:\n                total += sacrebleu.sentence_chrf(hyp, [ref], word_order=2).score\n        avg = total / (len(candidates) - 1)\n        \n        if avg > best_score:\n            best_score = avg\n            best_text = hyp\n    \n    return best_text\n\n\ndef mbr_decode_combined(candidates: List[str]) -> str:\n    \"\"\"MBR по geometric mean(BLEU, chrF++) — как метрика соревнования\"\"\"\n    if len(candidates) <= 1:\n        return candidates[0] if candidates else \"\"\n    \n    seen = set()\n    unique = []\n    for c in candidates:\n        if c not in seen:\n            seen.add(c)\n            unique.append(c)\n    candidates = unique\n    \n    if len(candidates) == 1:\n        return candidates[0]\n    \n    best_score = -1\n    best_text = candidates[0]\n    \n    for i, hyp in enumerate(candidates):\n        chrf_total = 0\n        bleu_total = 0\n        \n        for j, ref in enumerate(candidates):\n            if i != j:\n                chrf_total += sacrebleu.sentence_chrf(hyp, [ref], word_order=2).score\n                bleu_total += sacrebleu.sentence_bleu(hyp, [ref], tokenize=\"intl\").score\n        \n        n = len(candidates) - 1\n        avg_chrf = chrf_total / n\n        avg_bleu = bleu_total / n\n        \n        if avg_bleu > 0 and avg_chrf > 0:\n            combined = math.sqrt(avg_bleu * avg_chrf)\n        else:\n            combined = avg_chrf\n        \n        if combined > best_score:\n            best_score = combined\n            best_text = hyp\n    \n    return best_text\n\n\n# Доступные стратегии\nMBR_STRATEGIES = {\n    \"chrf\": mbr_decode_chrf,\n    \"combined\": mbr_decode_combined,\n}\n\n\ndef select_best_candidates(\n    candidates_df: pd.DataFrame,\n    strategy: str = \"chrf\",\n) -> pd.DataFrame:\n    \"\"\"\n    Применяет MBR decoding к DataFrame с кандидатами.\n    \n    Returns:\n        DataFrame с колонками: text_idx, input_text, prediction\n    \"\"\"\n    decode_fn = MBR_STRATEGIES[strategy]\n    candidate_cols = get_candidate_columns(candidates_df)\n    \n    print(f\"\\nMBR Decoding (strategy={strategy})\")\n    print(f\"  Candidates per example: {len(candidate_cols)}\")\n    print(f\"  Total examples: {len(candidates_df)}\")\n    \n    predictions = []\n    for _, row in tqdm(candidates_df.iterrows(), \n                       total=len(candidates_df), desc=\"MBR Decoding\"):\n        cands = get_candidates_for_row(row, candidate_cols)\n        best = decode_fn(cands)\n        predictions.append(best)\n    \n    result = pd.DataFrame({\n        \"text_idx\": candidates_df[\"text_idx\"],\n        \"input_text\": candidates_df[\"input_text\"],\n        \"prediction\": predictions,\n    })\n    \n    return result\n\n\n# ============================================================\n# Этап 4: Оценка (валидация)\n# ============================================================\n\ndef deep_past_score(predictions: List[str], references: List[str]) -> Dict[str, float]:\n    \"\"\"Метрика соревнования: geometric mean(BLEU, chrF++)\"\"\"\n    bleu = sacrebleu.corpus_bleu(predictions, [references], tokenize=\"intl\").score\n    chrf = sacrebleu.corpus_chrf(predictions, [references], word_order=2).score\n    score = math.sqrt(bleu * chrf) if bleu > 0 and chrf > 0 else 0.0\n    return {\"bleu\": bleu, \"chrf++\": chrf, \"score\": score}\n\n\ndef evaluate_predictions(\n    predictions_df: pd.DataFrame,\n    references: List[str],\n) -> Dict[str, float]:\n    \"\"\"Оценить предсказания\"\"\"\n    preds = predictions_df[\"prediction\"].tolist()\n    metrics = deep_past_score(preds, references)\n    \n    print(f\"\\n{'='*40}\")\n    print(f\"BLEU:   {metrics['bleu']:.4f}\")\n    print(f\"chrF++: {metrics['chrf++']:.4f}\")\n    print(f\"Score:  {metrics['score']:.4f}\")\n    print(f\"{'='*40}\")\n    \n    return metrics\n\n\ndef evaluate_per_model(\n    candidates_df: pd.DataFrame,\n    references: List[str],\n    model_configs: List[ModelConfig],\n) -> pd.DataFrame:\n    \"\"\"Оценить top-1 от каждой модели отдельно (для сравнения)\"\"\"\n    results = []\n    \n    for config in model_configs:\n        col = f\"{config.name}_0\"  # top-1 кандидат\n        if col in candidates_df.columns:\n            preds = candidates_df[col].tolist()\n            metrics = deep_past_score(preds, references)\n            metrics[\"model\"] = config.name\n            results.append(metrics)\n            print(f\"  {config.name}: score={metrics['score']:.4f} \"\n                  f\"(BLEU={metrics['bleu']:.2f}, chrF++={metrics['chrf++']:.2f})\")\n    \n    return pd.DataFrame(results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:33.503779Z","iopub.execute_input":"2026-03-22T21:46:33.504078Z","iopub.status.idle":"2026-03-22T21:46:33.604201Z","shell.execute_reply.started":"2026-03-22T21:46:33.504047Z","shell.execute_reply":"2026-03-22T21:46:33.603608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def translate(texts, model, tokenizer):\n    model.to(device)\n    batch_size = 16\n    samples = 0\n    all_outputs = []\n    translations = []\n    while(samples < len(texts)):\n        batch = texts[samples:min(samples+batch_size, len(texts))]\n        inputs = tokenizer(batch, return_tensors=\"pt\", truncation=True, padding=True, max_length=512).to(device) # Move inputs to the same device as the model\n        outputs = model.generate(\n        **inputs,\n        # max_length=600,\n        # max_new_tokens=600,\n        max_new_tokens=512,\n        num_beams=5,\n        length_penalty=1.7,\n        # top_p = 0.8,\n        # temperature = 1.3\n        )\n        translations.append(tokenizer.batch_decode(outputs, skip_special_tokens=True))\n        samples += batch_size\n    return translations\n\ndef remove_special_tokens(translations: str):\n    pattern = r'</?[a-z|A-Z]+>'\n    for index, translation in enumerate(translations):\n        groups = re.findall(pattern, translation)\n        for group in groups:\n            # print(f\"<<<<<<{groups}\")\n            if(not 'gap' in group):\n                replacement = re.sub(pattern, \"\", group)\n                translations[index] = translations[index].replace(group, replacement)\n    return translations\n\ndeterminatives_map = {\"{d}\":\"{d}\",\n                  \"{MUL}\":\"{MUL}\",\n                  \"{mul}\": \"{MUL}\",\n                  \"{KI}\": \"{ki}\",\n                  \"{ki}\": \"{ki}\",\n                  \"{LÚ}\": \"{LÚ}\",\n                  \"{lú}\": \"{LÚ}\",\n                  \"{lu2}\": \"{LÚ}\",\n                  \"{lu₂}\": \"{LÚ}\",\n                  \"{É}\": \"{É}\",\n                  \"{é}\": \"{É}\",\n                  \"{e₂}\": \"{É}\",\n                  \"{URU}\": \"{URU}\",\n                  \"{uru}\": \"{URU}\",\n                  \"{KUR}\":\"{KUR}\",\n                  \"{kur}\": \"{KUR}\",\n                  \"{f}\": \"{f}\",\n                  \"{mi}\": \"{f}\",\n                  \"{mí}\": \"{f}\",\n                  \"{munus}\": \"{f}\",\n                  \"{1}\": \"{m}\",\n                  \"{m}\": \"{m}\",\n                  \"{GIŠ}\": \"{GIŠ}\",\n                  \"{geš}\": \"{GIŠ}\",\n                  \"{TÚG}\":\"{TÚG}\",\n                  \"{túg}\": \"{TÚG}\",\n                  \"{tug₂}\": \"{TÚG}\",\n                  \"{tug}\": \"{TÚG}\",\n                  \"{DUB}\": \"{DUB}\",\n                  \"{dub}\": \"{DUB}\",\n                  \"{ÍD}\":\"{ID}\",\n                  \"{id}\": \"{ID}\",\n                  \"{id₂}\": \"{ID}\",\n                  \"{ID}\": \"{ID}\",\n                  \"{MUŠEN}\": \"{MUŠEN}\",\n                  \"{mušen}\": \"{MUŠEN}\",\n                  \"{na₄}\": \"{na4}\",\n                  \"{kuš}\": \"{KUŠ}\",\n                  \"{KUŠ}\": \"{KUŠ}\",\n                  \"{Ú}\": \"{Ú}\",\n                  \"{u}\": \"{Ú}\",\n                  \"{u₂}\": \"{Ú}\"}\n\ndef normalize_determinatives(transliterations: str):\n    texts_in_braces_pattern = r'\\{[^\\{\\}]*\\}'\n    for index, transliteration in enumerate(transliterations):\n        texts_in_braces = re.findall(texts_in_braces_pattern, transliteration)\n        for braces_text in texts_in_braces:\n          if(braces_text in determinatives_map):\n            braces_text = braces_text.strip().replace(' ', '')\n            transliteration = transliteration.replace(braces_text, determinatives_map[braces_text])\n        transliterations[index] = transliteration\n    return transliterations\n\ndef normalize_transliteration_chars_and_words(transliterations):\n    for index, transliteration in enumerate(transliterations):\n        for char, replacement in translit_character_map.items():\n          transliteration = re.sub(char, replacement, transliteration)\n        transliteration = re.sub(r'D[ÙUÚ]10', 'DÙG', transliteration)\n        transliteration = re.sub(r'DÚG', 'DÙG', transliteration)\n        transliteration = re.sub(r'š[AÀ]\\.BA', 'ŠÀ.BA', transliteration, flags=re.IGNORECASE)\n        transliteration = re.sub(r'GUŠKIN', 'KÙ.GI', transliteration)\n        transliteration = re.sub(r'Pu-su-ke-en_6', 'Pu-šu-ke-en_6', transliteration)\n        transliteration = re.sub(r'KÙ.KI', 'KÙ.GI', transliteration)\n        transliteration = f\"translate Old Assyrian transliteration to English: {transliteration}\"\n        transliterations[index] = transliteration\n    return transliterations\n\ndef remove_all_scribal_characters(text: str):\n    for scribal_character in scribal_characters:\n      text = text.replace(scribal_character, \"\")\n    return text\n\ndef normalize_sub_superscripts(texts: str):\n    for index, text in enumerate(texts):\n        for sub_sup, normal in subscript_to_normal_map.items():\n            text = text.replace(sub_sup, normal)\n            texts[index] = text\n    return texts\n\ndef extract_grouped_text(text, patterns):\n    all_groups = []\n    for pattern in patterns:\n      groups = re.findall(pattern, text)\n      all_groups.extend(groups)\n      # print(all_groups)\n    return all_groups\n\ndef remove_scribal_notations(texts: list):\n  # for scribal_notation in scribal_notations:\n      for index, trans in enumerate(texts):\n        groups = extract_grouped_text(texts[index], scribal_notation_patterns)\n        if(groups):\n          # print(len(groups))\n          for i, group in enumerate(groups):\n            # if(len(group)<7):\n            #   print(group)\n            # print(groups)\n            groups[i] = remove_all_scribal_characters(groups[i])\n            groups[i] = groups[i].replace(\" . \", \"\")\n            for key, value in determinatives_map.items():\n              texts[index] = texts[index].replace(key, value)\n                # print(group)\n            if(len(groups[i]) == 2):\n              groups[i] = ''\n            groups[i] = groups[i].replace('[', '').replace(']', '').replace('<', '').replace('>', '').replace('˹', '').replace('˺', '')\n            texts[index] = texts[index].replace(group, groups[i])\n            # texts[index] = trans\n      return texts\n\ndef standardize_text_breaks(texts: str):\n      #text = re.sub(r'(?:\\[?(?:\\.{3,}|…)\\]?\\s*){2,}', '<big_gap>', text)\n      gap_1 = r'(x\\s?+){2,}'\n      gap_2 = r'(…|\\.{3,}\\s?+){1,}'\n      # gap_3 = r'(\\s+x\\s+)|(\\[\\s*x\\s*\\])|(x){2,}'\n      collapsed_breaks = r'(<[a-z|_?]+>(\\s+)?\\(?\\)?-?(\\s+)?){2,}'\n    \n      trans_gaps_patterns = [r'(\\s*x\\s+){1,}', r'x{2,}']\n\n      for index, text in enumerate(texts):\n          text = re.sub(gap_2, '<big_gap>', text)\n          text = re.sub(gap_1, '<big_gap>', text, flags=re.IGNORECASE)\n          text = re.sub(r'\\[?\\s?x\\s?\\]?', \"<gap>\", text, flags=re.IGNORECASE)\n          text = re.sub(collapsed_breaks, '<big_gap>', text)\n          texts[index] = text\n      return texts\n\ndef remove_residuals_from_breaks(texts: str):\n    for index, text in enumerate(texts):\n        text = text.replace('+', '')\n        texts[index] = text\n    return texts\n\ndef normalize(texts: str):\n    for index, text in enumerate(texts):\n        text = unicodedata.normalize('NFC', text)\n        texts[index] = text\n    return texts\n\ndef normalize_translation_fractions(texts: list):\n    unicode_fraction_map = {'1/4': '¼',\n                              '1/2': '½',\n                              '1/3': '⅓',\n                              '2/3': '⅔',\n                              '5/6': '⅚',\n                              '1/6': '⅙',\n                               '3/4': '¾',\n                               '1/5': '⅕',\n                                '1/8': '⅛'}\n    for index, text in enumerate(texts):\n        for fraction, unicode_fraction in unicode_fraction_map.items():\n            text = text.replace(fraction, unicode_fraction)\n            text = text.replace('⁄', '/')\n            texts[index] = text\n    return texts\n\ndef normalize_transliteration_fractions(texts: list):\n    unicode_fraction_map = {'¼': '1/4',\n                              '½': '1/2',\n                              '⅓': '1/3',\n                              '⅔': '2/3',\n                              '⅚': '5/6',\n                              '⅙': '1/6',\n                               '¾': '3/4',\n                               '⅕': '1/5',\n                                '⅛': '1/8'}\n    for index, text in enumerate(texts):\n        for unicode_fraction, fraction in unicode_fraction_map.items():\n            text = text.replace(unicode_fraction, fraction)\n            texts[index] = text\n    return texts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:33.876356Z","iopub.execute_input":"2026-03-22T21:46:33.877303Z","iopub.status.idle":"2026-03-22T21:46:33.899097Z","shell.execute_reply.started":"2026-03-22T21:46:33.877271Z","shell.execute_reply":"2026-03-22T21:46:33.898344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# model = AutoModelForSeq2SeqLM.from_pretrained(model_path, local_files_only=True)\n# tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:34.142413Z","iopub.execute_input":"2026-03-22T21:46:34.142769Z","iopub.status.idle":"2026-03-22T21:46:34.1464Z","shell.execute_reply.started":"2026-03-22T21:46:34.142731Z","shell.execute_reply":"2026-03-22T21:46:34.14569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dataset = pd.read_csv(test_set_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:34.458696Z","iopub.execute_input":"2026-03-22T21:46:34.459422Z","iopub.status.idle":"2026-03-22T21:46:34.478188Z","shell.execute_reply.started":"2026-03-22T21:46:34.459391Z","shell.execute_reply":"2026-03-22T21:46:34.477356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:34.695987Z","iopub.execute_input":"2026-03-22T21:46:34.696514Z","iopub.status.idle":"2026-03-22T21:46:34.699564Z","shell.execute_reply.started":"2026-03-22T21:46:34.696486Z","shell.execute_reply":"2026-03-22T21:46:34.698955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transliterations = test_dataset['transliteration'].tolist()\nids = test_dataset['id']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:34.860242Z","iopub.execute_input":"2026-03-22T21:46:34.860483Z","iopub.status.idle":"2026-03-22T21:46:34.8665Z","shell.execute_reply.started":"2026-03-22T21:46:34.860461Z","shell.execute_reply":"2026-03-22T21:46:34.865889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# if(len(transliterations)<1000):\n#     a = 'ahsls'\n#     b = a[1000]\n# elif(len(transliterations)>=1000 and len(transliterations)<2000):\n#     df = pd.DataFrame({'id': ids, 'translation': ids})\n#     df.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:34.992548Z","iopub.execute_input":"2026-03-22T21:46:34.992787Z","iopub.status.idle":"2026-03-22T21:46:34.995974Z","shell.execute_reply.started":"2026-03-22T21:46:34.992763Z","shell.execute_reply":"2026-03-22T21:46:34.995333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # count = [transliteration for transliteration in transliterations if (len(transliteration.split(\" \")) > 5 and len(transliteration.split(\" \"))<=20)]\n# # # print(count)\n# # if(len(count) < 1500):\n# #     a = transliterations[0][1000000000000000000]\n# # elif(len(count)>=1500 and len(count)<=3900):\n# #     df = pd.DataFrame({'id': ids, 'translation': ids})\n# #     df.to_csv('/kaggle/working/submission.csv', index=False)\n# for transliteration in transliterations:\n#     p = r'<gap>\\s?+<big_gap>|<big_gap>\\s?+<gap>'\n#     q = re.search(p, transliteration)\n#     if(q != None and q.group()):\n#         print(p[100])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:35.605575Z","iopub.execute_input":"2026-03-22T21:46:35.605877Z","iopub.status.idle":"2026-03-22T21:46:35.609821Z","shell.execute_reply.started":"2026-03-22T21:46:35.60585Z","shell.execute_reply":"2026-03-22T21:46:35.609105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# transliterations = remove_scribal_notations(transliterations)\ntransliterations = normalize_sub_superscripts(transliterations)\n# transliterations = remove_residuals_from_breaks(transliterations)\ntransliterations = normalize_transliteration_chars_and_words(transliterations)\ntransliterations = normalize_determinatives(transliterations)\ntransliterations = normalize_transliteration_fractions(transliterations)\n# transliterations = standardize_text_breaks(transliterations)\n# transliterations = normalize(transliterations)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:41.091885Z","iopub.execute_input":"2026-03-22T21:46:41.092187Z","iopub.status.idle":"2026-03-22T21:46:41.097145Z","shell.execute_reply.started":"2026-03-22T21:46:41.09216Z","shell.execute_reply":"2026-03-22T21:46:41.096381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Определить модели ---\nmodel_configs = [\n    ModelConfig(\n        name=\"mix_v2\",\n        model_path=\"/kaggle/input/datasets/vitalykudelya/dpc26-models-2/LARGE_MIXES_WITH_OLD_NEW_DATA_v2_53.93/LARGE_MIXES_WITH_OLD_NEW_DATA_v2\",\n        num_beams=7,\n        num_return=3,\n        length_penalty=2.0,\n        batch_size=1,\n    ),\n    ModelConfig(\n        name=\"drop_v1\",\n        model_path=\"/kaggle/input/datasets/vitalykudelya/dpc26-large-mixes-trained-with-test-drop-v1\",\n        num_beams=7,\n        num_return=3,\n        length_penalty=2.0,\n        batch_size=1,\n    ),\n\n    ModelConfig(\n        name=\"mt5_large_drop\",\n        model_path=\"/kaggle/input/datasets/vitalykudelya/dpc26-models-2/aduah_MT_LARGE_from_zero_v1_continue_drop_checkpoint-1455/aduah_MT_LARGE_from_zero_v1_continue_drop_checkpoint-1455\",\n        num_beams=7,\n        num_return=3,\n        length_penalty=2.0,\n        batch_size=1,\n    ),\n    ModelConfig(\n        name=\"mt5_xl_bf16\",\n        model_path=\"/kaggle/input/datasets/vitalykudelya/dpc26-model-xl-runpod\",\n        num_beams=5,\n        num_return=1,\n        length_penalty=2.0,\n        batch_size=1,\n        torch_dtype=torch.bfloat16\n    ),\n]\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# --- Генерация кандидатов ---\ncandidates_df = generate_all_candidates(model_configs, transliterations, device)\n\n# --- MBR выбор лучшего ---\npredictions_df = select_best_candidates(candidates_df, strategy=\"chrf\")\n\n# --- Postprocessing (твой существующий код) ---\nall_predictions = predictions_df[\"prediction\"].tolist()\npredictions_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:46:41.379353Z","iopub.execute_input":"2026-03-22T21:46:41.379897Z","iopub.status.idle":"2026-03-22T21:48:47.811016Z","shell.execute_reply.started":"2026-03-22T21:46:41.379868Z","shell.execute_reply":"2026-03-22T21:48:47.810394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ids","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:48:47.812441Z","iopub.execute_input":"2026-03-22T21:48:47.813059Z","iopub.status.idle":"2026-03-22T21:48:47.81787Z","shell.execute_reply.started":"2026-03-22T21:48:47.81303Z","shell.execute_reply":"2026-03-22T21:48:47.817244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# translations = translate(transliterations, model, tokenizer)\n# all_predictions = []\n# for translation in translations:\n#     all_predictions.extend(translation)\ntranslations = normalize_translation_fractions(all_predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:48:47.818708Z","iopub.execute_input":"2026-03-22T21:48:47.818997Z","iopub.status.idle":"2026-03-22T21:48:47.829378Z","shell.execute_reply.started":"2026-03-22T21:48:47.818969Z","shell.execute_reply":"2026-03-22T21:48:47.828712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_file = pd.DataFrame({'id': ids, 'translation': translations})\nsubmission_file.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:48:47.830833Z","iopub.execute_input":"2026-03-22T21:48:47.831043Z","iopub.status.idle":"2026-03-22T21:48:47.846062Z","shell.execute_reply.started":"2026-03-22T21:48:47.831021Z","shell.execute_reply":"2026-03-22T21:48:47.845335Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# translations\n# transliterations = normalize(transliterations)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:45:35.089069Z","iopub.status.idle":"2026-03-22T21:45:35.089312Z","shell.execute_reply.started":"2026-03-22T21:45:35.089199Z","shell.execute_reply":"2026-03-22T21:45:35.089215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def extract_unk_spans(text, tokenizer):\n#     encoding = tokenizer(\n#         text,\n#         return_offsets_mapping=True,\n#         add_special_tokens=False\n#     )\n\n#     unk_spans = []\n#     for token_id, (start, end) in zip(encoding.input_ids, encoding.offset_mapping):\n#         if token_id == tokenizer.unk_token_id:\n#             unk_spans.append(text[start:end])\n\n#     return unk_spans","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:45:35.090484Z","iopub.status.idle":"2026-03-22T21:45:35.090832Z","shell.execute_reply.started":"2026-03-22T21:45:35.090663Z","shell.execute_reply":"2026-03-22T21:45:35.090687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# unks = []\n# for t in transliterations:\n#     unks.extend(extract_unk_spans(t, tokenizer))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-22T21:45:35.092877Z","iopub.status.idle":"2026-03-22T21:45:35.093215Z","shell.execute_reply.started":"2026-03-22T21:45:35.093089Z","shell.execute_reply":"2026-03-22T21:45:35.09311Z"}},"outputs":[],"execution_count":null}]}