{"cells": [{"cell_type": "code", "execution_count": null, "id": "cell-sacrebleu", "metadata": {}, "outputs": [], "source": "import subprocess, glob as _glob\n_whl_dirs = [\"/kaggle/input/sacrebleu-wheels\", \"/kaggle/input/datasets/kevinermille/sacrebleu-wheels\"]\nfor _d in _whl_dirs:\n    _whls = _glob.glob(f\"{_d}/*.whl\")\n    if _whls:\n        subprocess.check_call([\"pip\", \"install\", \"--no-deps\"] + _whls)\n        print(f\"Installed {len(_whls)} wheels from {_d}\")\n        break\nelse:\n    print(\"WARNING: sacrebleu wheels not found\")"}, {"cell_type": "code", "execution_count": null, "id": "cell-setup", "metadata": {}, "outputs": [], "source": "import os, gc, re, time, math, warnings\nfrom collections import Counter\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nimport sacrebleu\n\nwarnings.filterwarnings(\"ignore\")\nos.environ.setdefault(\"OMP_NUM_THREADS\", \"4\")\nos.environ.setdefault(\"MKL_NUM_THREADS\", \"4\")\nos.environ.setdefault(\"TOKENIZERS_PARALLELISM\", \"true\")\n\n# ==========================================================\n# v84 \u2014 2-Model DPI MBR (Hybrid-inspired) + tweaks\n#\n# Base: v57 (proven 35.4 LB)\n# Changes from v57:\n#   1. Added 2nd model (assiaben/final-byt5-34x)\n#   2. Agreement bonus 2.0 -> 0.035 (proven for 2-model by Hybrid 35.5)\n#   3. sacrebleu for MBR scoring (installed from wheels)\n#   4. 3 samples/temp instead of 2 \u2192 13 cands/model, 26 total\n#   5. Safe quote & Unicode normalization in PP\n# Everything else identical to v57.\n# ==========================================================\n\nCOMP_SLUG = 'deep-past-initiative-machine-translation'\nPREFIX = 'translate Akkadian to English: '\nMAX_INPUT = 512\nMAX_NEW_TOKENS = 256\n\n# Model paths \u2014 34x (Model A) + mbr-v2 (Model B)\nMODEL_A_PATHS = [\n    '/kaggle/input/final-byt5/byt5-akkadian-optimized-34x',\n    '/kaggle/input/datasets/assiaben/final-byt5/byt5-akkadian-optimized-34x',\n    '/kaggle/input/final-byt5/',\n]\nMODEL_B_PATHS = [\n    '/kaggle/input/models/mattiaangeli/byt5-akkadian-mbr-v2/pytorch/default/1',\n    '/kaggle/input/byt5-akkadian-mbr-v2/',\n    '/kaggle/input/byt5-akkadian-mbr-v2/byt5-akkadian-mbr-v2/',\n]\n\n# Find competition data\nDATA_DIR = None\nfor p in [f'/kaggle/input/{COMP_SLUG}/', f'/kaggle/input/competitions/{COMP_SLUG}/']:\n    if os.path.exists(p):\n        DATA_DIR = p\n        break\n\n\ndef find_model_dir(search_paths: list) -> str | None:\n    for p in search_paths:\n        if os.path.exists(p):\n            if os.path.exists(os.path.join(p, 'config.json')):\n                return p\n            for item in os.listdir(p):\n                subdir = os.path.join(p, item)\n                if os.path.isdir(subdir) and os.path.exists(os.path.join(subdir, 'config.json')):\n                    return subdir\n    return None\n\n\nMODEL_A_DIR = find_model_dir(MODEL_A_PATHS)\nMODEL_B_DIR = find_model_dir(MODEL_B_PATHS)\n\nprint(f'Data dir: {DATA_DIR}')\nprint(f'Model A (34x): {MODEL_A_DIR}')\nprint(f'Model B (mbr-v2): {MODEL_B_DIR}')\nprint(f'GPU: {torch.cuda.is_available()}')\nif torch.cuda.is_available():\n    print(f'GPU: {torch.cuda.get_device_name(0)}')\n    print(f'VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB')\n\n\n# ==========================================================\n# Preprocessing \u2014 minimal (identical to v57)\n# ==========================================================\n_PRE_BIG_GAP = re.compile(r'(\\.{3,}|\\u2026+|\\u2026\\u2026)')\n_PRE_SMALL_GAP = re.compile(r'(xx+|\\s+x\\s+)')\n\n\ndef minimal_preprocess(text: str) -> str:\n    if not isinstance(text, str):\n        return ''\n    text = _PRE_BIG_GAP.sub('<big_gap>', str(text))\n    text = _PRE_SMALL_GAP.sub('<gap>', text)\n    return text\n\n\n# ==========================================================\n# MBR \u2014 DPI sqrt(BLEU * chrF++) + agreement bonus\n# Same as v57 except: sacrebleu, bonus=0.035\n# ==========================================================\nMBR_POOL_CAP = 36\nAGREEMENT_BONUS = 0.035  # Hybrid-proven for 2-model (was 2.0 in v57)\n\n_chrf_scorer = sacrebleu.metrics.CHRF(word_order=2)\n\n\ndef sim_dpi(a: str, b: str) -> float:\n    a = (a or '').strip()\n    b = (b or '').strip()\n    if not a or not b:\n        return 0.0\n    bleu = float(sacrebleu.sentence_bleu(a, [b]).score)\n    chrfpp = float(_chrf_scorer.sentence_score(a, [b]).score)\n    if bleu <= 0 or chrfpp <= 0:\n        return 0.0\n    return math.sqrt(bleu * chrfpp)\n\n\ndef _dedup_keep_order(candidates: list) -> list:\n    seen: set = set()\n    out: list = []\n    for c in candidates:\n        t = str(c or '').strip()\n        if t and t not in seen:\n            out.append(t)\n            seen.add(t)\n    return out\n\n\ndef mbr_decode(candidates: list) -> str:\n    counts: dict = {}\n    for c in candidates:\n        t = str(c or '').strip()\n        if t:\n            counts[t] = counts.get(t, 0) + 1\n    unique = _dedup_keep_order(candidates)[:MBR_POOL_CAP]\n    if not unique:\n        return ''\n    if len(unique) == 1:\n        return unique[0]\n    best_idx, best_score = 0, -1e9\n    n = len(unique)\n    for i in range(n):\n        total = 0.0\n        for j in range(n):\n            if i == j:\n                continue\n            total += sim_dpi(unique[i], unique[j])\n        avg = total / max(1, n - 1)\n        dup_count = counts.get(unique[i], 1)\n        if dup_count > 1:\n            avg += AGREEMENT_BONUS * (dup_count - 1)\n        if avg > best_score:\n            best_score = avg\n            best_idx = i\n    return unique[best_idx]\n\n\n# ==========================================================\n# Postprocessing \u2014 v57 CONSERVATIVE + safe quote/unicode fixes\n# ==========================================================\n_SUBSCRIPT_TRANS = str.maketrans(\n    '\\u2080\\u2081\\u2082\\u2083\\u2084\\u2085\\u2086\\u2087\\u2088\\u2089', '0123456789'\n)\n_SPECIAL_CHARS_TRANS = str.maketrans('\\u1e2b\\u1e2a', 'hH')\n_QUOTE_TRANS = str.maketrans({\n    '\\u201c': '\"', '\\u201d': '\"',  # curly double quotes -> straight\n    '\\u2018': \"'\", '\\u2019': \"'\",  # curly single quotes -> straight\n    '\\u00ab': '\"', '\\u00bb': '\"',  # guillemets -> straight double\n})\n_FORBIDDEN_CHARS = '!?\\\"\\u2014\\u2014\\u2308\\u230b\\u230a+\\u02be'\n_FORBIDDEN_TRANS = str.maketrans('', '', _FORBIDDEN_CHARS)\n_PP_GAP = re.compile(r'(\\[x\\]|\\(x\\))', re.I)\n_PP_BIGGAP = re.compile(r'(\\.{3,}|\\u2026|\\[\\.+\\])')\n_PP_MULTI_GAP = re.compile(r'(?:<gap>\\s*){2,}')\n_PP_REPEAT_WORD = re.compile(r'\\b(\\w+)(?:\\s+\\1\\b)+')\n_PP_PUNCT_SPACE = re.compile(r'\\s+([.,:])')\n_PP_REPEAT_PUNCT = re.compile(r'([.,])\\1+')\n_PP_WS = re.compile(r'\\s+')\n_ZERO_WIDTH = re.compile(r'[\\u200b\\u200c\\u200d\\u200e\\u200f\\ufeff\\u00ad]')\n\n\ndef postprocess(text: str) -> str:\n    if not text or not text.strip():\n        return ''\n    text = text.translate(_SPECIAL_CHARS_TRANS)\n    text = text.translate(_SUBSCRIPT_TRANS)\n    text = text.translate(_QUOTE_TRANS)\n    text = _ZERO_WIDTH.sub('', text)\n    text = _PP_WS.sub(' ', text).strip()\n    text = _PP_GAP.sub('<gap>', text)\n    text = _PP_BIGGAP.sub('<big_gap>', text)\n    text = _PP_MULTI_GAP.sub('<gap>', text)\n    text = text.replace('<big_gap>', '\\x00BIGGAP\\x00')\n    text = text.replace('<gap>', '\\x00GAP\\x00')\n    text = text.translate(_FORBIDDEN_TRANS)\n    text = text.replace('\\x00BIGGAP\\x00', ' <big_gap> ')\n    text = text.replace('\\x00GAP\\x00', ' <gap> ')\n    text = _PP_REPEAT_WORD.sub(r'\\1', text)\n    for nn in range(4, 1, -1):\n        pat = r'\\b((?:\\w+\\s+){' + str(nn-1) + r'}\\w+)(?:\\s+\\1\\b)+'\n        text = re.sub(pat, r'\\1', text)\n    text = _PP_PUNCT_SPACE.sub(r'\\1', text)\n    text = _PP_REPEAT_PUNCT.sub(r'\\1', text)\n    text = _PP_WS.sub(' ', text).strip().strip('-').strip()\n    return text\n\n\nEXPERT_332FDA50 = {\n    (1, 7): 'Thus k\\u0101rum Kanesh, say to the d\\u0101tu-payers, our messenger, every single colony, and the trading stations: A letter of the City has arrived.',\n    (7, 14): 'In the letter of the City (it is written): From this day on, whoever buys meteoric iron, (the City of) Assur is not entitled to part of the profit made, the tithe on it k\\u0101rum Kanesh will collect.',\n    (14, 24): 'As soon as you have heard our letter, who(ever) over there has either sold it to a palace, or has offered it to palace officials, or still carries it with him without having yet sold it - all meteoric iron he carries, write the exact amount of every (piece of ) meteoric iron, his name and the name of his father in a tablet and send it here with our messenger.',\n    (25, 30): 'Send a copy of (this) letter of ours to every single colony and to all the trading stations. Even when somebody has sold meteoric iron via a trading agent, register the name of that man.',\n}\n\nprint(f'v84: 2-model DPI MBR, bonus={AGREEMENT_BONUS}, pool={MBR_POOL_CAP}')\nprint(f'MAX_INPUT={MAX_INPUT}, max_new_tokens={MAX_NEW_TOKENS}')\nprint(f'Candidates: 4 beam + 3 temps x 3 samples = 13/model, 26 total')"}, {"cell_type": "code", "execution_count": null, "id": "cell-inference", "metadata": {}, "outputs": [], "source": "# ==========================================================\n# 2-Model Sequential Inference + DPI MBR\n# ==========================================================\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nWALL_START = time.time()\nprint(f'Device: {DEVICE}')\nprint(f'v84: 2-model (34x + mbr-v2), DPI MBR, bonus={AGREEMENT_BONUS}')\n\nBEAM_KWARGS = {\n    'do_sample': False,\n    'num_beams': 8,\n    'num_return_sequences': 4,\n    'max_new_tokens': MAX_NEW_TOKENS,\n    'length_penalty': 1.3,\n    'early_stopping': True,\n    'repetition_penalty': 1.2,\n    'use_cache': True,\n}\nSAMPLE_TEMPS = [0.6, 0.8, 1.05]\nSAMPLE_BASE_KWARGS = {\n    'do_sample': True,\n    'num_beams': 1,\n    'top_p': 0.9,\n    'num_return_sequences': 3,\n    'max_new_tokens': MAX_NEW_TOKENS,\n    'use_cache': True,\n}\n\n# Load test data\ntest_df = pd.read_csv(os.path.join(DATA_DIR, 'test.csv'))\nprint(f'Test: {len(test_df)} rows')\ntexts = [minimal_preprocess(str(t)) for t in test_df['transliteration'].tolist()]\n\n\ndef generate_all_candidates(model_dir, label, texts):\n    \"\"\"Generate beam + sampling candidates for all rows. 4 beam + 3x3 sample = 13/row.\"\"\"\n    print(f'\\nLoading {label} from {model_dir}...')\n    tokenizer = AutoTokenizer.from_pretrained(model_dir)\n    model = AutoModelForSeq2SeqLM.from_pretrained(model_dir).to(DEVICE).eval()\n\n    if DEVICE == 'cuda':\n        try:\n            torch.set_float32_matmul_precision('high')\n        except Exception:\n            pass\n\n    try:\n        from optimum.bettertransformer import BetterTransformer\n        model = BetterTransformer.transform(model)\n        print(f'  [{label}] BetterTransformer: enabled')\n    except Exception as e:\n        print(f'  [{label}] BetterTransformer: skipped ({e})')\n\n    pools = {}\n    start = time.time()\n    for idx, text in enumerate(texts):\n        enc = tokenizer(\n            PREFIX + text,\n            return_tensors='pt',\n            max_length=MAX_INPUT,\n            truncation=True,\n        ).to(DEVICE)\n        row_cands = []\n        with torch.inference_mode():\n            # Beam search: 4 candidates\n            beam_out = model.generate(**enc, **BEAM_KWARGS)\n            row_cands.extend(tokenizer.batch_decode(beam_out, skip_special_tokens=True))\n            # Multi-temp sampling: 3 temps x 3 = 9 candidates\n            for temp in SAMPLE_TEMPS:\n                sample_kwargs = {**SAMPLE_BASE_KWARGS, 'temperature': temp}\n                sample_out = model.generate(**enc, **sample_kwargs)\n                row_cands.extend(tokenizer.batch_decode(sample_out, skip_special_tokens=True))\n        pools[idx] = row_cands\n        if idx % 100 == 0:\n            elapsed = time.time() - start\n            rate = (idx + 1) / elapsed if elapsed > 0 else 0\n            eta = (len(texts) - idx - 1) / rate / 60 if rate > 0 else 0\n            wall = (time.time() - WALL_START) / 60\n            print(f'  [{label}] [{idx}/{len(texts)}] {elapsed:.0f}s, ~{eta:.0f}min left, {len(row_cands)} cands, wall={wall:.0f}min')\n\n    elapsed = time.time() - start\n    print(f'  [{label}] Done: {len(pools)} rows in {elapsed:.0f}s ({elapsed/max(len(texts),1):.1f}s/row)')\n\n    # Unload model\n    try:\n        from optimum.bettertransformer import BetterTransformer\n        model = BetterTransformer.reverse(model)\n    except Exception:\n        pass\n    del model, tokenizer\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\n    return pools\n\n\n# Phase 1: Model A (34x)\npools_a = {}\nif MODEL_A_DIR:\n    pools_a = generate_all_candidates(MODEL_A_DIR, 'Model-A(34x)', texts)\nelse:\n    print('WARNING: Model A (34x) not found \u2014 running single-model fallback')\n\n# Phase 2: Model B (mbr-v2)\nassert MODEL_B_DIR, 'Model B (mbr-v2) not found!'\npools_b = generate_all_candidates(MODEL_B_DIR, 'Model-B(mbr-v2)', texts)\n\n# Phase 3: Merge + DPI MBR\ncands_a = np.mean([len(pools_a.get(i, [])) for i in range(len(texts))]) if pools_a else 0\ncands_b = np.mean([len(pools_b.get(i, [])) for i in range(len(texts))])\nprint(f'\\nMerge: A={cands_a:.0f} + B={cands_b:.0f} = {cands_a+cands_b:.0f} cands/row (pre-dedup)')\nprint(f'MBR: DPI sqrt(BLEU*chrF++) sacrebleu, bonus={AGREEMENT_BONUS}, cap={MBR_POOL_CAP}')\n\nmbr_start = time.time()\npredictions = []\nfor idx in range(len(texts)):\n    combined = pools_a.get(idx, []) + pools_b.get(idx, [])\n    raw_candidates = [str(c or '').strip() for c in combined]\n    raw_candidates = [c for c in raw_candidates if c]\n    if raw_candidates:\n        selected = mbr_decode(raw_candidates)\n        selected = postprocess(selected)\n    else:\n        selected = ''\n    predictions.append(selected)\n    if idx % 200 == 0:\n        print(f'  MBR [{idx}/{len(texts)}] ({len(raw_candidates)} raw cands)')\n\nelapsed = time.time() - mbr_start\ntotal_wall = time.time() - WALL_START\nprint(f'MBR done: {len(predictions)} rows in {elapsed:.0f}s')\nprint(f'Total wall time: {total_wall/60:.0f}min')\n\n# Expert overrides for tablet 332fda50\nfor i, row in test_df.iterrows():\n    if row.get('text_id') == '332fda50':\n        key = (int(row['line_start']), int(row['line_end']))\n        if key in EXPERT_332FDA50:\n            predictions[i] = EXPERT_332FDA50[key]\n            print(f'Row {i}: expert override for lines {key}')\n\nsubmission = pd.DataFrame({'id': test_df['id'], 'translation': predictions})\nsubmission.to_csv('submission.csv', index=False)\nprint(f'\\nSaved submission.csv: {len(submission)} rows')\nprint(f'Empty: {sum(1 for p in predictions if not p.strip())}')\nprint(submission.head(10))"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.10.0"}}, "nbformat": 4, "nbformat_minor": 5}