{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":14899351,"datasetId":9366631,"databundleVersionId":15763950},{"sourceType":"modelInstanceVersion","sourceId":755680,"databundleVersionId":15750522,"modelInstanceId":577094,"modelId":589419,"isSourceIdPinned":false}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"698ce8ec-19cb-49c0-908f-9b805a7e2027","cell_type":"markdown","source":"# Deep Past Challenge — Akkadian to English Translation\n## 2-Model MBR Ensemble · Multi-Run Blend\n\n**Strategy:** Run the full 2-model MBR pipeline 3 times with different random seeds,\nthen do a final MBR blend across all runs. Stochastic sampling produces different\nhigh-quality candidates each run — the final MBR picks the consensus best.\n\n**Expected score: 35.9 – 36.3**","metadata":{}},{"id":"99a385d5-f787-4528-9689-eabc2949c4cd","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"5bb078b6-03c1-4d6b-84b5-cef5fb043e7b","cell_type":"code","source":"# ── Imports ───────────────────────────────────────────────────────────────────\nimport os, gc, re, logging, warnings, random\nfrom dataclasses import dataclass, field\nfrom typing import List\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom tqdm.auto import tqdm\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM, set_seed\n\ntry:\n    from sacrebleu.metrics import CHRF\nexcept ImportError:\n    os.system('pip install sacrebleu -q')\n    from sacrebleu.metrics import CHRF\n\n_chrf = CHRF(word_order=2)\n\nwarnings.filterwarnings('ignore')\nlogging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s')\nlog = logging.getLogger(__name__)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nUSE_BF16 = torch.cuda.is_available() and torch.cuda.is_bf16_supported()\nlog.info(f'Device: {device} | BF16: {USE_BF16}')\nif torch.cuda.is_available():\n    log.info(f'GPU: {torch.cuda.get_device_name(0)} | VRAM: {torch.cuda.get_device_properties(0).total_memory/1e9:.1f} GB')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T17:03:18.030626Z","iopub.execute_input":"2026-03-23T17:03:18.031571Z","iopub.status.idle":"2026-03-23T17:03:30.129665Z","shell.execute_reply.started":"2026-03-23T17:03:18.031520Z","shell.execute_reply":"2026-03-23T17:03:30.128939Z"}},"outputs":[],"execution_count":null},{"id":"624f5ae7-2c5b-4e55-ab9d-343fe2255882","cell_type":"code","source":"# ── Configuration ─────────────────────────────────────────────────────────────\n@dataclass\nclass Config:\n    # Paths\n    test_path: str = '/kaggle/input/deep-past-initiative-machine-translation/test.csv'\n    out_path:  str = '/kaggle/working/submission.csv'\n\n    # The two proven models\n    model_paths: List[str] = field(default_factory=lambda: [\n        '/kaggle/input/datasets/assiaben/final-byt5/byt5-akkadian-optimized-34x',\n        '/kaggle/input/models/mattiaangeli/byt5-akkadian-mbr-v2/pytorch/default/1',\n    ])\n\n    # Multi-run: run the full pipeline this many times with different seeds\n    # Each run takes ~25-30 min on GPU. 3 runs = ~75-90 min total (within Kaggle 9h limit)\n    num_runs: int = 3\n    run_seeds: List[int] = field(default_factory=lambda: [42, 1337, 2024])\n\n    # Generation — proven parameters from the 35.7 run\n    max_input_tokens:   int   = 1024\n    max_new_tokens:     int   = 384\n    length_penalty:     float = 1.3\n    num_beams:          int   = 8\n    repetition_penalty:  float = 1.2\n\n    # Beam candidates per model per run\n    n_beam_cands:      int   = 7   # increased since diverse beam is disabled\n    n_diverse_cands:   int   = 5\n    diversity_penalty: float = 0.9\n    use_diverse_beam:  bool  = False  # not supported for T5 in this transformers version\n\n    # Sampling candidates per model per run\n    n_sample_per_temp: int         = 2\n    temperatures:      List[float] = field(default_factory=lambda: [0.55, 0.75, 0.95])\n    top_p:             float       = 0.92\n\n    # MBR\n    mbr_pool_cap: int = 100   # larger cap for the final cross-run blend\n\n    # Batching\n    batch_size: int = 8\n\ncfg = Config()\n\nn_sample_cands  = cfg.n_sample_per_temp * len(cfg.temperatures)\ncands_per_model = cfg.n_beam_cands + (cfg.n_diverse_cands if cfg.use_diverse_beam else 0) + n_sample_cands\ncands_per_run   = cands_per_model * len(cfg.model_paths)\ntotal_cands     = cands_per_run * cfg.num_runs\n\nlog.info(f'Candidates / model / run : {cands_per_model}')\nlog.info(f'Candidates / run         : {cands_per_run}')\nlog.info(f'Total pool (all runs)    : {total_cands} per sample')\nlog.info(f'Number of runs           : {cfg.num_runs} (seeds: {cfg.run_seeds})')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T17:03:30.131375Z","iopub.execute_input":"2026-03-23T17:03:30.131829Z","iopub.status.idle":"2026-03-23T17:03:30.144434Z","shell.execute_reply.started":"2026-03-23T17:03:30.131787Z","shell.execute_reply":"2026-03-23T17:03:30.143819Z"}},"outputs":[],"execution_count":null},{"id":"95f889a2-2962-4cdd-aa77-d519e7b18a55","cell_type":"code","source":"# ── Preprocessing ─────────────────────────────────────────────────────────────\n# str.maketrans only supports single-char keys; use sequential replace instead\n_DIAC_SUBS = [\n    ('s,', 'ṣ'), ('S,', 'Ṣ'),\n    ('t,', 'ṭ'), ('T,', 'Ṭ'),\n    ('s`', 'š'), ('S`', 'Š'),\n    ('h,', 'ḫ'), ('H,', 'Ḫ'),\n]\n_GAP_RE = re.compile(r'\\[\\.\\.\\.|\\.\\.\\.\\.|\\[\\s*\\]')\n\ndef preprocess(texts: List[str]) -> List[str]:\n    out = []\n    for t in texts:\n        t = str(t) if not isinstance(t, str) else t\n        for src, dst in _DIAC_SUBS:\n            t = t.replace(src, dst)\n        t = _GAP_RE.sub('[GAP]', t)\n        out.append(t.strip())\n    return out\n\nprint('Preprocessor ready')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T17:03:30.145240Z","iopub.execute_input":"2026-03-23T17:03:30.145448Z","iopub.status.idle":"2026-03-23T17:03:30.159395Z","shell.execute_reply.started":"2026-03-23T17:03:30.145419Z","shell.execute_reply":"2026-03-23T17:03:30.158660Z"}},"outputs":[],"execution_count":null},{"id":"75c8e6de-efd7-48e2-9cd3-6d8e216a2490","cell_type":"code","source":"# ── Postprocessing ────────────────────────────────────────────────────────────\n_FRAC = {\n    r'\\b1/2\\b': '½', r'\\b1/3\\b': '⅓', r'\\b2/3\\b': '⅔',\n    r'\\b1/4\\b': '¼', r'\\b3/4\\b': '¾',\n}\n\n_SUBS = [\n    (re.compile(p), r) for p, r in [\n        (r'\\[GAP\\]',                             '...'),\n        (r'(?i)^translate akkadian to english:\\s*', ''),\n        (r'\\|',                                  '/'),\n        (r'[\\x00-\\x08\\x0b-\\x1f\\x7f]',           ''),\n        (r'[ \\t]+',                              ' '),\n        (r' ,',                                  ','),\n        (r' \\.',                                 '.'),\n        (r' ;',                                  ';'),\n        (r' :',                                  ':'),\n        (r'\\( ',                                 '('),\n        (r' \\)',                                 ')'),\n        (r'\\b(\\w+(?:\\s+\\w+){1,3})\\s+\\1\\b',      r'\\1'),\n        # single-word repetition runs: 'word word word ...' → 'word'\n        (r'\\b(\\w+)(\\s+\\1){2,}\\b',                  r'\\1'),\n        (r'\\s+\\w{1,3}$',                         ''),\n        (r'\\.{2,}(?!\\.)',                        '...'),\n        (r',{2,}',                               ','),\n        (r'!{2,}',                               '!'),\n        (r'\"\\s+',                                '\"'),\n        (r'\\s+\"',                                '\"'),\n        (r'^\\s+|\\s+$',                           ''),\n    ]\n]\n\ndef postprocess(text: str) -> str:\n    if not text or not text.strip():\n        return ''\n    for pat, repl in _SUBS:\n        text = pat.sub(repl, text)\n    for pat, repl in _FRAC.items():\n        text = re.sub(pat, repl, text)\n    if text:\n        text = text[0].upper() + text[1:]\n    if text and text[-1] not in '.!?\"':\n        text += '.'\n    return text.strip()\n\nprint('Postprocessor ready')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T17:03:30.160333Z","iopub.execute_input":"2026-03-23T17:03:30.160811Z","iopub.status.idle":"2026-03-23T17:03:30.174367Z","shell.execute_reply.started":"2026-03-23T17:03:30.160762Z","shell.execute_reply":"2026-03-23T17:03:30.173731Z"}},"outputs":[],"execution_count":null},{"id":"fc77b63c-9e38-49ee-940f-b85f592b5d6f","cell_type":"code","source":"# ── MBR selection (chrF++ consensus) ─────────────────────────────────────────\ndef mbr_select(candidates: List[str], cap: int = cfg.mbr_pool_cap) -> str:\n    seen, unique = set(), []\n    for c in candidates:\n        c = c.strip()\n        if c and c not in seen:\n            seen.add(c)\n            unique.append(c)\n    if not unique:\n        return ''\n    if len(unique) == 1:\n        return unique[0]\n    pool = unique[:cap]\n    n = len(pool)\n    scores = np.zeros(n)\n    for i, hyp in enumerate(pool):\n        refs = [pool[j] for j in range(n) if j != i]\n        scores[i] = _chrf.sentence_score(hyp, refs).score\n    return pool[int(np.argmax(scores))]\n\nprint('MBR selector ready')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T17:03:30.175912Z","iopub.execute_input":"2026-03-23T17:03:30.176494Z","iopub.status.idle":"2026-03-23T17:03:30.187702Z","shell.execute_reply.started":"2026-03-23T17:03:30.176472Z","shell.execute_reply":"2026-03-23T17:03:30.187055Z"}},"outputs":[],"execution_count":null},{"id":"3287b2e8-d913-40a2-8ef3-966470207bd3","cell_type":"code","source":"# ── Load & sort data ──────────────────────────────────────────────────────────\ntest_df     = pd.read_csv(cfg.test_path)\ncorrect_ids = test_df['id'].tolist()\nraw_texts   = test_df['transliteration'].tolist()\npreprocessed = preprocess(raw_texts)\ninputs      = ['translate Akkadian to English: ' + t for t in preprocessed]\n\n# sort by length for efficient batching\nlengths       = [len(t) for t in inputs]\norder         = np.argsort(lengths)\ninputs_sorted = [inputs[i]      for i in order]\nids_sorted    = [correct_ids[i] for i in order]\n\nlog.info(f'Test samples: {len(inputs)} | avg length: {np.mean(lengths):.0f} chars')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T17:03:30.188571Z","iopub.execute_input":"2026-03-23T17:03:30.188913Z","iopub.status.idle":"2026-03-23T17:03:30.222688Z","shell.execute_reply.started":"2026-03-23T17:03:30.188878Z","shell.execute_reply":"2026-03-23T17:03:30.222112Z"}},"outputs":[],"execution_count":null},{"id":"14987631-2a7a-473b-af03-405beda71a86","cell_type":"code","source":"# ── Candidate generator (one model, one run) ──────────────────────────────────\ndef generate_candidates(\n    model_path: str,\n    texts: List[str],\n    seed: int,\n) -> List[List[str]]:\n    name = Path(model_path).name\n    set_seed(seed)\n\n    dtype = torch.bfloat16 if USE_BF16 else (torch.float16 if torch.cuda.is_available() else torch.float32)\n    try:\n        tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True)\n        model = AutoModelForSeq2SeqLM.from_pretrained(\n            model_path, local_files_only=True, dtype=dtype\n        ).eval()\n        try:\n            model = model.to(device)\n            active_device = device\n        except RuntimeError as e:\n            log.warning(f'CUDA error — falling back to CPU: {e}')\n            model = model.to('cpu')\n            active_device = torch.device('cpu')\n    except Exception as e:\n        log.error(f'Failed to load {name}: {e}')\n        return [[] for _ in texts]\n\n    log.info(f'  Loaded {name} on {active_device} (seed={seed})')\n\n    all_cands: List[List[str]] = [[] for _ in texts]\n\n    def _run_batch(batch_texts, n_return, gen_kwargs):\n        enc = tokenizer(\n            batch_texts, return_tensors='pt', padding=True,\n            truncation=True, max_length=cfg.max_input_tokens,\n        ).to(active_device)\n        amp_device = str(active_device).split(':')[0]\n        with torch.no_grad(), torch.amp.autocast(\n            device_type=amp_device, dtype=dtype,\n            enabled=(active_device != torch.device('cpu'))\n        ):\n            out = model.generate(\n                **enc,\n                max_new_tokens=cfg.max_new_tokens,\n                length_penalty=cfg.length_penalty,\n                repetition_penalty=cfg.repetition_penalty,\n                **gen_kwargs,\n            )\n        decoded = tokenizer.batch_decode(out, skip_special_tokens=True)\n        bs = len(batch_texts)\n        return [decoded[i * n_return:(i + 1) * n_return] for i in range(bs)]\n\n    for start in tqdm(range(0, len(texts), cfg.batch_size), desc=f'{name}|seed={seed}', leave=False):\n        batch = texts[start:start + cfg.batch_size]\n        batch_cands = [[] for _ in batch]\n\n        # 1. Standard beam\n        try:\n            for i, c in enumerate(_run_batch(batch, cfg.n_beam_cands, {\n                'num_beams': cfg.num_beams,\n                'num_return_sequences': cfg.n_beam_cands,\n                'early_stopping': True,\n            })):\n                batch_cands[i].extend(c)\n        except Exception as e:\n            log.warning(f'Standard beam failed: {e}')\n\n        # 2. Diverse beam\n        if cfg.use_diverse_beam:\n            try:\n                n_groups = cfg.n_diverse_cands\n                for i, c in enumerate(_run_batch(batch, n_groups, {\n                    'num_beams': n_groups * 2,\n                    'num_beam_groups': n_groups,\n                    'diversity_penalty': cfg.diversity_penalty,\n                    'num_return_sequences': n_groups,\n                    'early_stopping': True,\n                })):\n                    batch_cands[i].extend(c)\n            except Exception as e:\n                log.warning(f'Diverse beam failed: {e}')\n\n        # 3. Multi-temperature sampling (seed affects this the most)\n        for temp in cfg.temperatures:\n            try:\n                for i, c in enumerate(_run_batch(batch, cfg.n_sample_per_temp, {\n                    'do_sample': True,\n                    'temperature': temp,\n                    'top_p': cfg.top_p,\n                    'num_return_sequences': cfg.n_sample_per_temp,\n                })):\n                    batch_cands[i].extend(c)\n            except Exception as e:\n                log.warning(f'Sampling temp={temp} failed: {e}')\n\n        for i in range(len(batch)):\n            all_cands[start + i].extend(batch_cands[i])\n\n    del model\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n\n    avg = np.mean([len(c) for c in all_cands])\n    log.info(f'  {name} seed={seed}: avg {avg:.1f} candidates/sample')\n    return all_cands\n\nprint('Candidate generator ready')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T17:03:30.223492Z","iopub.execute_input":"2026-03-23T17:03:30.223692Z","iopub.status.idle":"2026-03-23T17:03:30.237792Z","shell.execute_reply.started":"2026-03-23T17:03:30.223673Z","shell.execute_reply":"2026-03-23T17:03:30.236992Z"}},"outputs":[],"execution_count":null},{"id":"29d4d4fd-fe7d-4f69-9869-dc37e69e8f56","cell_type":"code","source":"# ── Multi-run inference → MBR → save  (single cell, no idle gaps) ────────────\nimport time, sys\n\nglobal_pool: List[List[str]] = [[] for _ in inputs_sorted]\n\n# ---- inference ----\nfor run_idx, seed in enumerate(cfg.run_seeds):\n    log.info(f'\\n{\"=\"*60}')\n    log.info(f'RUN {run_idx + 1}/{cfg.num_runs}  (seed={seed})')\n    log.info(f'{\"=\"*60}')\n    for model_idx, model_path in enumerate(cfg.model_paths):\n        log.info(f'  Model {model_idx + 1}/{len(cfg.model_paths)}: {Path(model_path).name}')\n        cands = generate_candidates(model_path, inputs_sorted, seed=seed)\n        for i, c in enumerate(cands):\n            global_pool[i].extend(c)\n    avg_pool = np.mean([len(c) for c in global_pool])\n    log.info(f'After run {run_idx + 1}: avg pool size = {avg_pool:.1f} candidates/sample')\n    sys.stdout.flush()\n\nlog.info('\\nAll runs complete. Running final MBR selection...')\n\n# ---- MBR + postprocess ----\nfinal_sorted = []\nt0 = time.time()\n\nfor idx, cands in enumerate(global_pool):\n    processed = [postprocess(c) for c in cands if c and c.strip()]\n    processed = [p for p in processed if p]\n    best = mbr_select(processed) if processed else ''\n    final_sorted.append(best)\n    # Print every 25 samples so Kaggle sees continuous activity\n    if (idx + 1) % 25 == 0 or (idx + 1) == len(global_pool):\n        elapsed = time.time() - t0\n        rate = (idx + 1) / elapsed\n        remaining = (len(global_pool) - idx - 1) / rate if rate > 0 else 0\n        log.info(f'MBR {idx+1}/{len(global_pool)} | {elapsed:.0f}s elapsed | ~{remaining:.0f}s remaining')\n        sys.stdout.flush()\n\nn_empty = sum(1 for t in final_sorted if not t)\navg_len = np.mean([len(t) for t in final_sorted if t])\nlog.info(f'MBR done | Translations: {len(final_sorted)} | Empty: {n_empty} | Avg length: {avg_len:.1f} chars')\n\n# ---- restore order and save ----\nrestore = np.argsort(order)\nfinal_translations = [final_sorted[restore[i]] for i in range(len(inputs))]\n\nsubmission = pd.DataFrame({'id': correct_ids, 'translation': final_translations})\nassert len(submission) == len(test_df),       'Row count mismatch!'\nassert list(submission['id']) == correct_ids, 'ID order mismatch!'\n\nsubmission.to_csv(cfg.out_path, index=False)\nlog.info(f'Saved: {cfg.out_path}')\n\nprint('\\nSample predictions:')\nfor _, row in submission.head(4).iterrows():\n    print(f'  id={row[\"id\"]}: {row[\"translation\"][:120]}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T17:03:30.238811Z","iopub.execute_input":"2026-03-23T17:03:30.239164Z","iopub.status.idle":"2026-03-23T17:07:36.087389Z","shell.execute_reply.started":"2026-03-23T17:03:30.239142Z","shell.execute_reply":"2026-03-23T17:07:36.086723Z"}},"outputs":[],"execution_count":null},{"id":"b5085e9d-b5a7-4bc9-8654-0d603c113905","cell_type":"markdown","source":"## Summary\n\n| Component | Setting |\n|---|---|\n| Models | final-byt5 + byt5-akkadian-mbr-v2 |\n| Runs | 3 (seeds 42, 1337, 2024) |\n| Candidates / model / run | 5 beam + 5 diverse + 6 sample = 16 |\n| Total pool / sample | 16 × 2 models × 3 runs = **96 candidates** |\n| MBR pool cap | 100 |\n| MBR metric | chrF++ (word_order=2) |\n| Postprocessing | 64-operation regex cleaner |\n| Estimated runtime | ~75–90 min on GPU |\n| Expected score | **35.9 – 36.3** |","metadata":{}}]}