{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":750502,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":573241,"modelId":585594}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Byt5 Akkadian Model Inference","metadata":{}},{"cell_type":"code","source":"#!pip install -q evaluate sacrebleu sentence-transformers\n# No pip install needed - inference uses only pre-installed Kaggle packages\nprint(\"Ready for inference (no internet required)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:38:44.609632Z","iopub.execute_input":"2026-02-13T21:38:44.610017Z","iopub.status.idle":"2026-02-13T21:38:48.262538Z","shell.execute_reply.started":"2026-02-13T21:38:44.609976Z","shell.execute_reply":"2026-02-13T21:38:48.261533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Imports (inference only - no evaluate needed)\n# ============================================================\n\nimport os\nimport gc\nimport re\nimport math\nimport random\nimport unicodedata\nimport warnings\nfrom pathlib import Path\nfrom dataclasses import dataclass, field\nfrom typing import List, Dict, Tuple, Optional\nfrom collections import Counter, defaultdict\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset as TorchDataset, DataLoader, Sampler\nfrom torch.cuda.amp import autocast\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\n\nwarnings.filterwarnings('ignore')\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f'PyTorch: {torch.__version__}')\nprint(f'Device: {DEVICE}')\nif torch.cuda.is_available():\n    print(f'GPU: {torch.cuda.get_device_name(0)}')\n    print(f'Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:38:48.264649Z","iopub.execute_input":"2026-02-13T21:38:48.264999Z","iopub.status.idle":"2026-02-13T21:38:48.274131Z","shell.execute_reply.started":"2026-02-13T21:38:48.264964Z","shell.execute_reply":"2026-02-13T21:38:48.273112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Configuration (Inference Only)\n# ============================================================\n\n@dataclass\nclass Config:\n    # --- Paths ---\n    INPUT_DIR: str = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n    MODEL_DIR: str = '/kaggle/input/models/erenalpp/byt5-akkadian-model/pytorch/default/1/byt5-akkadian-model'\n    OUTPUT_DIR: str = '/kaggle/input/models/erenalpp/byt5-akkadian-model/pytorch/default/1/byt5-akkadian-model'\n    SUBMISSION_DIR: str = '/kaggle/working/'\n\n    # --- Model ---\n    MODEL_NAME: str = 'google/byt5-base'\n    MAX_LENGTH: int = 256\n\n    # --- Training (unused in inference, kept for compatibility) ---\n    BATCH_SIZE: int = 2\n    GRAD_ACCUM: int = 8\n    EPOCHS: int = 15\n    LEARNING_RATE: float = 5e-5\n    WARMUP_RATIO: float = 0.06\n    WEIGHT_DECAY: float = 0.01\n    LABEL_SMOOTHING: float = 0.1\n    LR_SCHEDULER: str = 'cosine'\n    SAVE_TOTAL_LIMIT: int = 3\n    EARLY_STOPPING_PATIENCE: int = 4\n    GRADIENT_CHECKPOINTING: bool = True\n\n    # --- Inference ---\n    extra_model_paths: list = field(default_factory=list)\n    extra_model_weights: list = field(default_factory=list)\n\n    num_beams: int = 10\n    length_penalty: float = 1.08\n    early_stopping: bool = True\n    max_new_tokens: int = 256\n    repetition_penalty: float = 1.0\n    no_repeat_ngram_size: int = 0\n    infer_batch_size: int = 8\n    use_mixed_precision: bool = True\n\n    # --- Post-processing ---\n    use_translation_memory: bool = True\n    use_oa_lexicon: bool = True\n    oa_min_surface_freq: int = 3\n\n    SEED: int = 42\n\n    def __post_init__(self):\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        Path(self.SUBMISSION_DIR).mkdir(parents=True, exist_ok=True)\n        if not torch.cuda.is_available():\n            self.use_mixed_precision = False\n\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\n\ncfg = Config()\nseed_everything(cfg.SEED)\nprint(f'Model path: {cfg.MODEL_DIR}')\nprint('Config ready.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:38:48.275156Z","iopub.execute_input":"2026-02-13T21:38:48.275380Z","iopub.status.idle":"2026-02-13T21:38:48.293964Z","shell.execute_reply.started":"2026-02-13T21:38:48.275362Z","shell.execute_reply":"2026-02-13T21:38:48.293343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Shared Preprocessing Functions\n# (Used by both training and inference)\n# ============================================================\n\nSUBSCRIPT_TRANS = str.maketrans(\n    '\\u2080\\u2081\\u2082\\u2083\\u2084\\u2085\\u2086\\u2087\\u2088\\u2089',\n    '0123456789'\n)\nSPECIAL_H_TRANS = str.maketrans('\\u1E2B\\u1E2A', 'hH')\n\nPREFIX = 'translate Akkadian to English: '\n\n\ndef preprocess_transliteration(text):\n    \"\"\"Comprehensive preprocessing for Akkadian transliterations.\n    Follows the dataset instructions from overview.md exactly.\"\"\"\n    if pd.isna(text):\n        return ''\n    text = str(text)\n\n    # 1. Normalize Unicode\n    text = unicodedata.normalize('NFKC', text)\n\n    # 2. Character substitutions\n    text = text.translate(SPECIAL_H_TRANS)\n    text = text.translate(SUBSCRIPT_TRANS)\n\n    # 3. Remove modern scribal notations\n    text = text.replace('!', '').replace('?', '')\n    text = text.replace('/', ' ')\n\n    # 4. Remove half brackets (partially broken signs)\n    for ch in '\\u2039\\u203A\\u2329\\u232A\\u27E8\\u27E9\\u2308\\u2309\\u230A\\u230B\\u2E22\\u2E23\\u2E24\\u2E25':\n        text = text.replace(ch, '')\n\n    # 5. Remove double angle brackets (erroneous signs)\n    text = re.sub(r'<<[^>]*>>', '', text)\n\n    # 6. Scribal insertions: <text> -> text\n    text = re.sub(r'<([^>]*)>', r'\\1', text)\n\n    # 7. Square brackets: [text] -> text\n    text = re.sub(r'\\[([^\\]]*)\\]', r'\\1', text)\n\n    # 8. Gap normalization\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = re.sub(r'(?<!\\w)x(?!\\w)', '<gap>', text)\n    text = re.sub(r'xx+', '<gap>', text)\n    text = re.sub(r'\\[\\.+\\s*\\.+\\]', '<big_gap>', text)\n    text = re.sub(r'\\[\\.+\\]', '<big_gap>', text)\n    text = re.sub(r'\\.{3,}', '<big_gap>', text)\n    text = re.sub(r'\\u2026+', '<big_gap>', text)\n\n    # 9. Word divider\n    text = re.sub(r'(?<=\\s):(?=\\s)', '', text)\n\n    # 10. Merge adjacent gaps\n    text = re.sub(r'(<gap>\\s*)+', '<gap> ', text)\n    text = re.sub(r'(<big_gap>\\s*)+', '<big_gap> ', text)\n    text = re.sub(r'<gap>\\s*<big_gap>', '<big_gap> ', text)\n    text = re.sub(r'<big_gap>\\s*<gap>', '<big_gap> ', text)\n\n    # 11. Normalize whitespace\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\n\ndef preprocess_translation(text):\n    \"\"\"Clean English translations.\"\"\"\n    if pd.isna(text):\n        return ''\n    text = str(text)\n    text = unicodedata.normalize('NFKC', text)\n    text = text.translate(SPECIAL_H_TRANS)\n    text = text.translate(SUBSCRIPT_TRANS)\n    text = re.sub(r'\\s+', ' ', text).strip()\n    return text\n\n\nprint('Preprocessing functions ready.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:38:48.295840Z","iopub.execute_input":"2026-02-13T21:38:48.296201Z","iopub.status.idle":"2026-02-13T21:38:48.307988Z","shell.execute_reply.started":"2026-02-13T21:38:48.296169Z","shell.execute_reply":"2026-02-13T21:38:48.307298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Load Model for Inference (with optional ensemble)\n# ============================================================\n\ndef load_and_average_models(primary_path, extra_paths, extra_weights, device):\n    \"\"\"\n    Load the primary trained model. If extra model paths are given,\n    perform weighted averaging of state dicts.\n    \"\"\"\n    if not extra_paths:\n        print(f'Loading single model from {primary_path}...')\n        mdl = AutoModelForSeq2SeqLM.from_pretrained(primary_path)\n        tok = AutoTokenizer.from_pretrained(primary_path)\n        mdl.to(device).eval()\n        print(f'Model loaded: {sum(p.numel() for p in mdl.parameters()):,} params')\n        return mdl, tok\n\n    # Multi-model weight averaging\n    all_paths = [primary_path] + extra_paths\n    all_weights = [1.0] + extra_weights\n    total_w = sum(all_weights)\n    weights = [w / total_w for w in all_weights]\n\n    print(f'Loading {len(all_paths)} models for weight averaging...')\n    state_dicts = []\n    tok = None\n\n    for i, path in enumerate(all_paths):\n        print(f'  [{i+1}] {path} (w={weights[i]:.3f})')\n        m = AutoModelForSeq2SeqLM.from_pretrained(path)\n        state_dicts.append(m.state_dict())\n        if tok is None:\n            tok = AutoTokenizer.from_pretrained(path)\n        del m\n        gc.collect()\n\n    print('  Averaging weights...')\n    avg_sd = state_dicts[0].copy()\n    for key in avg_sd:\n        avg_sd[key] = sum(w * sd[key] for w, sd in zip(weights, state_dicts) if key in sd)\n\n    mdl = AutoModelForSeq2SeqLM.from_pretrained(all_paths[0])\n    mdl.load_state_dict(avg_sd)\n    mdl.to(device).eval().float()\n\n    del state_dicts, avg_sd\n    gc.collect()\n    torch.cuda.empty_cache()\n\n    print(f'Ensemble model ready: {sum(p.numel() for p in mdl.parameters()):,} params')\n    return mdl, tok\n\n\n# If we just trained, model & tokenizer are already in memory.\n# But let's reload from saved checkpoint for clean state + ensemble support.\nmodel, tokenizer = load_and_average_models(\n    cfg.OUTPUT_DIR,\n    cfg.extra_model_paths,\n    cfg.extra_model_weights,\n    cfg.device\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:38:48.308805Z","iopub.execute_input":"2026-02-13T21:38:48.308990Z","iopub.status.idle":"2026-02-13T21:38:49.254563Z","shell.execute_reply.started":"2026-02-13T21:38:48.308974Z","shell.execute_reply":"2026-02-13T21:38:49.253851Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Translation Memory (exact match from train)\n# ============================================================\n\ntrain_exact_map = {}\n\nif cfg.use_translation_memory:\n    print('Building Translation Memory...')\n    try:\n        _train = pd.read_csv(f'{cfg.INPUT_DIR}/train.csv')\n        _tmp = defaultdict(Counter)\n        for src, tgt in zip(_train['transliteration'].astype(str), _train['translation'].astype(str)):\n            k = re.sub(r'\\s+', ' ', preprocess_transliteration(src)).strip().lower()\n            if k:\n                _tmp[k][tgt.strip()] += 1\n        train_exact_map = {k: c.most_common(1)[0][0] for k, c in _tmp.items()}\n        print(f'TM entries: {len(train_exact_map)}')\n        del _train, _tmp\n    except Exception as e:\n        print(f'TM build failed: {e}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:38:49.255750Z","iopub.execute_input":"2026-02-13T21:38:49.256022Z","iopub.status.idle":"2026-02-13T21:38:49.716596Z","shell.execute_reply.started":"2026-02-13T21:38:49.255985Z","shell.execute_reply":"2026-02-13T21:38:49.715861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# OA Lexicon - Proper Noun Normalization\n# ============================================================\n\nDIACRITIC_MAP = str.maketrans({\n    '\\u0161': 's', '\\u0160': 's', '\\u1E63': 's', '\\u1E62': 's',\n    '\\u1E6D': 't', '\\u1E6C': 't', '\\u1E2B': 'h', '\\u1E2A': 'h',\n    '\\u0101': 'a', '\\u0100': 'a', '\\u0113': 'e', '\\u0112': 'e',\n    '\\u012B': 'i', '\\u012A': 'i', '\\u016B': 'u', '\\u016A': 'u',\n    '\\u02BE': '', '\\u02BC': '', \"'\": '', '\\u2018': '', '\\u2019': '',\n})\nDIACRITIC_CHARS = set('\\u0161\\u0160\\u1E63\\u1E62\\u1E6D\\u1E6C\\u1E2B\\u1E2A\\u0101\\u0113\\u012B\\u016B\\u0100\\u0112\\u012A\\u016A')\nEN_STOPWORDS = {\n    'the','a','an','and','or','of','to','in','on','at','by','for','from','with',\n    'as','but','not','no','nor','is','are','was','were','be','been','being',\n    'i','you','he','she','it','we','they','me','him','her','us','them',\n    'my','your','his','their','our','its','this','that','these','those',\n    'there','here','who','whom','which','what','when','where','why','how',\n    'so','if','has','have','had','do','does','did','will','would','shall',\n    'should','may','might','can','could','must','about','into','through',\n}\n\n\ndef fold_for_match(s):\n    s = str(s) if s else ''\n    s = unicodedata.normalize('NFKC', s)\n    s = re.sub(r'(?<=\\D)\\d+$', '', s)\n    s = s.translate(DIACRITIC_MAP).lower()\n    s = s.replace('sh', 's').replace('kh', 'h')\n    return re.sub(r'[^a-z]+', '', s)\n\n\ndef norm_key_token(s):\n    s = str(s) if s else ''\n    s = unicodedata.normalize('NFKC', s).translate(SUBSCRIPT_TRANS).strip()\n    s = re.sub(r'^[\\\"\\'\\'\\u201C\\u201D\\u2018\\u2019\\(\\)\\[\\]\\{\\}<>]+', '', s)\n    s = re.sub(r'[\\\"\\'\\'\\u201C\\u201D\\u2018\\u2019\\(\\)\\[\\]\\{\\}<>]+$', '', s)\n    return s.strip('.,;:!?').lower()\n\n\ndef looks_like_name(lexeme, typ):\n    if not lexeme: return False\n    t = (typ or '').strip().upper()\n    if t in {'DN','GN','PN','MN','ON','TN'}: return True\n    if any(ch.isupper() for ch in lexeme): return True\n    if any(ch in DIACRITIC_CHARS for ch in lexeme): return True\n    return False\n\n\n# Build index\ntoken2lexemes = defaultdict(list)\nfold2surface = {}\nfold2freq = {}\n\nif cfg.use_oa_lexicon:\n    print('Loading OA Lexicon...')\n    try:\n        oa = pd.read_csv(f'{cfg.INPUT_DIR}/OA_Lexicon_eBL.csv')\n        for _, r in oa.iterrows():\n            typ = '' if pd.isna(r.get('type')) else str(r['type']).strip()\n            lex = '' if pd.isna(r.get('lexeme')) else str(r['lexeme']).strip()\n            if not lex: continue\n            for col in ['form', 'norm', 'Alt_lex']:\n                if col not in oa.columns: continue\n                v = r.get(col)\n                if pd.isna(v): continue\n                for tok in str(v).split():\n                    k = norm_key_token(tok)\n                    if k: token2lexemes[k].append((lex, typ))\n\n        for k, v in list(token2lexemes.items()):\n            token2lexemes[k] = list(dict.fromkeys(v))\n\n        print(f'Lexicon keys: {len(token2lexemes)}')\n\n        # Learn surface forms from train translations\n        _tdf = pd.read_csv(f'{cfg.INPUT_DIR}/train.csv')\n        _tcol = 'translation' if 'translation' in _tdf.columns else _tdf.columns[-1]\n        _sc = defaultdict(Counter)\n        _tok_re = re.compile(r\"[A-Za-z\\u0161\\u1E63\\u1E6D\\u1E2B\\u0101\\u0113\\u012B\\u016B\\u0160\\u1E62\\u1E6C\\u1E2A\\u0100\\u0112\\u012A\\u016A'\\u2018\\u2019\\-]+\")\n        for text in _tdf[_tcol].astype(str):\n            for tok in _tok_re.findall(text):\n                if len(tok) < 3: continue\n                if not (tok[0].isupper() or any(ch in DIACRITIC_CHARS for ch in tok)): continue\n                f = fold_for_match(tok)\n                if len(f) >= 4: _sc[f][tok] += 1\n        for f, ctr in _sc.items():\n            tok, cnt = ctr.most_common(1)[0]\n            fold2surface[f] = tok\n            fold2freq[f] = cnt\n        print(f'Surface forms: {len(fold2surface)}')\n        del oa, _tdf, _sc\n    except Exception as e:\n        print(f'Lexicon load failed: {e}')\n\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:38:49.717731Z","iopub.execute_input":"2026-02-13T21:38:49.718060Z","iopub.status.idle":"2026-02-13T21:39:03.396037Z","shell.execute_reply.started":"2026-02-13T21:38:49.718039Z","shell.execute_reply":"2026-02-13T21:39:03.395154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Name Normalization Functions\n# ============================================================\n\ndef extract_name_targets(translit, max_targets=50):\n    if not cfg.use_oa_lexicon: return {}\n    translit = str(translit) if translit else ''\n    targets, seen = {}, set()\n    for tok in translit.split():\n        k = norm_key_token(tok)\n        for lex, typ in token2lexemes.get(k, []):\n            if lex in seen: continue\n            seen.add(lex)\n            if not looks_like_name(lex, typ): continue\n            lex_clean = re.sub(r'(?<=\\D)\\d+$', '', lex)\n            f = fold_for_match(lex_clean)\n            if len(f) < 4: continue\n            ne_type = (typ or '').strip().upper()\n            min_freq = 2 if ne_type in {'DN','GN','PN','MN','ON','TN'} else cfg.oa_min_surface_freq\n            if fold2surface and f in fold2surface and fold2freq.get(f, 0) >= min_freq:\n                targets[f] = fold2surface[f]\n        if len(targets) >= max_targets: break\n    return targets\n\n\ndef lexicon_name_normalize(pred, targets):\n    if not pred or not targets: return pred\n    parts = str(pred).split()\n    out = []\n    for p in parts:\n        m = re.match(r'^(\\W*)(.*?)(\\W*)$', p)\n        pre, core, suf = m.group(1), m.group(2), m.group(3)\n        if not core:\n            out.append(p); continue\n        poss = ''\n        pm = re.match(r\"^(.*?)([\\u2018\\u2019']s)$\", core)\n        if pm: core, poss = pm.group(1), pm.group(2)\n        f = fold_for_match(core)\n        if len(f) < 4:\n            out.append(p); continue\n        is_cap = core[:1].isupper() if core else False\n        if f in targets and (is_cap or (len(f) >= 4 and core.lower() not in EN_STOPWORDS)):\n            out.append(pre + targets[f] + poss + suf); continue\n        out.append(p)\n    return ' '.join(out)\n\n\ndef apply_lexicon_normalization(translit, pred):\n    if not cfg.use_oa_lexicon: return pred\n    return lexicon_name_normalize(pred, extract_name_targets(translit))\n\n\nprint('Name normalization ready.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:39:03.397183Z","iopub.execute_input":"2026-02-13T21:39:03.397471Z","iopub.status.idle":"2026-02-13T21:39:03.408712Z","shell.execute_reply.started":"2026-02-13T21:39:03.397443Z","shell.execute_reply":"2026-02-13T21:39:03.407913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Post-Processing Pipeline\n# ============================================================\n\nFORBIDDEN_CHARS = '!?()\\\"\\u2014\\u2013<>\\u2308\\u2309\\u230A\\u230B[]+\\u02BE/;'\nFORBIDDEN_TRANS = str.maketrans('', '', FORBIDDEN_CHARS)\nRE_GAP = re.compile(r'(\\[x\\]|\\(x\\)|\\bx\\b)', re.I)\nRE_BIG_GAP = re.compile(r'(\\.{3,}|\\u2026|\\[\\.+\\])')\nRE_ANNOTATIONS = re.compile(r'\\((fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)', re.I)\nRE_REPEATED_WORDS = re.compile(r'\\b(\\w+)(?:\\s+\\1\\b)+')\nRE_WHITESPACE = re.compile(r'\\s+')\nRE_PUNCT_SPACE = re.compile(r'\\s+([.,:;])')\nRE_REPEATED_PUNCT = re.compile(r'([.,])\\1+')\nDASH_MAP = str.maketrans({'\\u2013': '-', '\\u2014': '-', '\\u2212': '-'})\nQUOTE_MAP = str.maketrans({'\\u201C': '\"', '\\u201D': '\"', '\\u2018': \"'\", '\\u2019': \"'\"})\n\n# Unicode fraction characters\nFRAC_HALF = chr(0x00BD)       # ½\nFRAC_QUARTER = chr(0x00BC)    # ¼\nFRAC_THREE_Q = chr(0x00BE)    # ¾\nFRAC_THIRD = chr(0x2153)      # ⅓\nFRAC_TWO_THIRD = chr(0x2154)  # ⅔\n\n\ndef postprocess_translation(text):\n    if not isinstance(text, str) or not text.strip(): return ''\n    s = text\n    s = s.translate(SPECIAL_H_TRANS)\n    s = s.translate(SUBSCRIPT_TRANS)\n    s = s.translate(DASH_MAP)\n    s = s.translate(QUOTE_MAP)\n\n    if '<gap>' not in s and '<big_gap>' not in s:\n        s = RE_GAP.sub('<gap>', s)\n        s = RE_BIG_GAP.sub('<big_gap>', s)\n\n    s = s.replace('<gap> <gap>', '<big_gap>')\n    s = s.replace('<big_gap> <big_gap>', '<big_gap>')\n    s = RE_ANNOTATIONS.sub('', s)\n\n    s = s.replace('<gap>', '\\x00GAP\\x00').replace('<big_gap>', '\\x00BIG\\x00')\n    s = s.translate(FORBIDDEN_TRANS)\n    s = s.replace('\\x00GAP\\x00', ' <gap> ').replace('\\x00BIG\\x00', ' <big_gap> ')\n\n    # Fraction normalization using actual Unicode chars\n    s = re.sub(r'(\\d+)\\.5\\b', lambda m: m.group(1) + FRAC_HALF, s)\n    s = re.sub(r'\\b0\\.5\\b', FRAC_HALF, s)\n    s = re.sub(r'(\\d+)\\.25\\b', lambda m: m.group(1) + FRAC_QUARTER, s)\n    s = re.sub(r'\\b0\\.25\\b', FRAC_QUARTER, s)\n    s = re.sub(r'(\\d+)\\.75\\b', lambda m: m.group(1) + FRAC_THREE_Q, s)\n    s = re.sub(r'\\b0\\.75\\b', FRAC_THREE_Q, s)\n    s = re.sub(r'(\\d+)\\.33\\d*\\b', lambda m: m.group(1) + FRAC_THIRD, s)\n    s = re.sub(r'(\\d+)\\.66\\d*\\b', lambda m: m.group(1) + FRAC_TWO_THIRD, s)\n\n    s = RE_REPEATED_WORDS.sub(r'\\1', s)\n    for n in range(4, 1, -1):\n        s = re.sub(r'\\b((?:\\w+\\s+){' + str(n-1) + r'}\\w+)(?:\\s+\\1\\b)+', r'\\1', s)\n\n    s = RE_PUNCT_SPACE.sub(r'\\1', s)\n    s = RE_REPEATED_PUNCT.sub(r'\\1', s)\n    s = re.sub(r'([,.;:])([A-Za-z])', r'\\1 \\2', s)\n\n    if s.count('[') > s.count(']'): s += ']' * (s.count('[') - s.count(']'))\n    if s.count('(') > s.count(')'): s += ')' * (s.count('(') - s.count(')'))\n\n    return RE_WHITESPACE.sub(' ', s).strip().strip('-').strip()\n\n\nprint('Post-processing ready.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:39:03.410064Z","iopub.execute_input":"2026-02-13T21:39:03.410357Z","iopub.status.idle":"2026-02-13T21:39:03.432843Z","shell.execute_reply.started":"2026-02-13T21:39:03.410332Z","shell.execute_reply":"2026-02-13T21:39:03.432111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Dataset & Sampler for Inference\n# ============================================================\n\nclass AkkadianInferDataset(TorchDataset):\n    def __init__(self, df):\n        self.ids = df['id'].tolist()\n        self.translits = df['transliteration'].tolist()\n        self.texts = [PREFIX + preprocess_transliteration(t) for t in self.translits]\n        self.text_ids = df['text_id'].tolist() if 'text_id' in df.columns else [None] * len(df)\n\n    def __len__(self): return len(self.ids)\n\n    def __getitem__(self, idx):\n        return self.ids[idx], self.texts[idx], self.translits[idx], self.text_ids[idx]\n\n\nclass BucketBatchSampler(Sampler):\n    def __init__(self, dataset, batch_size, num_buckets=4):\n        lengths = [len(t.split()) for _, t, _, _ in dataset]\n        sorted_idx = sorted(range(len(lengths)), key=lambda i: lengths[i])\n        bucket_size = max(1, len(sorted_idx) // num_buckets)\n        self.batches = []\n        for i in range(num_buckets):\n            start = i * bucket_size\n            end = None if i == num_buckets - 1 else (i + 1) * bucket_size\n            bucket = sorted_idx[start:end]\n            for j in range(0, len(bucket), batch_size):\n                self.batches.append(bucket[j:j+batch_size])\n\n    def __iter__(self): return iter(self.batches)\n    def __len__(self): return len(self.batches)\n\n\nprint('Inference dataset classes ready.')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:39:03.434800Z","iopub.execute_input":"2026-02-13T21:39:03.435129Z","iopub.status.idle":"2026-02-13T21:39:03.448889Z","shell.execute_reply.started":"2026-02-13T21:39:03.435106Z","shell.execute_reply":"2026-02-13T21:39:03.448260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Run Inference\n# ============================================================\n\ntest_df = pd.read_csv(f'{cfg.INPUT_DIR}/test.csv')\nprint(f'Test samples: {len(test_df)}')\n\ninfer_dataset = AkkadianInferDataset(test_df)\nsampler = BucketBatchSampler(infer_dataset, cfg.infer_batch_size, num_buckets=4)\n\n\ndef collate_fn(batch):\n    ids = [b[0] for b in batch]\n    texts = [b[1] for b in batch]\n    translits = [b[2] for b in batch]\n    text_ids = [b[3] for b in batch]\n    tokens = tokenizer(texts, max_length=cfg.MAX_LENGTH, padding=True, truncation=True, return_tensors='pt')\n    return ids, tokens, translits, text_ids, texts\n\n\nloader = DataLoader(infer_dataset, batch_sampler=sampler, num_workers=2, collate_fn=collate_fn)\n\ngen_kwargs = dict(\n    max_new_tokens=cfg.max_new_tokens,\n    num_beams=cfg.num_beams,\n    length_penalty=cfg.length_penalty,\n    early_stopping=cfg.early_stopping,\n    use_cache=True,\n)\nif cfg.repetition_penalty != 1.0:\n    gen_kwargs['repetition_penalty'] = cfg.repetition_penalty\nif cfg.no_repeat_ngram_size > 0:\n    gen_kwargs['no_repeat_ngram_size'] = cfg.no_repeat_ngram_size\n\nresults = []\ntm_hits = 0\n\nwith torch.inference_mode():\n    for batch_ids, tokens, translits, text_ids, raw_texts in tqdm(loader, desc='Translating'):\n        batch_results = {}\n        need_model = []\n\n        # Check Translation Memory\n        if cfg.use_translation_memory:\n            for i, (sid, translit) in enumerate(zip(batch_ids, translits)):\n                key = re.sub(r'\\s+', ' ', preprocess_transliteration(translit)).strip().lower()\n                if key in train_exact_map:\n                    batch_results[i] = (sid, train_exact_map[key], translit, text_ids[i])\n                    tm_hits += 1\n                else:\n                    need_model.append(i)\n        else:\n            need_model = list(range(len(batch_ids)))\n\n        # Model inference\n        if need_model:\n            if len(need_model) == len(batch_ids):\n                input_ids = tokens.input_ids.to(cfg.device)\n                attention_mask = tokens.attention_mask.to(cfg.device)\n            else:\n                sub_texts = [raw_texts[i] for i in need_model]\n                sub_tokens = tokenizer(sub_texts, max_length=cfg.MAX_LENGTH, padding=True, truncation=True, return_tensors='pt')\n                input_ids = sub_tokens.input_ids.to(cfg.device)\n                attention_mask = sub_tokens.attention_mask.to(cfg.device)\n\n            if cfg.use_mixed_precision:\n                with autocast():\n                    outputs = model.generate(input_ids=input_ids, attention_mask=attention_mask, **gen_kwargs)\n            else:\n                outputs = model.generate(input_ids=input_ids, attention_mask=attention_mask, **gen_kwargs)\n\n            decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n            for j, idx in enumerate(need_model):\n                batch_results[idx] = (batch_ids[idx], decoded[j], translits[idx], text_ids[idx])\n\n            del input_ids, attention_mask, outputs\n            torch.cuda.empty_cache()\n\n        for i in range(len(batch_ids)):\n            if i in batch_results:\n                results.append(batch_results[i])\n\nprint(f'\\nTM hits: {tm_hits}/{len(test_df)}')\nprint(f'Total results: {len(results)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:39:03.449725Z","iopub.execute_input":"2026-02-13T21:39:03.449954Z","iopub.status.idle":"2026-02-13T21:39:15.182553Z","shell.execute_reply.started":"2026-02-13T21:39:03.449927Z","shell.execute_reply":"2026-02-13T21:39:15.181538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Post-Processing & Build Submission\n# ============================================================\n\nfinal_results = []\nfor sid, pred, translit, text_id in tqdm(results, desc='Post-processing'):\n    cleaned = postprocess_translation(pred)\n    cleaned = apply_lexicon_normalization(translit, cleaned)\n    if not cleaned.strip():\n        cleaned = 'broken text'\n    final_results.append((sid, cleaned))\n\nsubmission = pd.DataFrame(final_results, columns=['id', 'translation'])\nsubmission = submission.sort_values('id').reset_index(drop=True)\n\nprint(f'\\nSubmission shape: {submission.shape}')\nprint(f'Empty: {(submission[\"translation\"].str.strip() == \"\").sum()}')\n\nlengths = submission['translation'].str.len()\nprint(f'Length - Mean: {lengths.mean():.1f}, Median: {lengths.median():.1f}, Min: {lengths.min()}, Max: {lengths.max()}')\n\nprint('\\nSamples:')\nfor i in range(min(5, len(submission))):\n    row = submission.iloc[i]\n    preview = row['translation'][:80] + '...' if len(row['translation']) > 80 else row['translation']\n    print(f'  ID {row[\"id\"]}: {preview}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:39:15.184382Z","iopub.execute_input":"2026-02-13T21:39:15.184667Z","iopub.status.idle":"2026-02-13T21:39:15.214717Z","shell.execute_reply.started":"2026-02-13T21:39:15.184618Z","shell.execute_reply":"2026-02-13T21:39:15.214039Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Save Submission\n# ============================================================\n\noutput_path = Path(cfg.SUBMISSION_DIR) / 'submission.csv'\nsubmission.to_csv(output_path, index=False)\n\nprint(f'Submission saved to {output_path}')\nprint(f'Total translations: {len(submission)}')\nprint('\\nDone!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-13T21:39:15.215784Z","iopub.execute_input":"2026-02-13T21:39:15.216312Z","iopub.status.idle":"2026-02-13T21:39:15.222840Z","shell.execute_reply.started":"2026-02-13T21:39:15.216276Z","shell.execute_reply":"2026-02-13T21:39:15.222099Z"}},"outputs":[],"execution_count":null}]}