{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ════════════════════════════════════════════════════════════════════════════\n# CELL 1 ─ IMPORTS\n# ════════════════════════════════════════════════════════════════════════════\n# Deep Past Challenge: Akkadian → English   v5.0\n#\n# FIXES vs v4 (which crashed on hidden test / scored 0.0):\n#   ✓ Removed ALL assert statements  → no crash on unexpected hidden data\n#   ✓ Wrap generation in try/except  → graceful fallback per batch\n#   ✓ Train on FULL dataset          → no val split for final submission\n#     (val used only to track training, not held out from training data)\n#   ✓ Robust ID handling             → str IDs supported, no int() crash\n#   ✓ NaN source handling            → every row gets a prediction\n#   ✓ Single-sentence output         → truncate at first sentence boundary\n#   ✓ max_new_tokens only            → no max_length conflict (newer transformers)\n#   ✓ Graceful column detection      → handles any test schema\n#   ✓ Verified submission format     → exactly {id, translation}\n# ════════════════════════════════════════════════════════════════════════════\n\nimport os, re, gc, math, random, warnings, logging, time\nfrom dataclasses import dataclass, field\nfrom collections import defaultdict, Counter\nfrom typing import List, Dict, Optional\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport transformers\nfrom transformers import (\n    MarianTokenizer, MarianMTModel,\n    DataCollatorForSeq2Seq,\n    get_cosine_schedule_with_warmup,\n)\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import train_test_split\n\nwarnings.filterwarnings('ignore')\nlogging.getLogger('transformers').setLevel(logging.ERROR)\nos.environ['TOKENIZERS_PARALLELISM'] = 'false'\n\nSEED   = 42\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nHAS_GPU = (DEVICE == 'cuda')\n\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\nif HAS_GPU: torch.cuda.manual_seed_all(SEED)\n\nprint(\"=\" * 64)\nprint(\"  DEEP PAST CHALLENGE — v5.0  (crash-proof + full data)\")\nprint(\"=\" * 64)\nprint(f\"  Device : {DEVICE}\")\nprint(f\"  Trans  : {transformers.__version__}\")\nprint(f\"  Torch  : {torch.__version__}\")\nprint(\"=\" * 64)\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 2 ─ CONFIGURATION\n# ════════════════════════════════════════════════════════════════════════════\n\n@dataclass\nclass Config:\n    DATA_DIR : str = '/kaggle/input/competitions/deep-past-initiative-machine-translation'\n    MDL_DIR  : str = '/kaggle/working/models'\n    SUB_FILE : str = '/kaggle/working/submission.csv'\n\n    # Best available Semitic→English MarianMT models\n    MARIAN_CANDIDATES: List[str] = field(default_factory=lambda: [\n        'Helsinki-NLP/opus-mt-ar-en',\n        'Helsinki-NLP/opus-mt-he-en',\n        'Helsinki-NLP/opus-mt-mul-en',\n    ])\n\n    # GPU settings\n    EPOCHS_GPU  : int   = 15\n    LR_GPU      : float = 3e-4\n    BATCH_GPU   : int   = 16\n    GACC_GPU    : int   = 2\n    MAX_LEN_GPU : int   = 256\n\n    # CPU settings  (~20 min)\n    EPOCHS_CPU  : int   = 6\n    LR_CPU      : float = 4e-4\n    BATCH_CPU   : int   = 8\n    GACC_CPU    : int   = 2\n    MAX_LEN_CPU : int   = 128\n\n    WARMUP_RATIO : float = 0.08\n    WEIGHT_DECAY : float = 0.01\n    LABEL_SMOOTH : float = 0.10\n    GRAD_CLIP    : float = 1.0\n\n    # Validation  (track progress; FULL data used for training)\n    VAL_RATIO    : float = 0.10\n    QUICK_VAL_N  : int   = 50\n    EVAL_EVERY   : int   = 2\n\n    # Decoding\n    TRAIN_BEAMS : int   = 1      # greedy during training\n    TEST_BEAMS  : int   = 5      # full beam for final output\n    LEN_PENALTY : float = 1.2\n    NO_RPT_NGR  : int   = 4\n    RPT_PEN     : float = 1.3\n    GEN_TIMEOUT : int   = 300    # seconds per generation batch before skip\n\n    SEED   : int = SEED\n    DEVICE : str = DEVICE\n\n    def __post_init__(self):\n        os.makedirs(self.MDL_DIR, exist_ok=True)\n        if HAS_GPU:\n            self.EPOCHS  = self.EPOCHS_GPU;  self.LR     = self.LR_GPU\n            self.BATCH   = self.BATCH_GPU;   self.GACC   = self.GACC_GPU\n            self.MAX_LEN = self.MAX_LEN_GPU; self.BINFER = 32\n        else:\n            self.EPOCHS  = self.EPOCHS_CPU;  self.LR     = self.LR_CPU\n            self.BATCH   = self.BATCH_CPU;   self.GACC   = self.GACC_CPU\n            self.MAX_LEN = self.MAX_LEN_CPU; self.BINFER = 16\n\nCFG = Config()\nprint(f\"  Epochs={CFG.EPOCHS}  LR={CFG.LR}  \"\n      f\"Batch={CFG.BATCH}x{CFG.GACC}={CFG.BATCH*CFG.GACC}  \"\n      f\"MaxLen={CFG.MAX_LEN}\")\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 3 ─ PREPROCESSOR\n# ════════════════════════════════════════════════════════════════════════════\n\nclass Prep:\n    \"\"\"Akkadian transliteration preprocessor (competition Dataset Instructions).\"\"\"\n\n    _SUB = str.maketrans({chr(0x2080+i): str(i) for i in range(10)} |\n                         {chr(0x208A): 'x'})\n    _CH  = str.maketrans({\n        '\\u1e2a':'H',  '\\u1e2b':'h',   # Hh/hh -> H/h  CRITICAL\n        '\\u0161':'sh', '\\u0160':'SH',\n        '\\u1e63':'ts', '\\u1e62':'TS',\n        '\\u1e6d':'th', '\\u1e6c':'TH',\n        '\\u02be':'',   '\\u02bf':'',\n        '\\u0101':'a',  '\\u0100':'A',\n        '\\u0113':'e',  '\\u0112':'E',\n        '\\u012b':'i',  '\\u012a':'I',\n        '\\u016b':'u',  '\\u016a':'U',\n        '\\xe1':'a2','\\xe0':'a3','\\xe9':'e2','\\xe8':'e3',\n        '\\xed':'i2','\\xec':'i3','\\xfa':'u2','\\xf9':'u3',\n    })\n    _D   = re.compile(r'<<[^>]*>>')\n    _S   = re.compile(r'[!?/]')\n    _H   = re.compile(r'[\\u02f9\\u02fa\\u2308\\u2309]')\n    _A   = re.compile(r'[<>]')\n    _GB  = re.compile(r'\\[[^\\]]*\\.\\.\\.[^\\]]*\\]|\\.\\.\\.|[\\u2026]')\n    _GS  = re.compile(r'\\[\\s*[xX?]\\s*\\]')\n    _SQ  = re.compile(r'[\\[\\]]')\n    _DT  = re.compile(r'\\{[^}]*\\}')\n    _DV  = re.compile(r'(?<!\\d):(?!\\d)')\n    _LN  = re.compile(r'(?m)^\\s*\\d+\\'{0,2}\\s+')\n\n    def src(self, t: str) -> str:\n        t = self._D.sub(' ', t);   t = self._S.sub('', t)\n        t = self._H.sub('', t);    t = self._A.sub('', t)\n        t = self._GB.sub(' <big_gap> ', t)\n        t = self._GS.sub(' <gap> ', t)\n        t = self._SQ.sub('', t);   t = self._DT.sub('', t)\n        t = self._DV.sub(' ', t);  t = self._LN.sub('', t)\n        t = t.translate(self._SUB); t = t.translate(self._CH)\n        return re.sub(r'\\s+', ' ', t).strip() or '<gap>'\n\n    def tgt(self, t: str) -> str:\n        t = re.sub(r'\\s+', ' ', t).strip()\n        return re.sub(r'\\s+([.,;:!?])', r'\\1', t)\n\n    def __call__(self, text, mode='source') -> str:\n        if text is None or (isinstance(text, float) and math.isnan(text)):\n            return '<gap>'\n        return self.src(str(text)) if mode == 'source' else self.tgt(str(text))\n\nprep = Prep()\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 4 ─ METRICS\n# ════════════════════════════════════════════════════════════════════════════\n\nclass Metrics:\n    @staticmethod\n    def wng(s, n):\n        w = s.split()\n        return Counter(tuple(w[i:i+n]) for i in range(max(0, len(w)-n+1)))\n\n    @staticmethod\n    def cng(s, n):\n        return Counter(s[i:i+n] for i in range(max(0, len(s)-n+1)))\n\n    @classmethod\n    def bleu(cls, P, R, mx=4):\n        st = defaultdict(int)\n        for p, r in zip(P, R):\n            pw, rw = p.split(), r.split()\n            st['h'] += len(pw); st['r'] += len(rw)\n            for n in range(1, mx+1):\n                pn, rn = cls.wng(p,n), cls.wng(r,n)\n                st[f'm{n}'] += sum((pn&rn).values())\n                st[f't{n}'] += max(len(pw)-n+1, 0)\n        bp = 1.0 if st['h'] >= st['r'] else math.exp(1-st['r']/max(st['h'],1))\n        lp = 0.0\n        for n in range(1, mx+1):\n            if not st[f't{n}'] or not st[f'm{n}']: return 0.0\n            lp += math.log(st[f'm{n}']/st[f't{n}'])\n        return 100.0 * bp * math.exp(lp/mx)\n\n    @classmethod\n    def chrf(cls, P, R, co=6, wo=2, beta=2.0):\n        st = defaultdict(float)\n        for p, r in zip(P, R):\n            for n in range(1, co+1):\n                pn,rn = cls.cng(p,n),cls.cng(r,n); m=sum((pn&rn).values())\n                st[f'cm{n}']+=m; st[f'cp{n}']+=sum(pn.values()); st[f'cr{n}']+=sum(rn.values())\n            for n in range(1, wo+1):\n                pn,rn = cls.wng(p,n),cls.wng(r,n); m=sum((pn&rn).values())\n                st[f'wm{n}']+=m; st[f'wp{n}']+=sum(pn.values()); st[f'wr{n}']+=sum(rn.values())\n        fs = []\n        for pfx,mx in [('c',co),('w',wo)]:\n            for n in range(1,mx+1):\n                m,p,r=st[f'{pfx}m{n}'],st[f'{pfx}p{n}'],st[f'{pfx}r{n}']\n                prec=m/p if p else 0.0; rec=m/r if r else 0.0\n                d=beta**2*prec+rec\n                fs.append((1+beta**2)*prec*rec/d if d else 0.0)\n        return 100.0*(sum(fs)/len(fs)) if fs else 0.0\n\n    @classmethod\n    def score(cls, P, R):\n        b=cls.bleu(P,R); c=cls.chrf(P,R)\n        return {'bleu':round(b,4),'chrf':round(c,4),\n                'gm':round(math.sqrt(b*c) if b>0 and c>0 else 0.0, 4)}\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 5 ─ DATASET\n# ════════════════════════════════════════════════════════════════════════════\n\nclass DS(Dataset):\n    \"\"\"No deprecated APIs. Direct tokenizer() call for labels.\"\"\"\n    def __init__(self, srcs, tgts, tok, max_len):\n        self.srcs=srcs; self.tgts=tgts; self.tok=tok; self.max=max_len\n    def __len__(self): return len(self.srcs)\n    def __getitem__(self, i):\n        enc = self.tok(self.srcs[i], max_length=self.max,\n                       truncation=True, padding=False)\n        item = {'input_ids': enc['input_ids'],\n                'attention_mask': enc['attention_mask']}\n        if self.tgts is not None:\n            lbl = self.tok(self.tgts[i], max_length=self.max,\n                           truncation=True, padding=False)\n            item['labels'] = lbl['input_ids']\n        return item\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 6 ─ TRAINER\n# ════════════════════════════════════════════════════════════════════════════\n\nclass Trainer:\n    def __init__(self, model, tok, cfg, name):\n        self.model=model; self.tok=tok; self.cfg=cfg; self.name=name\n        self.best=-1.0\n        self.ckpt=os.path.join(cfg.MDL_DIR, f'best_{name}')\n        os.makedirs(self.ckpt, exist_ok=True)\n\n    def _opt(self, lr):\n        no_wd={'bias','LayerNorm.weight','layer_norm.weight',\n               'layernorm.weight','final_layer_norm.weight'}\n        return torch.optim.AdamW([\n            {'params':[p for n,p in self.model.named_parameters()\n                       if not any(x in n for x in no_wd)],\n             'weight_decay':self.cfg.WEIGHT_DECAY},\n            {'params':[p for n,p in self.model.named_parameters()\n                       if any(x in n for x in no_wd)],\n             'weight_decay':0.0}], lr=lr, eps=1e-8)\n\n    @staticmethod\n    def _ce(logits, labels, eps=0.1):\n        V    = logits.size(-1)\n        logp = torch.nn.functional.log_softmax(logits.float(), dim=-1)\n        mask = labels != -100\n        safe = labels.clamp(min=0)      # avoid -100 crash in gather\n        nll  = -logp.gather(-1, safe.unsqueeze(-1)).squeeze(-1)\n        smo  = -logp.sum(-1) / V\n        return ((1-eps)*nll + eps*smo)[mask].mean()\n\n    def _epoch(self, loader, opt, sched, ep):\n        self.model.train(); tot=0.0; n=0; opt.zero_grad()\n        pad = self.tok.pad_token_id or 0\n        pbar = tqdm(loader, desc=f\"  Ep{ep:02d}\", leave=False,\n                    mininterval=30, miniters=50)\n        for step, batch in enumerate(pbar):\n            inp  = batch['input_ids'].to(DEVICE)\n            attn = batch['attention_mask'].to(DEVICE)\n            lbls = batch['labels'].to(DEVICE)\n            lbls[lbls==pad] = -100\n            out  = self.model(input_ids=inp, attention_mask=attn, labels=lbls)\n            loss = self._ce(out.logits, lbls, self.cfg.LABEL_SMOOTH)\n            (loss/self.cfg.GACC).backward()\n            tot+=loss.item(); n+=1\n            if (step+1)%self.cfg.GACC==0:\n                torch.nn.utils.clip_grad_norm_(\n                    self.model.parameters(), self.cfg.GRAD_CLIP)\n                opt.step(); sched.step(); opt.zero_grad()\n            if n%50==0: pbar.set_postfix(loss=f'{tot/n:.4f}')\n        return tot/max(n,1)\n\n    @torch.no_grad()\n    def _gen(self, srcs, beams=1):\n        self.model.eval(); out=[]\n        for i in range(0, len(srcs), self.cfg.BINFER):\n            batch = srcs[i:i+self.cfg.BINFER]\n            try:\n                enc = self.tok(batch, return_tensors='pt', padding=True,\n                               truncation=True,\n                               max_length=self.cfg.MAX_LEN).to(DEVICE)\n                kw = dict(max_new_tokens=self.cfg.MAX_LEN, num_beams=beams,\n                          early_stopping=(beams>1))\n                if beams>1:\n                    kw.update(length_penalty=self.cfg.LEN_PENALTY,\n                              no_repeat_ngram_size=self.cfg.NO_RPT_NGR,\n                              repetition_penalty=self.cfg.RPT_PEN)\n                preds = self.model.generate(**enc, **kw)\n                out.extend(self.tok.batch_decode(preds,\n                                                 skip_special_tokens=True))\n            except Exception as e:\n                print(f\"    [gen warning batch {i//self.cfg.BINFER}]: {e}\")\n                out.extend([\"\"] * len(batch))\n        return out\n\n    def fit(self, tr_ds, va_src, va_ref, lr):\n        collator = DataCollatorForSeq2Seq(\n            self.tok, model=self.model,\n            label_pad_token_id=-100, padding=True)\n        loader = DataLoader(tr_ds, batch_size=self.cfg.BATCH,\n                            shuffle=True, collate_fn=collator, num_workers=0)\n        opt   = self._opt(lr)\n        total = (len(loader)//self.cfg.GACC)*self.cfg.EPOCHS\n        warm  = max(1, int(total*self.cfg.WARMUP_RATIO))\n        sched = get_cosine_schedule_with_warmup(opt, warm, total)\n\n        n_q = min(self.cfg.QUICK_VAL_N, len(va_src))\n        qs, qr = va_src[:n_q], va_ref[:n_q]\n\n        print(f\"\\n  [{self.name}]  Train={len(tr_ds)}  \"\n              f\"Val(quick)={n_q}  Steps={total}  LR={lr}\")\n\n        for ep in range(1, self.cfg.EPOCHS+1):\n            t0   = time.time()\n            loss = self._epoch(loader, opt, sched, ep)\n            gp   = self._gen(qs, beams=1)\n            gm_q = Metrics.score(gp, qr)\n            dt   = time.time()-t0\n            flag = ''\n\n            if ep%self.cfg.EVAL_EVERY==0 or ep==self.cfg.EPOCHS:\n                fp  = self._gen(va_src, beams=self.cfg.TRAIN_BEAMS)\n                fm  = Metrics.score(fp, va_ref)\n                if fm['gm']>self.best:\n                    self.best=fm['gm']\n                    self.model.save_pretrained(self.ckpt)\n                    self.tok.save_pretrained(self.ckpt)\n                    flag='  BEST'\n                print(f\"  Ep{ep:02d} loss={loss:.4f}  \"\n                      f\"quick={gm_q['gm']:.3f}  val={fm['gm']:.4f}  \"\n                      f\"{dt:.0f}s{flag}\")\n            else:\n                print(f\"  Ep{ep:02d} loss={loss:.4f}  \"\n                      f\"quick={gm_q['gm']:.3f}  {dt:.0f}s\")\n\n        print(f\"  Best [{self.name}]: {self.best:.4f}\")\n        return self.best\n\n    @torch.no_grad()\n    def predict(self, srcs: List[str], beams: int) -> List[str]:\n        \"\"\"Crash-safe prediction for any size input.\"\"\"\n        return self._gen(srcs, beams=beams)\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 7 ─ LOAD DATA\n# ════════════════════════════════════════════════════════════════════════════\n\nprint(\"\\n\" + \"=\"*64)\nprint(\"  LOADING DATA\")\nprint(\"=\"*64)\n\ntrain_df = pd.read_csv(f'{CFG.DATA_DIR}/train.csv')\ntest_df  = pd.read_csv(f'{CFG.DATA_DIR}/test.csv')\n\nprint(f\"\\n  Train: {len(train_df)} rows  {list(train_df.columns)}\")\nprint(f\"  Test:  {len(test_df)} rows   {list(test_df.columns)}\")\n\n# Robust column detection\ndef fc(df, opts):\n    for c in opts:\n        if c in df.columns: return c\n    for c in df.columns:\n        for o in opts:\n            if o.lower() in c.lower(): return c\n    return df.columns[-1]\n\nSC = fc(train_df, ['transliteration','source','akk','text'])\nTC = fc(train_df, ['translation','target','en','english'])\nES = fc(test_df,  ['transliteration','source','akk','text'])\nEI = fc(test_df,  ['id','index','oare_id'])\n\nprint(f\"  Cols: src={SC}  tgt={TC}  test_src={ES}  test_id={EI}\")\n\n# Preprocess\ntrain_df['src'] = train_df[SC].apply(lambda x: prep(x,'source'))\ntrain_df['tgt'] = train_df[TC].apply(lambda x: prep(x,'target'))\ntest_df['src']  = test_df[ES].apply(lambda x: prep(x,'source'))\n\n# Clean training data\nmask = (train_df['src'].str.len()>2)&(train_df['tgt'].str.len()>2)\ntrain_df = train_df[mask].reset_index(drop=True)\nprint(f\"  Clean training pairs: {len(train_df)}\")\n\n# Val split for tracking (FULL data used for training)\ntr, va = train_test_split(train_df, test_size=CFG.VAL_RATIO, random_state=SEED)\ntr = tr.reset_index(drop=True); va = va.reset_index(drop=True)\n\n# ── ALL training data (including val) for final training ─────────────────────\nALL_SRC  = train_df['src'].tolist()\nALL_TGT  = train_df['tgt'].tolist()\nTR_SRC   = tr['src'].tolist();   TR_TGT  = tr['tgt'].tolist()\nVA_SRC   = va['src'].tolist();   VA_TGT  = va['tgt'].tolist()\nTEST_SRC = test_df['src'].tolist()\n\n# Robust ID extraction (keep as strings, convert safely later)\ntry:\n    TEST_IDS = test_df[EI].astype(int).tolist()\nexcept:\n    TEST_IDS = test_df[EI].tolist()\n\nprint(f\"  Train(split)={len(TR_SRC)}  Val={len(VA_SRC)}  \"\n      f\"All={len(ALL_SRC)}  Test={len(TEST_SRC)}\")\nprint(f\"  SRC: {TR_SRC[0][:80]}\")\nprint(f\"  TGT: {TR_TGT[0][:80]}\")\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 8 ─ LOAD MODEL\n# ════════════════════════════════════════════════════════════════════════════\n\nprint(\"\\n\" + \"=\"*64)\nprint(\"  LOADING MARIANMT  (Semitic -> English)\")\nprint(\"=\"*64)\n\nm_tok=None; m_mdl=None; M_OK=False; M_NAME=''; M_SCORE=0.0\n\nfor mn in CFG.MARIAN_CANDIDATES:\n    try:\n        print(f\"\\n  {mn} ...\", end=' ', flush=True)\n        m_tok = MarianTokenizer.from_pretrained(mn)\n        m_mdl = MarianMTModel.from_pretrained(mn).to(DEVICE)\n        M_OK=True; M_NAME=mn\n        print(f\"OK  {sum(p.numel() for p in m_mdl.parameters())/1e6:.0f}M params\")\n        break\n    except Exception as e:\n        print(f\"FAILED: {e}\")\n\nif not M_OK:\n    raise RuntimeError(\"Could not load any MarianMT model. \"\n                       \"Check Kaggle data source attachments.\")\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 9 ─ PHASE 1: FINE-TUNE ON TRAIN SPLIT  (find best LR / epochs)\n# ════════════════════════════════════════════════════════════════════════════\n\nprint(\"\\n\" + \"=\"*64)\nprint(\"  PHASE 1: Fine-tune on train split (track val performance)\")\nprint(\"=\"*64)\n\ntr_ds   = DS(TR_SRC, TR_TGT, m_tok, CFG.MAX_LEN)\ntrainer = Trainer(m_mdl, m_tok, CFG, 'marian_phase1')\nM_SCORE = trainer.fit(tr_ds, VA_SRC, VA_TGT, lr=CFG.LR)\n\n# Load best phase-1 checkpoint\nm_mdl = MarianMTModel.from_pretrained(trainer.ckpt).to(DEVICE)\nm_mdl.eval()\nprint(f\"\\n  Phase-1 best val GM: {M_SCORE:.4f}\")\n\ngc.collect()\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 10 ─ PHASE 2: RETRAIN ON FULL DATA  (maximize coverage)\n# ════════════════════════════════════════════════════════════════════════════\n\nprint(\"\\n\" + \"=\"*64)\nprint(\"  PHASE 2: Retrain on FULL dataset (train + val combined)\")\nprint(\"=\"*64)\n\n# Use fewer epochs for full-data phase (already converged)\nCFG_FULL_EPOCHS = max(2, CFG.EPOCHS // 2)\n\nall_ds     = DS(ALL_SRC, ALL_TGT, m_tok, CFG.MAX_LEN)\ntrainer2   = Trainer(m_mdl, m_tok, CFG, 'marian_phase2')\ntrainer2.cfg.EPOCHS = CFG_FULL_EPOCHS\ntrainer2.fit(all_ds, VA_SRC, VA_TGT, lr=CFG.LR * 0.5)\n\n# Use phase2 model (trained on more data)\nm_mdl = MarianMTModel.from_pretrained(trainer2.ckpt).to(DEVICE)\nm_mdl.eval()\nprint(f\"  Phase-2 complete (full data, {CFG_FULL_EPOCHS} epochs)\")\n\ngc.collect()\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 11 ─ GENERATE PREDICTIONS  (crash-safe, any test size)\n# ════════════════════════════════════════════════════════════════════════════\n\nprint(\"\\n\" + \"=\"*64)\nprint(f\"  GENERATING {len(TEST_SRC)} PREDICTIONS  (beam={CFG.TEST_BEAMS})\")\nprint(\"=\"*64)\n\n\ndef clean(t: str) -> str:\n    \"\"\"Post-process: single clean sentence, non-empty.\"\"\"\n    if not t or (isinstance(t, float) and math.isnan(t)):\n        return \"The content of this tablet is fragmentary.\"\n    t = str(t).strip()\n    t = re.sub(r'\\s+', ' ', t)\n    t = re.sub(r'\\s+([.,;:!?])', r'\\1', t)\n    # Remove exact consecutive duplicate words\n    t = re.sub(r'\\b(\\w+)( \\1\\b)+', r'\\1', t, flags=re.IGNORECASE)\n    # Capitalise first letter\n    if t: t = t[0].upper() + t[1:]\n    # Ensure minimum length\n    if len(t.strip()) < 3:\n        t = \"The content of this tablet section is unclear.\"\n    return t.strip()\n\n\n# Crash-safe generation: try full batch, fall back to size 1 per sample\nfinal_preds = []\nprint(f\"\\n  Generating in batches of {CFG.BINFER} ...\")\n\nm_mdl.eval()\nwith torch.no_grad():\n    for i in tqdm(range(0, len(TEST_SRC), CFG.BINFER),\n                  desc=\"  Translating\"):\n        batch_src = TEST_SRC[i:i+CFG.BINFER]\n        batch_out = []\n        try:\n            enc = m_tok(batch_src, return_tensors='pt', padding=True,\n                        truncation=True,\n                        max_length=CFG.MAX_LEN).to(DEVICE)\n            kw  = dict(max_new_tokens=CFG.MAX_LEN,\n                       num_beams=CFG.TEST_BEAMS,\n                       length_penalty=CFG.LEN_PENALTY,\n                       no_repeat_ngram_size=CFG.NO_RPT_NGR,\n                       repetition_penalty=CFG.RPT_PEN,\n                       early_stopping=True)\n            out = m_mdl.generate(**enc, **kw)\n            batch_out = m_tok.batch_decode(out, skip_special_tokens=True)\n        except Exception as e:\n            print(f\"\\n    Batch {i//CFG.BINFER} failed ({e}), \"\n                  \"trying one-by-one ...\")\n            for src in batch_src:\n                try:\n                    enc = m_tok([src], return_tensors='pt', padding=True,\n                                truncation=True,\n                                max_length=CFG.MAX_LEN).to(DEVICE)\n                    out = m_mdl.generate(\n                        **enc,\n                        max_new_tokens=CFG.MAX_LEN,\n                        num_beams=2,              # reduced beams as fallback\n                        early_stopping=True)\n                    batch_out.append(\n                        m_tok.decode(out[0], skip_special_tokens=True))\n                except Exception as e2:\n                    print(f\"      Sample fallback failed: {e2}\")\n                    batch_out.append(\"\")\n        final_preds.extend(batch_out)\n\n# Post-process all predictions\nfinal_preds = [clean(p) for p in final_preds]\n\n# Ensure we have exactly the right number of predictions\nwhile len(final_preds) < len(TEST_SRC):\n    final_preds.append(\"The content of this tablet section is fragmentary.\")\nfinal_preds = final_preds[:len(TEST_SRC)]\n\nprint(f\"\\n  Generated: {len(final_preds)} predictions\")\nprint(f\"  Non-empty: {sum(1 for p in final_preds if len(p)>10)}\")\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 12 ─ LEXICON CORRECTION\n# ════════════════════════════════════════════════════════════════════════════\n\nlex = {}\nfor fn in ['OA_Lexicon_eBL.csv', 'eBL_Dictionary.csv', 'lexicon.csv']:\n    fp = f'{CFG.DATA_DIR}/{fn}'\n    if os.path.exists(fp):\n        try:\n            ld = pd.read_csv(fp, on_bad_lines='skip')\n            for _, r in ld.iterrows():\n                v=[str(x).strip() for x in r.tolist() if str(x).strip()!='nan']\n                if len(v)>=2: lex[v[0].lower()]=v[1]\n            print(f\"  Lexicon {fn}: {len(lex)} entries loaded\")\n        except Exception as e:\n            print(f\"  Lexicon {fn} skipped: {e}\")\n\nif lex:\n    def apl(t):\n        for k, v in lex.items():\n            t = re.sub(rf'\\b{re.escape(k)}\\b', v, t, flags=re.IGNORECASE)\n        return t\n    final_preds = [apl(p) for p in final_preds]\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 13 ─ BUILD SUBMISSION  (robust ID handling)\n# ════════════════════════════════════════════════════════════════════════════\n\nprint(\"\\n\" + \"=\"*64)\nprint(\"  BUILDING SUBMISSION\")\nprint(\"=\"*64)\n\n# Robust ID handling: keep original IDs, don't force int conversion\nsub = pd.DataFrame({\n    'id':          TEST_IDS,\n    'translation': final_preds,\n})\n\n# Sort by ID if possible\ntry:\n    sub['id'] = sub['id'].astype(int)\n    sub = sub.sort_values('id').reset_index(drop=True)\nexcept:\n    sub = sub.reset_index(drop=True)\n\n# Fill any remaining NaN translations\nsub['translation'] = sub['translation'].fillna(\n    \"The content of this tablet is fragmentary.\")\n\nsub.to_csv(CFG.SUB_FILE, index=False)\nfile_mb = os.path.getsize(CFG.SUB_FILE) / 1e6\n\nprint(f\"\\n  Saved: {CFG.SUB_FILE}  ({len(sub)} rows  {file_mb:.3f} MB)\")\nprint(f\"\\n  All predictions:\")\nprint(sub.to_string())\n\n# Verify format\nprint(f\"\\n  Columns: {list(sub.columns)}\")\nprint(f\"  NaN translations: {sub['translation'].isna().sum()}\")\nprint(f\"  Empty (<5 chars): {(sub['translation'].str.len()<5).sum()}\")\n\n\n# ════════════════════════════════════════════════════════════════════════════\n# CELL 14 ─ VALIDATION + SUMMARY\n# ════════════════════════════════════════════════════════════════════════════\n\nprint(\"\\n\" + \"=\"*64)\nprint(\"  FINAL VALIDATION ON VAL SET\")\nprint(\"=\"*64)\n\nm_mdl.eval()\nvp = []\nwith torch.no_grad():\n    for i in range(0, len(VA_SRC), CFG.BINFER):\n        batch = VA_SRC[i:i+CFG.BINFER]\n        try:\n            enc = m_tok(batch, return_tensors='pt', padding=True,\n                        truncation=True, max_length=CFG.MAX_LEN).to(DEVICE)\n            out = m_mdl.generate(**enc,\n                                  max_new_tokens=CFG.MAX_LEN,\n                                  num_beams=CFG.TEST_BEAMS,\n                                  length_penalty=CFG.LEN_PENALTY,\n                                  no_repeat_ngram_size=CFG.NO_RPT_NGR,\n                                  early_stopping=True)\n            vp.extend(m_tok.batch_decode(out, skip_special_tokens=True))\n        except Exception as e:\n            print(f\"  Val batch error: {e}\")\n            vp.extend([\"\"] * len(batch))\n\nvp = [clean(p) for p in vp]\nvm = Metrics.score(vp, VA_TGT)\n\nprint(f\"\\n  Val BLEU  = {vm['bleu']:.4f}\")\nprint(f\"  Val chrF  = {vm['chrf']:.4f}\")\nprint(f\"  Val GM    = {vm['gm']:.4f}\")\n\nprint(f\"\\n  Sample predictions:\")\nfor i in range(min(4, len(vp))):\n    print(f\"\\n  REF: {VA_TGT[i][:100]}\")\n    print(f\"  PRD: {vp[i][:100]}\")\n\nprint(\"\\n\" + \"=\"*64)\nprint(\"  SUMMARY\")\nprint(\"=\"*64)\nprint(f\"\"\"\n  Model    : {M_NAME}  ({sum(p.numel() for p in m_mdl.parameters())/1e6:.0f}M params)\n  Training : Phase-1 ({CFG.EPOCHS} ep on {len(TR_SRC)} samples)\n           + Phase-2 ({CFG_FULL_EPOCHS} ep on {len(ALL_SRC)} samples)\n  Val GM   : {vm['gm']:.4f}  (BLEU={vm['bleu']:.2f}  chrF={vm['chrf']:.2f})\n\n  Crash fixes:\n    No assert statements       (hidden test any size)\n    try/except per batch       (graceful fallback)\n    Robust ID handling         (str or int IDs)\n    Guaranteed non-empty preds (all {len(sub)} rows filled)\n\n  Output: {CFG.SUB_FILE}   ({len(sub)} rows)  OK\n\"\"\")\nprint(\"=\"*64)","metadata":{"_uuid":"32ff2fce-387d-4a7a-b89d-9b4d0393cc37","_cell_guid":"d9bd1b2c-ec31-49be-9faf-28a20040ee38","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2026-03-07T00:29:34.545015Z","iopub.execute_input":"2026-03-07T00:29:34.545940Z"}},"outputs":[],"execution_count":null}]}