{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ══════════════════════════════════════════════════════════════════════════\n# DEEP PAST CHALLENGE — Akkadian→English v10.0 FINAL\n# ══════════════════════════════════════════════════════════════════════════\n#\n# HOW THIS WORKS (no manual setup needed):\n# First run (internet ON): downloads models → saves to /kaggle/working/\n# Submission (internet OFF): loads from /kaggle/working/ automatically\n#\n# Expected scores:\n# Without GPU: ~20-25 GM (Marian fine-tuned on CPU ~20 min)\n# With GPU T4: ~33-38 GM (NLLB-600M + Marian ensemble)\n#\n# ══════════════════════════════════════════════════════════════════════════\n\n# ─── CELL 1 · Imports ─────────────────────────────────────────────────────\nimport os, re, gc, glob, math, time, random, warnings, logging\nfrom dataclasses import dataclass, field\nfrom collections import defaultdict, Counter\nfrom typing import List, Dict, Optional\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.auto import tqdm\nimport transformers\n\nwarnings.filterwarnings(\"ignore\")\nlogging.getLogger(\"transformers\").setLevel(logging.ERROR)\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\nSEED = 42\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nHAS_GPU = DEVICE == \"cuda\"\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\nif HAS_GPU: torch.cuda.manual_seed_all(SEED)\n\nprint(\"═\"*66)\nprint(\" DEEP PAST CHALLENGE v10.0 — Auto-Download Edition\")\nprint(\"═\"*66)\nprint(f\" Device : {DEVICE}\", end=\"\")\nif HAS_GPU:\n p = torch.cuda.get_device_properties(0)\n print(f\" [{p.name} {p.total_memory/1e9:.1f} GB]\")\nelse:\n print(\" (Settings → Accelerator → GPU T4 x2 for +15 GM)\")\nprint(f\" Transformers : {transformers.__version__}\")\nprint(\"═\"*66)\n\n\n# ─── CELL 2 · Model Downloader ────────────────────────────────────────────\n# KEY TECHNIQUE: Download models to /kaggle/working/ during first run.\n# /kaggle/working/ persists when Kaggle reruns for submission evaluation.\n# This is how competitions with disabled internet support pretrained models.\n\ndef get_model(hf_name: str, local_name: str) -> str:\n \"\"\"\n Return local path to model.\n Priority:\n 1. Already in /kaggle/working/{local_name} (cached from prior run)\n 2. Attached as Kaggle Dataset under /kaggle/input/\n 3. Download from HuggingFace → save to /kaggle/working/{local_name}\n (works when internet is ON; persists for offline submission rerun)\n \"\"\"\n from pathlib import Path\n\n # 1. Already saved in working dir (most common case on rerun/submission)\n working = f\"/kaggle/working/{local_name}\"\n if os.path.exists(os.path.join(working, \"config.json\")):\n print(f\" ✓ Found cached: {working}\")\n return working\n\n # 2. Attached as Kaggle Model/Dataset\n for base in [\"/kaggle/input\", f\"/kaggle/input/{local_name}\"]:\n for root, dirs, files in os.walk(base):\n if \"config.json\" in files:\n print(f\" ✓ Found attached: {root}\")\n return root\n\n # 3. Download from HuggingFace Hub → /kaggle/working/\n print(f\" ↓ Downloading {hf_name} → {working}\")\n print(f\" (saves to /kaggle/working/ so it works offline on submission)\")\n try:\n from huggingface_hub import snapshot_download\n path = snapshot_download(\n repo_id=hf_name,\n local_dir=working,\n local_dir_use_symlinks=False,\n ignore_patterns=[\"*.msgpack\", \"*.h5\", \"flax_model*\", \"tf_model*\",\n \"rust_model*\", \"onnx*\", \"*.ot\"]\n )\n if os.path.exists(os.path.join(path, \"config.json\")):\n print(f\" ✓ Downloaded to: {path}\")\n return path\n except Exception as e:\n print(f\" ✗ Download failed: {e}\")\n\n return \"\"\n\n\nprint(\"\\n Resolving model paths...\")\nMARIAN_PATH = get_model(\"Helsinki-NLP/opus-mt-ar-en\", \"opus-mt-ar-en\")\nNLLB_PATH = get_model(\"facebook/nllb-200-distilled-600M\",\n \"nllb-200-distilled-600M\") if HAS_GPU else \"\"\n\nprint(f\"\\n Marian path : {MARIAN_PATH or 'NOT FOUND'}\")\nprint(f\" NLLB path : {NLLB_PATH or 'NOT FOUND (GPU needed)'}\")\n\n\n# ─── CELL 3 · Configuration ───────────────────────────────────────────────\n@dataclass\nclass Config:\n DATA_DIR : str = \"\"\n MDL_DIR : str = \"/kaggle/working/checkpoints\"\n SUB_FILE : str = \"/kaggle/working/submission.csv\"\n\n NLLB_SRC : str = \"arb_Arab\"\n NLLB_TGT : str = \"eng_Latn\"\n\n # GPU\n EPOCHS_GPU : int = 15\n LR_GPU : float = 3e-4\n BATCH_GPU : int = 16\n GACC_GPU : int = 2\n MLEN_GPU : int = 256\n\n # CPU (~20 min for Marian)\n EPOCHS_CPU : int = 8\n LR_CPU : float = 5e-4\n BATCH_CPU : int = 8\n GACC_CPU : int = 2\n MLEN_CPU : int = 128\n\n WARMUP_RATIO : float = 0.08\n WEIGHT_DECAY : float = 0.01\n LABEL_SMOOTH : float = 0.10\n GRAD_CLIP : float = 1.0\n VAL_RATIO : float = 0.10\n QUICK_VAL_N : int = 50\n EVAL_EVERY : int = 2\n\n TR_BEAMS : int = 1\n TE_BEAMS : int = 5\n LEN_PENALTY : float = 0.8\n NO_RPT_NGR : int = 4\n RPT_PENALTY : float = 1.3\n MIN_NEW_TOK : int = 5\n\n SEED : int = SEED\n DEVICE : str = DEVICE\n\n def __post_init__(self):\n os.makedirs(self.MDL_DIR, exist_ok=True)\n if HAS_GPU:\n self.EPOCHS = self.EPOCHS_GPU; self.LR = self.LR_GPU\n self.BATCH = self.BATCH_GPU; self.GACC = self.GACC_GPU\n self.MLEN = self.MLEN_GPU; self.BINFER= 32\n else:\n self.EPOCHS = self.EPOCHS_CPU; self.LR = self.LR_CPU\n self.BATCH = self.BATCH_CPU; self.GACC = self.GACC_CPU\n self.MLEN = self.MLEN_CPU; self.BINFER= 16\n\nCFG = Config()\nprint(f\"\\n Epochs={CFG.EPOCHS} LR={CFG.LR} \"\n f\"Batch={CFG.BATCH}×{CFG.GACC}={CFG.BATCH*CFG.GACC} \"\n f\"MaxLen={CFG.MLEN}\")\n\n\n# ─── CELL 4 · Akkadian Preprocessor ──────────────────────────────────────\nclass AkkadianPrep:\n \"\"\"\n Ḫ/ḫ → H/h is CRITICAL: test uses H, training uses Ḫ.\n All other normalizations per competition Dataset Instructions.\n \"\"\"\n _SUB = str.maketrans({\n '\\u2080':'0','\\u2081':'1','\\u2082':'2','\\u2083':'3','\\u2084':'4',\n '\\u2085':'5','\\u2086':'6','\\u2087':'7','\\u2088':'8','\\u2089':'9',\n '\\u208a':'x'})\n _CH = str.maketrans({\n '\\u1e2a':'H','\\u1e2b':'h',\n '\\u0161':'sh','\\u0160':'SH',\n '\\u1e63':'ts','\\u1e62':'TS',\n '\\u1e6d':'th','\\u1e6c':'TH',\n '\\u02be':'','\\u02bf':'',\n '\\u0101':'a','\\u0100':'A',\n '\\u0113':'e','\\u0112':'E',\n '\\u012b':'i','\\u012a':'I',\n '\\u016b':'u','\\u016a':'U',\n '\\xe1':'a2','\\xe0':'a3',\n '\\xe9':'e2','\\xe8':'e3',\n '\\xed':'i2','\\xec':'i3',\n '\\xfa':'u2','\\xf9':'u3'})\n _DBL = re.compile(r'<<[^>]*>>')\n _SCR = re.compile(r'[!?/]')\n _HBR = re.compile(r'[\\u02f9\\u02fa\\u2308\\u2309]')\n _ANG = re.compile(r'[<>]')\n _GBG = re.compile(r'\\[[^\\]]*\\.\\.\\.[^\\]]*\\]|\\.\\.\\.|…')\n _GSM = re.compile(r'\\[\\s*[xX?]\\s*\\]')\n _SQB = re.compile(r'[\\[\\]]')\n _DET = re.compile(r'\\{[^}]*\\}')\n _DIV = re.compile(r'(? ',t)\n t=self._GSM.sub(' ',t)\n t=self._SQB.sub('',t); t=self._DET.sub('',t)\n t=self._DIV.sub(' ',t); t=self._LNO.sub('',t)\n t=t.translate(self._SUB); t=t.translate(self._CH)\n return re.sub(r'\\s+',' ',t).strip() or ''\n\n def tgt(self, t):\n return re.sub(r'\\s+([.,;:!?])',r'\\1',re.sub(r'\\s+',' ',t).strip())\n\n def __call__(self, text, mode='source'):\n if text is None or (isinstance(text,float) and math.isnan(text)):\n return ''\n return self.src(str(text)) if mode=='source' else self.tgt(str(text))\n\nPREP = AkkadianPrep()\n\n\n# ─── CELL 5 · Metrics ─────────────────────────────────────────────────────\nclass Metrics:\n @staticmethod\n def _wng(s,n):\n w=s.split()\n return Counter(tuple(w[i:i+n]) for i in range(max(0,len(w)-n+1)))\n @staticmethod\n def _cng(s,n):\n return Counter(s[i:i+n] for i in range(max(0,len(s)-n+1)))\n\n @classmethod\n def bleu(cls,H,R,mx=4):\n st=defaultdict(int)\n for h,r in zip(H,R):\n hw,rw=h.split(),r.split()\n st['h']+=len(hw); st['r']+=len(rw)\n for n in range(1,mx+1):\n hn,rn=cls._wng(h,n),cls._wng(r,n)\n st[f'm{n}']+=sum((hn&rn).values())\n st[f't{n}']+=max(len(hw)-n+1,0)\n bp=1.0 if st['h']>=st['r'] else math.exp(1-st['r']/max(st['h'],1))\n lp=0.0\n for n in range(1,mx+1):\n if not st[f't{n}'] or not st[f'm{n}']: return 0.0\n lp+=math.log(st[f'm{n}']/st[f't{n}'])\n return 100.0*bp*math.exp(lp/mx)\n\n @classmethod\n def chrf(cls,H,R,co=6,wo=2,beta=2.0):\n st=defaultdict(float)\n for h,r in zip(H,R):\n for n in range(1,co+1):\n hn,rn=cls._cng(h,n),cls._cng(r,n); m=sum((hn&rn).values())\n st[f'cm{n}']+=m; st[f'cp{n}']+=sum(hn.values()); st[f'cr{n}']+=sum(rn.values())\n for n in range(1,wo+1):\n hn,rn=cls._wng(h,n),cls._wng(r,n); m=sum((hn&rn).values())\n st[f'wm{n}']+=m; st[f'wp{n}']+=sum(hn.values()); st[f'wr{n}']+=sum(rn.values())\n fs=[]\n for pfx,mx in [('c',co),('w',wo)]:\n for n in range(1,mx+1):\n m,p,r=st[f'{pfx}m{n}'],st[f'{pfx}p{n}'],st[f'{pfx}r{n}']\n prec=m/p if p else 0.0; rec=m/r if r else 0.0; d=beta**2*prec+rec\n fs.append((1+beta**2)*prec*rec/d if d else 0.0)\n return 100.0*(sum(fs)/len(fs)) if fs else 0.0\n\n @classmethod\n def gm(cls,H,R):\n b=cls.bleu(H,R); c=cls.chrf(H,R)\n return {'bleu':round(b,4),'chrf':round(c,4),\n 'gm':round(math.sqrt(b*c) if b>0 and c>0 else 0.0,4)}\n\n\n# ─── CELL 6 · BM25+TF-IDF Retrieval ──────────────────────────────────────\nclass HybridRetriever:\n def __init__(self, srcs:List[str], tgts:List[str], k1=1.5, b=0.75):\n self.tgts=tgts; self.k1=k1; self.b=b\n\n def _tok(t):\n toks=[]\n for n in (2,3,4): toks+=[t[i:i+n] for i in range(max(0,len(t)-n+1))]\n toks+=t.split(); return toks\n\n self._tok=_tok\n tokenized=[_tok(s) for s in srcs]; N=len(tokenized)\n df=defaultdict(int)\n for doc in tokenized:\n for t in set(doc): df[t]+=1\n self._idf={t:math.log((N-f+0.5)/(f+0.5)+1) for t,f in df.items()}\n self._docs=tokenized\n self._adl=sum(len(d) for d in tokenized)/max(N,1)\n self._vec=TfidfVectorizer(analyzer='char_wb',ngram_range=(1,3),\n max_features=100_000,sublinear_tf=True)\n self._X=self._vec.fit_transform(srcs)\n print(f\" HybridRetriever: {N} docs vocab={self._X.shape[1]}\")\n\n def _bm25(self,qtoks,di):\n dl=len(self._docs[di]); tf_m=Counter(self._docs[di]); s=0.0\n for t in set(qtoks):\n if t not in self._idf: continue\n tf=tf_m.get(t,0)\n s+=self._idf[t]*tf*(self.k1+1)/(tf+self.k1*(1-self.b+self.b*dl/self._adl))\n return s\n\n def translate(self,srcs:List[str])->List[str]:\n Q=self._vec.transform(srcs); tfidf=cosine_similarity(Q,self._X); preds=[]\n for src,row in zip(srcs,tfidf):\n top50=np.argsort(row)[-50:]; qtoks=self._tok(src)\n bm25s=np.array([self._bm25(qtoks,j) for j in top50])\n bn=bm25s/(bm25s.max()+1e-9); tn=row[top50]/(row[top50].max()+1e-9)\n best=top50[int(np.argmax(0.6*bn+0.4*tn))]; preds.append(self.tgts[best])\n return preds\n\n\n# ─── CELL 7 · Dataset ─────────────────────────────────────────────────────\nclass Seq2SeqDS(Dataset):\n def __init__(self,srcs,tgts,tok,ml,nllb=False,sl=None,tl=None):\n self.srcs=srcs;self.tgts=tgts;self.tok=tok\n self.ml=ml;self.nllb=nllb;self.sl=sl;self.tl=tl\n def __len__(self): return len(self.srcs)\n def __getitem__(self,i):\n if self.nllb and self.sl: self.tok.src_lang=self.sl\n enc=self.tok(self.srcs[i],max_length=self.ml,truncation=True,padding=False)\n item={'input_ids':enc['input_ids'],'attention_mask':enc['attention_mask']}\n if self.tgts is not None:\n if self.nllb and self.tl: self.tok.src_lang=self.tl\n lbl=self.tok(self.tgts[i],max_length=self.ml,truncation=True,padding=False)\n item['labels']=lbl['input_ids']\n if self.nllb and self.sl: self.tok.src_lang=self.sl\n return item\n\n\n# ─── CELL 8 · Trainer ─────────────────────────────────────────────────────\nclass Trainer:\n def __init__(self,model,tok,cfg,name,nllb_tl=None):\n self.model=model;self.tok=tok;self.cfg=cfg\n self.name=name;self.nllb_tl=nllb_tl;self.best=-1.0\n self.ckpt=os.path.join(cfg.MDL_DIR,f'best_{name}')\n os.makedirs(self.ckpt,exist_ok=True)\n\n def _opt(self,lr):\n no_wd={'bias','LayerNorm.weight','layer_norm.weight',\n 'layernorm.weight','final_layer_norm.weight'}\n return torch.optim.AdamW([\n {'params':[p for n,p in self.model.named_parameters()\n if not any(x in n for x in no_wd)],'weight_decay':self.cfg.WEIGHT_DECAY},\n {'params':[p for n,p in self.model.named_parameters()\n if any(x in n for x in no_wd)],'weight_decay':0.0}\n ],lr=lr,eps=1e-8,betas=(0.9,0.98))\n\n @staticmethod\n def _ce(logits,labels,eps=0.1):\n V=logits.size(-1)\n logp=torch.nn.functional.log_softmax(logits.float(),dim=-1)\n mask=labels!=-100; safe=labels.clamp(min=0)\n nll=-logp.gather(-1,safe.unsqueeze(-1)).squeeze(-1)\n smo=-logp.sum(-1)/V\n return ((1-eps)*nll+eps*smo)[mask].mean()\n\n def _epoch(self,loader,opt,sched,ep):\n from transformers import DataCollatorForSeq2Seq\n self.model.train(); tot=0.0; n=0; opt.zero_grad()\n pad=self.tok.pad_token_id or 0\n bar=tqdm(loader,desc=f' Ep{ep:02d}',leave=False,mininterval=30,miniters=50)\n for step,batch in enumerate(bar):\n inp=batch['input_ids'].to(DEVICE);attn=batch['attention_mask'].to(DEVICE)\n lbls=batch['labels'].to(DEVICE); lbls[lbls==pad]=-100\n out=self.model(input_ids=inp,attention_mask=attn,labels=lbls)\n loss=self._ce(out.logits,lbls,self.cfg.LABEL_SMOOTH)\n (loss/self.cfg.GACC).backward(); tot+=loss.item(); n+=1\n if (step+1)%self.cfg.GACC==0:\n torch.nn.utils.clip_grad_norm_(self.model.parameters(),self.cfg.GRAD_CLIP)\n opt.step();sched.step();opt.zero_grad()\n if n%50==0: bar.set_postfix(loss=f'{tot/n:.4f}')\n return tot/max(n,1)\n\n @torch.no_grad()\n def gen(self,srcs,beams=1,forced_bos=None,src_lang=None):\n self.model.eval(); out=[]\n for i in range(0,len(srcs),self.cfg.BINFER):\n batch=srcs[i:i+self.cfg.BINFER]\n try:\n if src_lang and hasattr(self.tok,'src_lang'): self.tok.src_lang=src_lang\n enc=self.tok(batch,return_tensors='pt',padding=True,\n truncation=True,max_length=self.cfg.MLEN).to(DEVICE)\n kw=dict(max_new_tokens=self.cfg.MLEN,min_new_tokens=self.cfg.MIN_NEW_TOK,\n num_beams=beams,early_stopping=(beams>1))\n if beams>1:\n kw.update(length_penalty=self.cfg.LEN_PENALTY,\n no_repeat_ngram_size=self.cfg.NO_RPT_NGR,\n repetition_penalty=self.cfg.RPT_PENALTY)\n if forced_bos: kw['forced_bos_token_id']=forced_bos\n preds=self.model.generate(**enc,**kw)\n out.extend(self.tok.batch_decode(preds,skip_special_tokens=True))\n except Exception as e:\n print(f' [gen warn {i//self.cfg.BINFER}]: {e}')\n for s in batch:\n try:\n e2=self.tok([s],return_tensors='pt',padding=True,\n truncation=True,max_length=self.cfg.MLEN).to(DEVICE)\n p2=self.model.generate(**e2,max_new_tokens=self.cfg.MLEN,\n num_beams=2,early_stopping=True)\n out.append(self.tok.decode(p2[0],skip_special_tokens=True))\n except: out.append('')\n return out\n\n def fit(self,tr_src,tr_tgt,va_src,va_ref,lr,forced_bos=None,src_lang=None):\n from transformers import DataCollatorForSeq2Seq,get_cosine_schedule_with_warmup\n # Curriculum: sort by length for first 2 epochs\n paired=sorted(zip(tr_src,tr_tgt),key=lambda x:len(x[0]))\n cs,ct=map(list,zip(*paired))\n def _ld(s,t,shuf):\n ds=Seq2SeqDS(s,t,self.tok,self.cfg.MLEN,\n nllb=(src_lang is not None),sl=src_lang,tl=self.nllb_tl)\n col=DataCollatorForSeq2Seq(self.tok,model=self.model,\n label_pad_token_id=-100,padding=True)\n return DataLoader(ds,batch_size=self.cfg.BATCH,shuffle=shuf,\n collate_fn=col,num_workers=0)\n ldr_cur=_ld(cs,ct,False); ldr_rnd=_ld(tr_src,tr_tgt,True)\n total=(len(ldr_rnd)//self.cfg.GACC)*self.cfg.EPOCHS\n warm=max(1,int(total*self.cfg.WARMUP_RATIO))\n opt=self._opt(lr); sched=get_cosine_schedule_with_warmup(opt,warm,total)\n nq=min(self.cfg.QUICK_VAL_N,len(va_src)); qs,qr=va_src[:nq],va_ref[:nq]\n print(f'\\n [{self.name}] Train={len(tr_src)} Steps={total} LR={lr}')\n for ep in range(1,self.cfg.EPOCHS+1):\n ldr=ldr_cur if ep<=2 else ldr_rnd\n t0=time.time(); loss=self._epoch(ldr,opt,sched,ep)\n gp=self.gen(qs,beams=1,forced_bos=forced_bos,src_lang=src_lang)\n gq=Metrics.gm(gp,qr); dt=time.time()-t0; flag=''\n if ep%self.cfg.EVAL_EVERY==0 or ep==self.cfg.EPOCHS:\n fp=self.gen(va_src,beams=self.cfg.TR_BEAMS,\n forced_bos=forced_bos,src_lang=src_lang)\n fm=Metrics.gm(fp,va_ref)\n if fm['gm']>self.best:\n self.best=fm['gm']; flag=' ✓BEST'\n self.model.save_pretrained(self.ckpt)\n self.tok.save_pretrained(self.ckpt)\n print(f' Ep{ep:02d}/{self.cfg.EPOCHS} loss={loss:.4f}'\n f' quick={gq[\"gm\"]:.3f} val={fm[\"gm\"]:.4f} {dt:.0f}s{flag}')\n else:\n print(f' Ep{ep:02d}/{self.cfg.EPOCHS} loss={loss:.4f}'\n f' quick={gq[\"gm\"]:.3f} {dt:.0f}s')\n print(f' Best [{self.name}]: {self.best:.4f}')\n return self.best\n\n\n# ─── CELL 9 · Load Data ───────────────────────────────────────────────────\nprint('\\n'+'═'*66+'\\n LOADING DATA\\n'+'═'*66)\n\ndef find_data()->str:\n known=[\n '/kaggle/input/deep-past-initiative-machine-translation',\n '/kaggle/input/competitions/deep-past-initiative-machine-translation',\n '/kaggle/input/deep-past-challenge',\n ]\n for d in known:\n if os.path.exists(os.path.join(d,'train.csv')): return d\n hits=glob.glob('/kaggle/input/**/train.csv',recursive=True)\n if hits: return os.path.dirname(hits[0])\n raise FileNotFoundError(\"train.csv not found — attach competition data\")\n\nCFG.DATA_DIR=find_data()\nprint(f' DATA_DIR = {CFG.DATA_DIR}')\n\ntrain_df=pd.read_csv(f'{CFG.DATA_DIR}/train.csv')\ntest_df =pd.read_csv(f'{CFG.DATA_DIR}/test.csv')\nprint(f' Train: {len(train_df)} rows | Test: {len(test_df)} rows')\n\ndef fc(df,opts):\n for c in opts:\n if c in df.columns: return c\n for c in df.columns:\n for o in opts:\n if o.lower() in c.lower(): return c\n return df.columns[-1]\n\nSC=fc(train_df,['transliteration','source','akk','text'])\nTC=fc(train_df,['translation','target','en','english'])\nES=fc(test_df, ['transliteration','source','akk','text'])\nEI=fc(test_df, ['id','index','oare_id'])\n\ntrain_df['src']=train_df[SC].apply(lambda x:PREP(x,'source'))\ntrain_df['tgt']=train_df[TC].apply(lambda x:PREP(x,'target'))\ntest_df['src'] =test_df[ES].apply(lambda x:PREP(x,'source'))\n\nmask=((train_df['src'].str.len()>2)&(train_df['tgt'].str.len()>2)&\n (train_df['src'].str.len()<1200)&(train_df['tgt'].str.len()<700))\ntrain_df=train_df[mask].reset_index(drop=True)\nprint(f' Clean pairs: {len(train_df)}')\n\ntr,va=train_test_split(train_df,test_size=CFG.VAL_RATIO,random_state=SEED)\ntr=tr.reset_index(drop=True); va=va.reset_index(drop=True)\n\nALL_SRC=train_df['src'].tolist(); ALL_TGT=train_df['tgt'].tolist()\nTR_SRC=tr['src'].tolist(); TR_TGT=tr['tgt'].tolist()\nVA_SRC=va['src'].tolist(); VA_TGT=va['tgt'].tolist()\nTEST_SRC=test_df['src'].tolist()\ntry: TEST_IDS=test_df[EI].astype(int).tolist()\nexcept: TEST_IDS=test_df[EI].tolist()\n\nprint(f' TR={len(TR_SRC)} VA={len(VA_SRC)} TEST={len(TEST_SRC)}')\n\n# Load Lexicon\nLEX:Dict[str,str]={}\nfor fn in ['OA_Lexicon_eBL.csv','eBL_Dictionary.csv','lexicon.csv']:\n fp=f'{CFG.DATA_DIR}/{fn}'\n if os.path.exists(fp):\n try:\n ld=pd.read_csv(fp,on_bad_lines='skip')\n for _,r in ld.iterrows():\n v=[str(x).strip() for x in r.tolist() if str(x).strip() not in('nan','')]\n if len(v)>=2: LEX[v[0].lower()]=v[1]\n except: pass\nprint(f' Lexicon: {len(LEX)} entries')\n\n\n# ─── CELL 10 · Tier 1: BM25+TF-IDF (ALL data) ────────────────────────────\nprint('\\n'+'═'*66+'\\n TIER 1 · BM25+TF-IDF (ALL 1561 pairs)\\n'+'═'*66)\n\nret=HybridRetriever(ALL_SRC,ALL_TGT)\nret_test=ret.translate(TEST_SRC)\n# Honest val score using TR-only index\nret_tr=HybridRetriever(TR_SRC,TR_TGT)\nrv=Metrics.gm(ret_tr.translate(VA_SRC),VA_TGT)\nRET_SCORE=rv['gm']\nprint(f' BM25+TF-IDF honest val: BLEU={rv[\"bleu\"]:.2f} chrF={rv[\"chrf\"]:.2f} GM={RET_SCORE:.4f}')\n\n\n# ─── CELL 11 · Tier 2: Fine-tuned MarianMT ────────────────────────────────\nprint('\\n'+'═'*66+'\\n TIER 2 · Fine-tuned MarianMT\\n'+'═'*66)\n\nfrom transformers import MarianTokenizer,MarianMTModel\n\nM_OK=False; M_SCORE=0.0; m_test:List[str]=[]\nprint(f' Path: {MARIAN_PATH or \"not found\"}')\n\nif MARIAN_PATH:\n try:\n m_tok=MarianTokenizer.from_pretrained(MARIAN_PATH)\n m_mdl=MarianMTModel.from_pretrained(MARIAN_PATH).to(DEVICE)\n print(f' Loaded: {sum(p.numel() for p in m_mdl.parameters())/1e6:.0f}M params')\n\n # Phase 1: train split\n t1=Trainer(m_mdl,m_tok,CFG,'mar_p1')\n M_SCORE=t1.fit(TR_SRC,TR_TGT,VA_SRC,VA_TGT,lr=CFG.LR)\n m_mdl=MarianMTModel.from_pretrained(t1.ckpt).to(DEVICE); gc.collect()\n\n # Phase 2: ALL data\n ep2=max(2,CFG.EPOCHS//2)\n t2=Trainer(m_mdl,m_tok,CFG,'mar_p2'); t2.cfg.EPOCHS=ep2\n s2=t2.fit(ALL_SRC,ALL_TGT,VA_SRC,VA_TGT,lr=CFG.LR*0.5)\n if s2>M_SCORE: M_SCORE=s2; m_mdl=MarianMTModel.from_pretrained(t2.ckpt).to(DEVICE)\n m_mdl.eval(); gc.collect()\n\n inf_m=Trainer(m_mdl,m_tok,CFG,'mar_inf')\n m_test=inf_m.gen(TEST_SRC,beams=CFG.TE_BEAMS)\n M_OK=True; print(f' Marian: val GM={M_SCORE:.4f}')\n except Exception as e: print(f' Marian failed: {e}')\n\nif HAS_GPU: torch.cuda.empty_cache()\ngc.collect()\n\n\n# ─── CELL 12 · Tier 3: NLLB-600M (GPU) ───────────────────────────────────\nprint('\\n'+'═'*66)\nprint(f' TIER 3 · NLLB-200-600M ({\"GPU — running\" if HAS_GPU else \"CPU — skipped\"})')\nprint('═'*66)\n\nfrom transformers import NllbTokenizerFast,AutoModelForSeq2SeqLM\n\nN_OK=False; N_SCORE=0.0; n_test:List[str]=[]\nprint(f' Path: {NLLB_PATH or \"not found (GPU needed)\"}')\n\nif HAS_GPU and NLLB_PATH:\n try:\n n_tok=NllbTokenizerFast.from_pretrained(NLLB_PATH)\n n_tok.src_lang=CFG.NLLB_SRC\n n_mdl=AutoModelForSeq2SeqLM.from_pretrained(\n NLLB_PATH,torch_dtype=torch.float16,low_cpu_mem_usage=True).to(DEVICE)\n n_bos=n_tok.convert_tokens_to_ids(CFG.NLLB_TGT)\n print(f' Loaded: {sum(p.numel() for p in n_mdl.parameters())/1e6:.0f}M params')\n\n tn1=Trainer(n_mdl,n_tok,CFG,'nllb_p1',nllb_tl=CFG.NLLB_TGT)\n N_SCORE=tn1.fit(TR_SRC,TR_TGT,VA_SRC,VA_TGT,\n lr=CFG.LR*0.2,forced_bos=n_bos,src_lang=CFG.NLLB_SRC)\n n_mdl=AutoModelForSeq2SeqLM.from_pretrained(tn1.ckpt).to(DEVICE); gc.collect()\n\n ep2n=max(2,CFG.EPOCHS//2)\n tn2=Trainer(n_mdl,n_tok,CFG,'nllb_p2',nllb_tl=CFG.NLLB_TGT); tn2.cfg.EPOCHS=ep2n\n s2n=tn2.fit(ALL_SRC,ALL_TGT,VA_SRC,VA_TGT,\n lr=CFG.LR*0.1,forced_bos=n_bos,src_lang=CFG.NLLB_SRC)\n if s2n>N_SCORE: N_SCORE=s2n; n_mdl=AutoModelForSeq2SeqLM.from_pretrained(tn2.ckpt).to(DEVICE)\n n_mdl.eval(); gc.collect()\n\n inf_n=Trainer(n_mdl,n_tok,CFG,'nllb_inf',nllb_tl=CFG.NLLB_TGT)\n n_test=inf_n.gen(TEST_SRC,beams=CFG.TE_BEAMS,forced_bos=n_bos,src_lang=CFG.NLLB_SRC)\n N_OK=True; print(f' NLLB: val GM={N_SCORE:.4f}')\n except Exception as e: print(f' NLLB failed: {e}')\n\nif HAS_GPU: torch.cuda.empty_cache()\ngc.collect()\n\n\n# ─── CELL 13 · Ensemble + Post-process ───────────────────────────────────\nprint('\\n'+'═'*66+'\\n ENSEMBLE\\n'+'═'*66)\nprint(f' BM25+TF-IDF GM={RET_SCORE:.4f}')\nprint(f' Marian GM={M_SCORE:.4f} ({\"✓\" if M_OK else \"✗\"})')\nprint(f' NLLB-600M GM={N_SCORE:.4f} ({\"✓\" if N_OK else \"✗\"})')\n\nif N_OK and M_OK:\n wm=M_SCORE/(M_SCORE+N_SCORE); wn=N_SCORE/(M_SCORE+N_SCORE)\n print(f' Weights: Marian={wm:.2f} NLLB={wn:.2f}')\n raw=[]\n for pm,pn in zip(m_test,n_test):\n cm=pm.strip(); cn=pn.strip()\n choice=cm if (wm>=wn or (abs(wm-wn)<0.05 and len(cm)>=len(cn))) else cn\n raw.append(choice)\nelif M_OK: raw=m_test\nelif N_OK: raw=n_test\nelse: raw=ret_test\n\ndef first_sent(t):\n t=t.strip()\n parts=re.split(r'(?<=[.!?])\\s+(?=[A-Z\\'\"\\u201c])',t)\n first=parts[0].strip() if parts else t\n if len(first)>400: first=first[:first[:400].rfind(' ')]+'...'\n return first if len(first)>3 else t\n\ndef clean(t):\n if not t or (isinstance(t,float) and math.isnan(t)):\n return 'The content of this tablet is fragmentary.'\n t=str(t).strip(); t=re.sub(r'\\s+',' ',t)\n t=re.sub(r'\\s+([.,;:!?])',r'\\1',t)\n t=re.sub(r'\\b(\\w+)( \\1\\b)+',r'\\1',t,flags=re.IGNORECASE)\n if t: t=t[0].upper()+t[1:]\n return t if len(t)>3 else 'The content of this tablet section is unclear.'\n\ndef apply_lex(t):\n for k,v in LEX.items():\n t=re.sub(rf'\\b{re.escape(k)}\\b',v,t,flags=re.IGNORECASE)\n return t\n\nfinal:List[str]=[]\nfor p in raw:\n p=clean(p); p=first_sent(p)\n if LEX: p=apply_lex(p)\n final.append(p)\n\nwhile len(final)