{"cells": [{"cell_type": "markdown", "metadata": {}, "source": "# Deep Past - NLLB-200 v7\n\nBased on v3 (27.7) + Lower LR (1e-4)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Fix protobuf version FIRST\nimport subprocess\nimport sys\nsubprocess.check_call([sys.executable, '-m', 'pip', 'install', '--force-reinstall', \n                       '/kaggle/input/protobuf-old/protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl', '-q'])\nprint('protobuf downgraded to 5.29.2')\n\nimport os, re, math, random, gc\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n)\n\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\n\nMODEL_NAME = \"/kaggle/input/nllb-akkadian-model\"\nDATA_DIR = \"/kaggle/input/deep-past-initiative-machine-translation\"\nOUTPUT_DIR = \"/kaggle/working/model\"\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", DEVICE)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "train_df = pd.read_csv(f\"{DATA_DIR}/train.csv\")\ntest_df = pd.read_csv(f\"{DATA_DIR}/test.csv\")\nprint(\"train:\", train_df.shape, \"test:\", test_df.shape)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "def clean_ws(x: str) -> str:\n    if pd.isna(x): return \"\"\n    return re.sub(r\"\\s+\", \" \", str(x)).strip()\n\ndef split_src_transliteration(doc: str):\n    doc = \"\" if pd.isna(doc) else str(doc)\n    parts = re.split(r\"\\n+\", doc)\n    parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    if len(parts) <= 1:\n        parts = re.split(r\"\\s{2,}|;\\s+\", clean_ws(doc))\n        parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    return parts\n\ndef split_tgt_english(doc: str):\n    doc = clean_ws(doc)\n    if not doc: return []\n    sents = re.split(r\"(?<=[.!?])\\s+\", doc)\n    return [clean_ws(s) for s in sents if clean_ws(s)]\n\ndef tok_len(s: str) -> int:\n    return max(1, len(s.split()))\n\ndef align_monotone(src_units, tgt_units, max_src_merge=2, max_tgt_merge=2):\n    i = j = 0\n    pairs = []\n    while i < len(src_units) and j < len(tgt_units):\n        best = None\n        for si in range(1, max_src_merge+1):\n            for tj in range(1, max_tgt_merge+1):\n                if i+si <= len(src_units) and j+tj <= len(tgt_units):\n                    src_chunk = \" \".join(src_units[i:i+si])\n                    tgt_chunk = \" \".join(tgt_units[j:j+tj])\n                    ls, lt = tok_len(src_chunk), tok_len(tgt_chunk)\n                    score = abs(math.log(ls/lt))\n                    if (best is None) or (score < best[0]):\n                        best = (score, si, tj, src_chunk, tgt_chunk)\n        if best is None: break\n        _, si, tj, s_chunk, t_chunk = best\n        pairs.append((s_chunk, t_chunk))\n        i += si; j += tj\n    if pairs:\n        if i < len(src_units): pairs[-1] = (pairs[-1][0] + \" \" + \" \".join(src_units[i:]), pairs[-1][1])\n        if j < len(tgt_units): pairs[-1] = (pairs[-1][0], pairs[-1][1] + \" \" + \" \".join(tgt_units[j:]))\n    return pairs"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "pairs = []\nfor _, row in train_df.iterrows():\n    src_lines = split_src_transliteration(row[\"transliteration\"])\n    tgt_sents = split_tgt_english(row[\"translation\"])\n    if len(src_lines) == 0 or len(tgt_sents) == 0: continue\n    pairs.extend(align_monotone(src_lines, tgt_sents))\n\npairs_df = pd.DataFrame(pairs, columns=[\"src\", \"tgt\"])\npairs_df = pairs_df[(pairs_df[\"src\"].str.len() > 0) & (pairs_df[\"tgt\"].str.len() > 0)].drop_duplicates()\nprint(f\"Sentence-level pairs: {len(pairs_df)}\")"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, src_lang=\"arb_Arab\", tgt_lang=\"eng_Latn\")\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)\nENG_TOKEN_ID = tokenizer.convert_tokens_to_ids(\"eng_Latn\")\nprint(f\"English token ID: {ENG_TOKEN_ID}\")\nmodel.to(DEVICE)\nprint(f\"Model on {DEVICE}\")"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "MAX_SRC = 256\nMAX_TGT = 128\n\nclass MTDataset(Dataset):\n    def __init__(self, df):\n        self.src = df[\"src\"].tolist()\n        self.tgt = df[\"tgt\"].tolist()\n    def __len__(self): return len(self.src)\n    def __getitem__(self, idx):\n        x = tokenizer(self.src[idx], max_length=MAX_SRC, truncation=True)\n        y = tokenizer(text_target=self.tgt[idx], max_length=MAX_TGT, truncation=True)\n        x[\"labels\"] = y[\"input_ids\"]\n        return x"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "val_frac = 0.08\npairs_df = pairs_df.sample(frac=1.0, random_state=SEED).reset_index(drop=True)\nn_val = max(1, int(len(pairs_df) * val_frac))\nval_df = pairs_df.iloc[:n_val].reset_index(drop=True)\ntrn_df = pairs_df.iloc[n_val:].reset_index(drop=True)\nprint(f\"Train: {len(trn_df)}, Val: {len(val_df)}\")\n\ntrain_ds = MTDataset(trn_df)\nval_ds = MTDataset(val_df)\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "args = Seq2SeqTrainingArguments(\n    output_dir=OUTPUT_DIR,\n    eval_strategy=\"epoch\",\n    save_strategy=\"no\",\n    logging_steps=50,\n    learning_rate=1e-4,  # Lowered from 3e-4\n    per_device_train_batch_size=8,\n    per_device_eval_batch_size=8,\n    gradient_accumulation_steps=2,\n    num_train_epochs=30,\n    warmup_ratio=0.1,\n    fp16=True,\n    report_to=\"none\",\n    remove_unused_columns=False,\n)\n\ntrainer = Seq2SeqTrainer(model=model, args=args, train_dataset=train_ds, eval_dataset=val_ds, data_collator=data_collator)\nprint(\"Training 30 epochs with LR=1e-4...\")\ntrainer.train()\nprint(\"Done!\")"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "del trainer\ngc.collect()\ntorch.cuda.empty_cache()\nmodel.eval()\n\ndef make_one_sentence(en: str) -> str:\n    en = clean_ws(en)\n    if not en: return \"...\"\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [re.sub(r\"[.!?]+\\s*$\", \"\", p).strip() for p in parts if p.strip()]\n    out = \", \".join([p for p in parts if p]).strip()\n    return out + \".\" if out and not out.endswith(\".\") else out or \"...\"\n\ndef generate_translation(src_text: str):\n    enc = tokenizer(clean_ws(src_text), return_tensors=\"pt\", max_length=MAX_SRC, truncation=True).to(DEVICE)\n    with torch.no_grad():\n        out_ids = model.generate(**enc, forced_bos_token_id=ENG_TOKEN_ID, num_beams=4, max_new_tokens=MAX_TGT, length_penalty=1.0, early_stopping=True)\n    return make_one_sentence(tokenizer.decode(out_ids[0], skip_special_tokens=True))\n\ndef long_translation(src_text: str):\n    src_text = clean_ws(src_text)\n    if len(src_text) <= 256: return generate_translation(src_text)\n    chunks = [src_text[i*256:(i+1)*256] for i in range((len(src_text)//256)+1)]\n    chunks = [c for c in chunks if c.strip()]\n    out = \" \".join([generate_translation(c) for c in chunks])\n    return out + \".\" if not out.endswith(\".\") else out\n\nprint(\"Ready for inference\")"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "print(\"Generating submission...\")\npreds = [long_translation(row[\"transliteration\"]) for _, row in test_df.iterrows()]\nfor i, p in enumerate(preds): print(f\"ID {i}: {p[:60]}...\")\n\nsub = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": preds})\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"Submission saved!\")"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.10.0"}}, "nbformat": 4, "nbformat_minor": 4}