{"cells": [{"cell_type": "markdown", "metadata": {}, "source": "# Deep Past - ByT5 v6 (byt5-base)\n\nUsing larger byt5-base model instead of byt5-small"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Fix protobuf version FIRST\nimport subprocess\nimport sys\nsubprocess.check_call([sys.executable, '-m', 'pip', 'install', '--force-reinstall', \n                       '/kaggle/input/protobuf-old/protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl', '-q'])\nprint('protobuf downgraded to 5.29.2')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "import os, re, math, random\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n)\n\nSEED = 1\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\n\nTRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\nTEST_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nMODEL_PATH = \"/kaggle/input/byt5-base\"  # Changed to byt5-base\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", DEVICE)\n\ntrain_df = pd.read_csv(TRAIN_PATH)\ntest_df = pd.read_csv(TEST_PATH)\nprint(\"train:\", train_df.shape, \"test:\", test_df.shape)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "def clean_ws(x: str) -> str:\n    if pd.isna(x): return \"\"\n    return re.sub(r\"\\s+\", \" \", str(x)).strip()\n\ndef split_src_transliteration(doc: str):\n    doc = \"\" if pd.isna(doc) else str(doc)\n    parts = re.split(r\"\\n+\", doc)\n    parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    if len(parts) <= 1:\n        parts = re.split(r\"\\s{2,}|;\\s+\", clean_ws(doc))\n        parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    return parts\n\ndef split_tgt_english(doc: str):\n    doc = clean_ws(doc)\n    if not doc: return []\n    sents = re.split(r\"(?<=[.!?])\\s+\", doc)\n    return [clean_ws(s) for s in sents if clean_ws(s)]\n\ndef tok_len(s: str) -> int:\n    return max(1, len(s.split()))\n\ndef align_monotone(src_units, tgt_units, max_src_merge=2, max_tgt_merge=2):\n    i = j = 0\n    pairs = []\n    while i < len(src_units) and j < len(tgt_units):\n        best = None\n        for si in range(1, max_src_merge+1):\n            for tj in range(1, max_tgt_merge+1):\n                if i+si <= len(src_units) and j+tj <= len(tgt_units):\n                    src_chunk = \" \".join(src_units[i:i+si])\n                    tgt_chunk = \" \".join(tgt_units[j:j+tj])\n                    ls, lt = tok_len(src_chunk), tok_len(tgt_chunk)\n                    score = abs(math.log(ls/lt))\n                    if (best is None) or (score < best[0]):\n                        best = (score, si, tj, src_chunk, tgt_chunk)\n        if best is None: break\n        _, si, tj, s_chunk, t_chunk = best\n        pairs.append((s_chunk, t_chunk))\n        i += si; j += tj\n    if pairs:\n        if i < len(src_units): pairs[-1] = (pairs[-1][0] + \" \" + \" \".join(src_units[i:]), pairs[-1][1])\n        if j < len(tgt_units): pairs[-1] = (pairs[-1][0], pairs[-1][1] + \" \" + \" \".join(tgt_units[j:]))\n    return pairs"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "pairs = []\nfor _, row in train_df.iterrows():\n    src_lines = split_src_transliteration(row[\"transliteration\"])\n    tgt_sents = split_tgt_english(row[\"translation\"])\n    if len(src_lines) == 0 or len(tgt_sents) == 0: continue\n    pairs.extend(align_monotone(src_lines, tgt_sents))\n\npairs_df = pd.DataFrame(pairs, columns=[\"src\", \"tgt\"])\npairs_df = pairs_df[(pairs_df[\"src\"].str.len() > 0) & (pairs_df[\"tgt\"].str.len() > 0)].drop_duplicates()\nprint(\"Sentence-level pairs:\", pairs_df.shape)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)\nprint(\"Loaded:\", MODEL_PATH)\nprint(\"Model params:\", sum(p.numel() for p in model.parameters()) / 1e6, \"M\")\nmodel.to(DEVICE)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "MAX_SRC = 256\nMAX_TGT = 128\n\nclass MTDataset(Dataset):\n    def __init__(self, df):\n        self.src = df[\"src\"].tolist()\n        self.tgt = df[\"tgt\"].tolist()\n    def __len__(self): return len(self.src)\n    def __getitem__(self, idx):\n        x = tokenizer(self.src[idx], max_length=MAX_SRC, truncation=True)\n        y = tokenizer(text_target=self.tgt[idx], max_length=MAX_TGT, truncation=True)\n        x[\"labels\"] = y[\"input_ids\"]\n        return x\n\nval_frac = 0.08\npairs_df = pairs_df.sample(frac=1.0, random_state=SEED).reset_index(drop=True)\nn_val = max(1, int(len(pairs_df) * val_frac))\nval_df = pairs_df.iloc[:n_val].reset_index(drop=True)\ntrn_df = pairs_df.iloc[n_val:].reset_index(drop=True)\n\ntrain_ds = MTDataset(trn_df)\nval_ds = MTDataset(val_df)\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Reduced batch size for larger model\nargs = Seq2SeqTrainingArguments(\n    output_dir=\"byt5_deeppast\",\n    eval_strategy=\"steps\",\n    eval_steps=100,\n    save_strategy=\"no\",\n    logging_steps=50,\n    learning_rate=3e-4,\n    per_device_train_batch_size=2,  # Reduced from 4\n    per_device_eval_batch_size=2,\n    gradient_accumulation_steps=8,  # Increased to compensate\n    num_train_epochs=30,\n    predict_with_generate=True,\n    fp16=False,\n    report_to=\"none\",\n    remove_unused_columns=False,\n    max_grad_norm=1.0,\n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=train_ds,\n    eval_dataset=val_ds,\n    data_collator=data_collator,\n)\nprint(\"Training ByT5-base (fp32)...\")\ntrainer.train()\nprint(\"Training Done!\")"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "import gc\ndel trainer\ngc.collect()\ntorch.cuda.empty_cache()\n\ndef make_one_sentence(en: str) -> str:\n    en = clean_ws(en)\n    if not en: return \"...\"\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [p.strip() for p in parts if p.strip()]\n    parts = [re.sub(r\"[.!?]+\\s*$\", \"\", p).strip() for p in parts]\n    parts = [p for p in parts if p]\n    out = \", \".join(parts).strip()\n    if out and not out.endswith(\",\"):\n        out += \",\"\n    return out\n\nmodel.eval()\n\ndef generate_translation(src_text: str):\n    enc = tokenizer(\n        clean_ws(src_text),\n        return_tensors=\"pt\",\n        max_length=MAX_SRC,\n        truncation=True,\n    ).to(DEVICE)\n    with torch.no_grad():\n        out_ids = model.generate(\n            **enc,\n            num_beams=4,\n            max_new_tokens=MAX_TGT,\n            length_penalty=1.0,\n            early_stopping=True,\n            no_repeat_ngram_size=3,\n            repetition_penalty=1.2,\n        )\n    pred = tokenizer.decode(out_ids[0], skip_special_tokens=True)\n    return make_one_sentence(pred)\n\ndef long_translation(src_text2: str):\n    end_list = []\n    len_num = len(src_text2) // 256\n    i_stage = 0\n    for i_stage in range(1, len_num + 1):\n        src_temp = src_text2[(i_stage - 1) * 256:i_stage * 256]\n        end_list.append(src_temp)\n    src_temp = src_text2[i_stage * 256:]\n    end_list.append(src_temp)\n    \n    pred_list = [generate_translation(t2) for t2 in end_list]\n    out = \" \".join(pred_list)\n    if not out.endswith(\".\"):\n        out += \".\"\n    return out\n\nprint(\"Ready for inference\")"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "print(\"Generating submission...\")\npreds = [long_translation(t) for t in test_df[\"transliteration\"].tolist()]\nfor i, p in enumerate(preds): print(f\"ID {i}: {p[:80]}...\")\n\nsub = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": preds})\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"Done!\")"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.10.0"}}, "nbformat": 4, "nbformat_minor": 4}