{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpuV5e8","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# cell 1: imports & tiny helpers\nimport re\nimport math\nimport random\nimport unicodedata\nfrom typing import List, Tuple\nimport pandas as pd\nfrom tqdm.auto import tqdm\n\n# set tqdm default for pandas apply if desired\ntqdm.pandas()\n\ndef ensure_seed(seed):\n    if seed is None:\n        seed = 0\n    random.seed(seed)\n    return seed\n\ndef clean_ws(x: str) -> str:\n    \"\"\"Normalize whitespace; convert NA to empty string.\"\"\"\n    if pd.isna(x):\n        return \"\"\n    return re.sub(r\"\\s+\", \" \", str(x)).strip()","metadata":{"_uuid":"079b9e43-2aa4-4075-8f8e-7496553df4c1","_cell_guid":"62d82555-e232-47f9-84c8-045365bd65f5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:01.500495Z","iopub.execute_input":"2025-12-22T20:41:01.500741Z","iopub.status.idle":"2025-12-22T20:41:02.585738Z","shell.execute_reply.started":"2025-12-22T20:41:01.500719Z","shell.execute_reply":"2025-12-22T20:41:02.58499Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cell 0 — MUST be the very first cell, restart kernel after inserting\nimport os\n# reduce fragmentation (set before torch import)\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n# optional: limit GPU memory growth (non-portable on some setups)\n# os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"max_split_size_mb:128,expandable_segments:True\"\nprint(\"PYTORCH_CUDA_ALLOC_CONF=\", os.environ.get(\"PYTORCH_CUDA_ALLOC_CONF\"))","metadata":{"_uuid":"5ff1cf96-bfa7-4e30-8e26-78770abdb586","_cell_guid":"e11653d3-c13c-4958-bb9d-042fa46ec606","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:02.58662Z","iopub.execute_input":"2025-12-22T20:41:02.587027Z","iopub.status.idle":"2025-12-22T20:41:02.591643Z","shell.execute_reply.started":"2025-12-22T20:41:02.586992Z","shell.execute_reply":"2025-12-22T20:41:02.590905Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cell 2: transliteration cleaning functions\n# map for unicode subscript digits\n_SUBSCRIPT_DIGITS = str.maketrans(\"0123456789\", \"₀₁₂₃₄₅₆₇₈₉\")\n\ndef convert_trailing_digits_to_subscript(word: str) -> str:\n    \"\"\"\n    Convert trailing digits in a token to subscript digits, e.g. il5 -> il₅.\n    This is conservative: only transforms trailing runs of digits.\n    \"\"\"\n    m = re.match(r\"^(.+?)(\\d+)$\", word)\n    if not m:\n        return word\n    base, digits = m.group(1), m.group(2)\n    return base + digits.translate(_SUBSCRIPT_DIGITS)\n\ndef convert_subscripts_in_text(text: str) -> str:\n    # Apply to tokens to avoid changing standalone numbers\n    parts = re.split(r\"(\\s+)\", text)\n    out = []\n    for p in parts:\n        if p.strip() == \"\":\n            out.append(p)\n        else:\n            out.append(convert_trailing_digits_to_subscript(p))\n    return \"\".join(out)\n\ndef replace_gaps_and_breaks(text: str) -> str:\n    \"\"\"\n    Replace various broken-text notations with <gap> and <big_gap> per your spec.\n    Handles patterns like [...] or … or … … etc.\n    \"\"\"\n    if not text:\n        return text\n    t = text\n    # Explicit patterns\n    t = re.sub(r\"\\[\\s*…\\s*\\]\", \"<big_gap>\", t)          # […] → <big_gap>\n    t = re.sub(r\"\\[\\s*[^]]{1,6}\\s*\\]\", lambda m: re.sub(r\"\\s+\", \" \", m.group(0)), t)  # keep small content inside brackets for intermediate step\n    # ellipses and special cases\n    t = t.replace(\"… …\", \"<big_gap>\")\n    t = t.replace(\"…\", \"<big_gap>\")\n    # square-bracketed explicit broken signs: remove brackets but keep content if short (handled above)\n    t = re.sub(r\"\\[\\s*([^\\]]+?)\\s*\\]\", r\"\\1\", t)\n    # common writing: [KÙ.BABBAR] -> KÙ.BABBAR (done by removing brackets)\n    # small single-sign gaps often represented as [x] or [K] etc — we replaced those above\n    # collapsed gaps\n    t = re.sub(r\"\\.\\.\\.+\", \"<big_gap>\", t)\n    return t\n\n# scribal notations removals\n_SCRIBAL_REMOVE_RE = re.compile(\n    r\"[!?/]\"               # remove ! ? /\n)\n\ndef remove_scribal_markers(text: str, remove_word_dividers: bool = True) -> str:\n    \"\"\"\n    Remove or normalize modern scribal notations:\n     - remove ! ? /\n     - optionally remove ':' and '.' used as word dividers in transliteration (set remove_word_dividers)\n     - remove half-brackets ˹ ˺\n     - remove square brackets (we already stripped many above)\n     - remove partially-broken markers like ˹ and ˺\n    \"\"\"\n    if not text:\n        return text\n    t = text\n    t = _SCRIBAL_REMOVE_RE.sub(\"\", t)\n    # remove half-brackets\n    t = t.replace(\"˹\", \"\").replace(\"˺\", \"\")\n    # remove square brackets residuals\n    t = t.replace(\"[\", \"\").replace(\"]\", \"\")\n    # remove single and double quotes used as scribal line markers (apostrophes after numbers)\n    t = re.sub(r\"(?<=\\d)''?\", \"\", t)  # remove trailing ' or ''\n    if remove_word_dividers:\n        # remove colon or centered dot used as word divider in transliteration\n        t = t.replace(\":\", \"\").replace(\".\", \"\")\n    return t\n\ndef normalize_transliteration(text: str, remove_word_dividers: bool = True) -> str:\n    \"\"\"\n    High-level normalizer for transliteration text.\n    Keeps curly-brace determinatives by default (handled elsewhere).\n    \"\"\"\n    if pd.isna(text) or not str(text).strip():\n        return \"\"\n    t = str(text)\n    t = clean_ws(t)\n    t = unicodedata.normalize(\"NFC\", t)\n    t = replace_gaps_and_breaks(t)\n    t = remove_scribal_markers(t, remove_word_dividers=remove_word_dividers)\n    t = convert_subscripts_in_text(t)\n    # collapse multi-space again\n    t = re.sub(r\"\\s+\", \" \", t).strip()\n    return t","metadata":{"_uuid":"f3e846fb-fa3c-415c-8248-dd283549bb62","_cell_guid":"c2bfe4c1-0afe-4674-bcc6-1a6437654dd2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:02.593259Z","iopub.execute_input":"2025-12-22T20:41:02.593569Z","iopub.status.idle":"2025-12-22T20:41:02.608704Z","shell.execute_reply.started":"2025-12-22T20:41:02.593534Z","shell.execute_reply":"2025-12-22T20:41:02.607921Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cell 3: determinatives helpers\n_DET_PATTERN = re.compile(r\"\\{\\s*([^}]+?)\\s*\\}\")\n\ndef extract_determinatives(text: str) -> List[str]:\n    \"\"\"Return list of determinative contents (without braces) in document order.\"\"\"\n    if not text:\n        return []\n    return _DET_PATTERN.findall(text)\n\ndef tokenize_determinatives(text: str) -> str:\n    \"\"\"Replace {ki} -> <DET:ki> (keeps a trace that a determinative was present).\"\"\"\n    def repl(m):\n        content = m.group(1).strip()\n        # sanitize token text (no spaces)\n        token = content.replace(\" \", \"_\")\n        return f\"<DET:{token}>\"\n    return _DET_PATTERN.sub(repl, text)\n\ndef remove_all_determinatives(text: str) -> str:\n    \"\"\"Remove all {...} including contents.\"\"\"\n    return _DET_PATTERN.sub(\"\", text)\n\ndef drop_determinatives_randomly(text: str, percent: float = 10.0, seed: int = None, mode: str = \"remove\") -> str:\n    \"\"\"\n    Randomly drop a percentage of determinatives present in `text`.\n    - percent: fraction 0..100 of determinatives to drop\n    - seed: RNG seed for reproducibility\n    - mode:\n        - \"remove\": delete the {det} (no replacement)\n        - \"tokenize\": replace all determinatives with <DET:xxx> but then DROP the chosen subset (so some become removed, others tokenized)\n        - \"keep\": leave determinatives in place (no change) -- useful for percent=0\n    Returns modified text.\n    \"\"\"\n    if not text or percent <= 0:\n        return text\n    seed = ensure_seed(seed)\n    dets = list(_DET_PATTERN.finditer(text))\n    if not dets:\n        return text\n    n = len(dets)\n    to_drop_count = int(round(n * (percent / 100.0)))\n    if to_drop_count <= 0:\n        return text\n\n    # choose indices to drop\n    idxs = list(range(n))\n    random.shuffle(idxs)\n    drop_idxs = set(sorted(idxs[:to_drop_count]))\n\n    # build result by iterating through matches and pieces\n    out_parts = []\n    last_idx = 0\n    for i, m in enumerate(dets):\n        start, end = m.span()\n        out_parts.append(text[last_idx:start])  # text before this determinative\n        content = m.group(1).strip()\n        if i in drop_idxs:\n            # drop it entirely\n            # no replacement -> remove\n            replacement = \"\"\n        else:\n            if mode == \"remove\":\n                replacement = m.group(0)  # keep original {..}\n            elif mode == \"tokenize\":\n                token = content.replace(\" \", \"_\")\n                replacement = f\"<DET:{token}>\"\n            elif mode == \"keep\":\n                replacement = m.group(0)\n            else:\n                replacement = m.group(0)\n        out_parts.append(replacement)\n        last_idx = end\n    out_parts.append(text[last_idx:])\n    out = \"\".join(out_parts)\n    # cleanup extra spaces that may have resulted from removals\n    out = re.sub(r\"\\s+\", \" \", out).strip()\n    return out","metadata":{"_uuid":"891b47e9-4199-4e5c-9da0-45ff57038fbd","_cell_guid":"79f015dd-007d-416b-b2b8-e9abc3c04d87","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:02.609546Z","iopub.execute_input":"2025-12-22T20:41:02.609929Z","iopub.status.idle":"2025-12-22T20:41:02.624369Z","shell.execute_reply.started":"2025-12-22T20:41:02.60989Z","shell.execute_reply":"2025-12-22T20:41:02.623755Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"s = \"a-lim{ki} A-mur-{d}UTU some{lu2} text{mi}\"\nprint(\"original:\", s)\nprint(\"extract:\", extract_determinatives(s))\nprint(\"tokenized:\", tokenize_determinatives(s))\nprint(\"drop 25% remove-mode:\", drop_determinatives_randomly(s, percent=25, seed=42, mode=\"remove\"))\nprint(\"drop 50% tokenize-mode:\", drop_determinatives_randomly(s, percent=50, seed=42, mode=\"tokenize\"))","metadata":{"_uuid":"a60adaea-8ad3-4772-aabd-657e890a7b15","_cell_guid":"8680be30-8790-491f-9cff-93e0795cb24f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:02.625192Z","iopub.execute_input":"2025-12-22T20:41:02.625454Z","iopub.status.idle":"2025-12-22T20:41:02.640723Z","shell.execute_reply.started":"2025-12-22T20:41:02.625426Z","shell.execute_reply":"2025-12-22T20:41:02.640108Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cell 4: translation cleaning\ndef normalize_translation(text: str) -> str:\n    \"\"\"\n    Clean English translation text for downstream MT:\n    - normalize whitespace\n    - fix obvious OCR artifacts (multiple hyphens, repeated punctuation)\n    - remove stray scribal markers like solitary / or ? used in translit\n    \"\"\"\n    if pd.isna(text) or not str(text).strip():\n        return \"\"\n    t = str(text)\n    t = unicodedata.normalize(\"NFC\", t)\n    t = clean_ws(t)\n    # collapse repeated punctuation\n    t = re.sub(r\"[-]{2,}\", \"-\", t)\n    t = re.sub(r\"[.]{2,}\", \".\", t)\n    # remove stray isolated characters that are clearly translit artifacts\n    t = re.sub(r\"\\s[/:]\\s\", \" \", t)\n    return t.strip()","metadata":{"_uuid":"29ff268c-a20e-49df-86a2-a8d1083a759c","_cell_guid":"4ca33e9a-deed-4328-a5b4-e2bb792c5ad8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:02.642153Z","iopub.execute_input":"2025-12-22T20:41:02.642471Z","iopub.status.idle":"2025-12-22T20:41:02.656484Z","shell.execute_reply.started":"2025-12-22T20:41:02.642442Z","shell.execute_reply":"2025-12-22T20:41:02.655833Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cell 5: splitting and aligner\ndef split_src_transliteration(doc: str) -> List[str]:\n    \"\"\"Split transliteration doc into 'lines' as proxy for sentences.\"\"\"\n    doc = \"\" if pd.isna(doc) else str(doc)\n    parts = re.split(r\"\\n+\", doc)\n    parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    # fallback: if no newlines, split on \" ; \" or long spaces\n    if len(parts) <= 1:\n        parts = re.split(r\"\\s{2,}|;\\s+\", clean_ws(doc))\n        parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    return parts\n\ndef split_tgt_english(doc: str) -> List[str]:\n    \"\"\"Split English translation into sentences (simple regex).\"\"\"\n    doc = clean_ws(doc)\n    if not doc:\n        return []\n    sents = re.split(r\"(?<=[.!?])\\s+\", doc)\n    sents = [clean_ws(s) for s in sents if clean_ws(s)]\n    return sents\n\ndef tok_len(s: str) -> int:\n    return max(1, len(s.split()))\n\ndef align_monotone(src_units: List[str], tgt_units: List[str], max_src_merge=2, max_tgt_merge=2) -> List[Tuple[str,str]]:\n    \"\"\"\n    Monotone greedy alignment by matching length proportions.\n    Returns list of (src_chunk, tgt_chunk).\n    \"\"\"\n    i = j = 0\n    pairs = []\n    while i < len(src_units) and j < len(tgt_units):\n        best = None\n        for si in range(1, max_src_merge+1):\n            for tj in range(1, max_tgt_merge+1):\n                if i+si <= len(src_units) and j+tj <= len(tgt_units):\n                    src_chunk = \" \".join(src_units[i:i+si])\n                    tgt_chunk = \" \".join(tgt_units[j:j+tj])\n                    ls, lt = tok_len(src_chunk), tok_len(tgt_chunk)\n                    # avoid division by zero and tiny ratios\n                    score = abs(math.log((ls + 1e-9)/(lt + 1e-9)))\n                    if (best is None) or (score < best[0]):\n                        best = (score, si, tj, src_chunk, tgt_chunk)\n        if best is None:\n            break\n        _, si, tj, s_chunk, t_chunk = best\n        pairs.append((s_chunk, t_chunk))\n        i += si\n        j += tj\n\n    # attach leftovers to last pair if any\n    if pairs:\n        if i < len(src_units):\n            pairs[-1] = (pairs[-1][0] + \" \" + \" \".join(src_units[i:]), pairs[-1][1])\n        if j < len(tgt_units):\n            pairs[-1] = (pairs[-1][0], pairs[-1][1] + \" \" + \" \".join(tgt_units[j:]))\n    return pairs","metadata":{"_uuid":"897f112c-f87c-4c11-8031-db243bea35df","_cell_guid":"9cf18a4f-ba71-4a05-aa57-fa3fb44bb417","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:02.657372Z","iopub.execute_input":"2025-12-22T20:41:02.657628Z","iopub.status.idle":"2025-12-22T20:41:02.6691Z","shell.execute_reply.started":"2025-12-22T20:41:02.6576Z","shell.execute_reply":"2025-12-22T20:41:02.668389Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cell 6: apply pipeline to train.csv\ndef process_train_csv(\n    path_in: str,\n    path_out_clean_docs: str = \"train_clean_docs.csv\",\n    path_out_sentence_pairs: str = \"train_sentence_pairs.csv\",\n    drop_det_percent: float = 0.0,\n    det_seed: int = 0,\n    det_mode: str = \"remove\"\n):\n    \"\"\"\n    Read train.csv, clean transliteration + translation, optionally randomly drop determinatives\n    across the transliteration text (percent), and produce sentence-level pairs.\n    Args:\n      path_in: input CSV path (expects columns transliteration, translation, id optional)\n      drop_det_percent: 0..100 percent of determinatives to randomly drop\n      det_mode: \"remove\" (leave remaining determinatives as-is), \"tokenize\", or \"keep\"\n    Returns:\n      (clean_docs_df, sentence_pairs_df)\n    \"\"\"\n    df = pd.read_csv(path_in)\n    # ensure expected columns\n    assert \"transliteration\" in df.columns and \"translation\" in df.columns, \"Input must have transliteration & translation columns.\"\n\n    # cleaning\n    clean_rows = []\n    for _, row in tqdm(df.iterrows(), total=len(df), desc=\"cleaning rows\"):\n        tid = row.get(\"id\", None)\n        raw_src = row[\"transliteration\"]\n        raw_tgt = row[\"translation\"]\n\n        src = normalize_transliteration(raw_src, remove_word_dividers=True)\n        # apply deterministic random dropping of determinatives BEFORE tokenization step\n        if drop_det_percent and drop_det_percent > 0:\n            src = drop_determinatives_randomly(src, percent=drop_det_percent, seed=det_seed, mode=det_mode)\n        # optionally tokenize determinatives for modeling convenience\n        src_tokenized = tokenize_determinatives(src)\n\n        tgt = normalize_translation(raw_tgt)\n\n        clean_rows.append({\n            \"id\": tid,\n            \"src_raw\": raw_src,\n            \"tgt_raw\": raw_tgt,\n            \"src_clean\": src,\n            \"src_tokenized\": src_tokenized,\n            \"tgt_clean\": tgt\n        })\n\n    clean_docs_df = pd.DataFrame(clean_rows)\n    clean_docs_df.to_csv(path_out_clean_docs, index=False)\n\n    # produce sentence pairs\n    pairs = []\n    for _, r in tqdm(clean_docs_df.iterrows(), total=len(clean_docs_df), desc=\"building sentence pairs\"):\n        src_doc = r[\"src_tokenized\"]\n        tgt_doc = r[\"tgt_clean\"]\n        src_units = split_src_transliteration(src_doc)\n        tgt_units = split_tgt_english(tgt_doc)\n        if not src_units or not tgt_units:\n            continue\n        doc_pairs = align_monotone(src_units, tgt_units, max_src_merge=2, max_tgt_merge=2)\n        for s, t in doc_pairs:\n            s_clean = clean_ws(s)\n            t_clean = clean_ws(t)\n            if s_clean and t_clean:\n                pairs.append((s_clean, t_clean))\n    pairs_df = pd.DataFrame(pairs, columns=[\"src\", \"tgt\"]).drop_duplicates().reset_index(drop=True)\n    pairs_df.to_csv(path_out_sentence_pairs, index=False)\n\n    return clean_docs_df, pairs_df\n\n# Example usage (uncomment and run in notebook):\nclean_docs_df, pairs_df = process_train_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\", drop_det_percent=10, det_seed=42, det_mode=\"tokenize\")\nprint(\"Clean docs:\", clean_docs_df.shape, \"Pairs:\", pairs_df.shape)","metadata":{"_uuid":"012e37b0-5ea3-4bc6-a479-11c15b94cd60","_cell_guid":"81711fba-22dd-4089-8cf2-eca356a20c1c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:02.670149Z","iopub.execute_input":"2025-12-22T20:41:02.670411Z","iopub.status.idle":"2025-12-22T20:41:04.217065Z","shell.execute_reply.started":"2025-12-22T20:41:02.670385Z","shell.execute_reply":"2025-12-22T20:41:04.216302Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pairs_df","metadata":{"_uuid":"2eceac43-06d9-40fd-a8b7-0eb390f93a0c","_cell_guid":"5d6240bd-3b13-4743-862d-faef23c1b7e5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:04.219692Z","iopub.execute_input":"2025-12-22T20:41:04.219972Z","iopub.status.idle":"2025-12-22T20:41:04.241685Z","shell.execute_reply.started":"2025-12-22T20:41:04.219947Z","shell.execute_reply":"2025-12-22T20:41:04.240993Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cell 7: tests\n_examples = [\n    \"a-lim{ki} A-mur-{d}UTU ... [KÙ.BABBAR] il5 il3\",\n    \"Line1 / Line2 ?! : : [x] ˹abc˺\",\n    \"Broken … … text [K] more\"\n]\n\nfor e in _examples:\n    print(\"ORIG:\", e)\n    n = normalize_transliteration(e)\n    print(\"CLEAN:\", n)\n    print(\"TOK:\", tokenize_determinatives(n))\n    print(\"DROP50:\", drop_determinatives_randomly(n, percent=50, seed=1, mode=\"tokenize\"))\n    print(\"---\")","metadata":{"_uuid":"26d5494c-79f7-4df8-9e1b-1fff2ef4fb51","_cell_guid":"da3a515a-f973-4fdf-9d51-ddfaf7ea4252","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:04.242764Z","iopub.execute_input":"2025-12-22T20:41:04.243178Z","iopub.status.idle":"2025-12-22T20:41:04.249631Z","shell.execute_reply.started":"2025-12-22T20:41:04.243143Z","shell.execute_reply":"2025-12-22T20:41:04.248774Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport time\nimport requests\nimport pandas as pd\nfrom bs4 import BeautifulSoup\nfrom langdetect import detect, DetectorFactory\n\nDetectorFactory.seed = 0  # reproducible","metadata":{"_uuid":"4d2f2c7a-63eb-4868-87b2-b2f4a980557b","_cell_guid":"22986d28-051c-49c8-a704-109d24a75efb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:11.154711Z","iopub.execute_input":"2025-12-22T20:41:11.155073Z","iopub.status.idle":"2025-12-22T20:41:11.550511Z","shell.execute_reply.started":"2025-12-22T20:41:11.155026Z","shell.execute_reply":"2025-12-22T20:41:11.549829Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pub = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv\", dtype=str)\nprint(\"originally:\",len(pub))\npub = pub[\n    pub[\"AICC_translation\"].notna() &\n    pub[\"AICC_translation\"].str.startswith(\"http\")\n].reset_index(drop=True)\n\nprint(\"Rows with AICC links:\", len(pub))","metadata":{"_uuid":"5e3d090b-5715-461d-8e20-60d02ec15800","_cell_guid":"586cb9fb-7861-4469-a806-e950f21034e1","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:11.55141Z","iopub.execute_input":"2025-12-22T20:41:11.55179Z","iopub.status.idle":"2025-12-22T20:41:11.840922Z","shell.execute_reply.started":"2025-12-22T20:41:11.551765Z","shell.execute_reply":"2025-12-22T20:41:11.840181Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import requests\nimport json\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\n\nBASE_JSON = \"https://aicuneiform.com/p/p{prefix}.json\"\nMAX_WORKERS = 6\n\n# ----------------------------\n# helpers\n# ----------------------------\ndef extract_pnums(text):\n    return re.findall(r\"\\bP\\d{5,7}\\b\", str(text))\n\ndef p_prefix(p):\n    return p[1:4]  # P359543 -> 359\n\ndef load_pjson(prefix):\n    url = BASE_JSON.format(prefix=prefix)\n    r = requests.get(url, timeout=20)\n    r.raise_for_status()\n    return r.json()\n\n# ----------------------------\n# core extraction\n# ----------------------------\ndef extract_tablet_texts(pjson, wanted_pnums):\n    \"\"\"\n    Returns: list of dicts\n      {tablet, akk_raw, ai_raw}\n    \"\"\"\n    rows = []\n    for pid, obj in pjson.items():\n        if pid not in wanted_pnums:\n            continue\n        html = obj.get(\"html\", \"\")\n        if not html:\n            continue\n\n        soup = BeautifulSoup(html, \"html.parser\")\n\n        akk = \" \".join(\n            x.get_text(\" \", strip=True)\n            for x in soup.select(\".lang-akk .line\")\n        )\n        ai = \" \".join(\n            x.get_text(\" \", strip=True)\n            for x in soup.select(\".lang-ml_en .line, .lang-en .line\")\n        )\n\n        if akk.strip() or ai.strip():\n            rows.append({\n                \"tablet\": pid,\n                \"akk_raw\": akk,\n                \"ai_raw\": ai\n            })\n    return rows\n\n# ----------------------------\n# main pipeline\n# ----------------------------\ndef build_parallel_corpus(pub_df, link_col=\"AICC_translation\",\n                          det_mode=\"tokenize\"):\n    \"\"\"\n    det_mode: 'tokenize' | 'remove' | 'keep'\n    \"\"\"\n    # 1. collect P numbers\n    all_pnums = set()\n    for x in pub_df[link_col]:\n        all_pnums.update(extract_pnums(x))\n\n    if not all_pnums:\n        raise ValueError(\"No P-numbers found\")\n\n    # 2. group by prefix\n    by_prefix = {}\n    for p in all_pnums:\n        by_prefix.setdefault(p_prefix(p), set()).add(p)\n\n    # 3. load JSONs\n    tablet_rows = []\n    with ThreadPoolExecutor(MAX_WORKERS) as ex:\n        futures = {\n            ex.submit(load_pjson, pref): pref\n            for pref in by_prefix\n        }\n        for fut in tqdm(as_completed(futures), total=len(futures), desc=\"JSON\"):\n            pref = futures[fut]\n            try:\n                pjson = fut.result()\n            except Exception:\n                continue\n            tablet_rows.extend(\n                extract_tablet_texts(pjson, by_prefix[pref])\n            )\n\n    df = pd.DataFrame(tablet_rows)\n    if df.empty:\n        return df\n\n    # ----------------------------\n    # cleaning (your functions)\n    # ----------------------------\n    df[\"akk_norm\"] = (\n        df[\"akk_raw\"]\n        .map(normalize_transliteration)\n        .map(lambda x: tokenize_determinatives(x)\n             if det_mode == \"tokenize\"\n             else remove_all_determinatives(x)\n             if det_mode == \"remove\"\n             else x)\n    )\n\n    df[\"ai_norm\"] = df[\"ai_raw\"].map(normalize_translation)\n\n    # drop tablets with no usable text\n    df = df[\n        (df[\"akk_norm\"].str.strip() != \"\") &\n        (df[\"ai_norm\"].str.strip() != \"\")\n    ].reset_index(drop=True)\n\n    # ----------------------------\n    # splitting + alignment\n    # ----------------------------\n    aligned_rows = []\n\n    for _, r in tqdm(df.iterrows(), total=len(df), desc=\"Align\"):\n        src_units = split_src_transliteration(r[\"akk_norm\"])\n        tgt_units = split_tgt_english(r[\"ai_norm\"])\n        if not src_units or not tgt_units:\n            continue\n\n        pairs = align_monotone(src_units, tgt_units)\n\n        for s, t in pairs:\n            aligned_rows.append({\n                \"tablet\": r[\"tablet\"],\n                \"src\": s,\n                \"tgt\": t,\n                \"src_len\": tok_len(s),\n                \"tgt_len\": tok_len(t),\n            })\n\n    out = pd.DataFrame(aligned_rows)\n\n    # final hygiene\n    out = out[\n        (out[\"src\"].str.strip() != \"\") &\n        (out[\"tgt\"].str.strip() != \"\")\n    ].reset_index(drop=True)\n\n    return out\nscraped_df = build_parallel_corpus(pub, link_col=\"AICC_translation\",\n                          det_mode=\"tokenize\")","metadata":{"_uuid":"35460a35-beae-4c4f-a49c-b2bc8e58e608","_cell_guid":"5f221acc-1605-4393-a0c7-8295e0aceee8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:30.068617Z","iopub.execute_input":"2025-12-22T20:41:30.068998Z","iopub.status.idle":"2025-12-22T20:41:45.579128Z","shell.execute_reply.started":"2025-12-22T20:41:30.068971Z","shell.execute_reply":"2025-12-22T20:41:45.578468Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(scraped_df.drop_duplicates())","metadata":{"_uuid":"0261c400-a533-49f6-8d8a-e2324ebeb727","_cell_guid":"39741109-b0b1-4ba3-a2d1-9023461fdf77","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:45.580033Z","iopub.execute_input":"2025-12-22T20:41:45.58035Z","iopub.status.idle":"2025-12-22T20:41:45.606242Z","shell.execute_reply.started":"2025-12-22T20:41:45.580316Z","shell.execute_reply":"2025-12-22T20:41:45.605568Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scraped_df","metadata":{"_uuid":"f22c82c6-fd8e-41cf-9d8e-b8a4e3bef019","_cell_guid":"a8ca9719-d0ff-48f8-a977-bfbf828bac3e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:45.607242Z","iopub.execute_input":"2025-12-22T20:41:45.607944Z","iopub.status.idle":"2025-12-22T20:41:45.616527Z","shell.execute_reply.started":"2025-12-22T20:41:45.607905Z","shell.execute_reply":"2025-12-22T20:41:45.615923Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# parallel_df['tgt_lang'].value_counts()","metadata":{"_uuid":"03d970f5-0354-4ab3-8276-4d84ae4f955b","_cell_guid":"e3a52f01-3e22-4232-9b83-a78aec2d1526","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:45.630193Z","iopub.execute_input":"2025-12-22T20:41:45.630481Z","iopub.status.idle":"2025-12-22T20:41:45.64004Z","shell.execute_reply.started":"2025-12-22T20:41:45.630436Z","shell.execute_reply":"2025-12-22T20:41:45.639369Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tok_len(s: str) -> int:\n    if not isinstance(s, str):\n        return 0\n    return len(s.split())\n\n\n\nscraped_df = scraped_df[\n    (scraped_df[\"src_len\"] > 0) &\n    (scraped_df[\"tgt_len\"] > 0)\n]\n\nscraped_df[\"len_ratio\"] = scraped_df[\"tgt_len\"] / scraped_df[\"src_len\"]\n\nscraped_df = scraped_df[\n    (scraped_df[\"len_ratio\"] >= 0.2) &\n    (scraped_df[\"len_ratio\"] <= 4.0)\n]\nlen(scraped_df)","metadata":{"_uuid":"dce81c6d-cff5-424f-a904-374fc612630c","_cell_guid":"69fd3fa9-0255-4404-a34e-2610a3196c83","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:45.640846Z","iopub.execute_input":"2025-12-22T20:41:45.641115Z","iopub.status.idle":"2025-12-22T20:41:45.656307Z","shell.execute_reply.started":"2025-12-22T20:41:45.641083Z","shell.execute_reply":"2025-12-22T20:41:45.655552Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef punct_ratio(s: str) -> float:\n    if not isinstance(s, str) or not s:\n        return 0.0\n    punct = len(re.findall(r\"[^\\w\\s]\", s))\n    return punct / max(len(s.split()), 1)\n\nscraped_df[\"src_punct\"] = scraped_df[\"src\"].apply(punct_ratio)\nscraped_df[\"tgt_punct\"] = scraped_df[\"tgt\"].apply(punct_ratio)\n\nscraped_df = scraped_df[\n    scraped_df[\"tgt_punct\"] <= 1.5 * scraped_df[\"src_punct\"] + 0.5\n]\nlen(scraped_df)","metadata":{"_uuid":"9af78982-67dd-47a2-a46c-e87c3045426d","_cell_guid":"dcbaa8cb-3e11-49b0-b421-8710aed94cea","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:45.657277Z","iopub.execute_input":"2025-12-22T20:41:45.657616Z","iopub.status.idle":"2025-12-22T20:41:45.918569Z","shell.execute_reply.started":"2025-12-22T20:41:45.657588Z","shell.execute_reply":"2025-12-22T20:41:45.917645Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_numbers(s: str):\n    return set(re.findall(r\"\\d+\", s))\n\nscraped_df[\"src_nums\"] = scraped_df[\"src\"].apply(extract_numbers)\nscraped_df[\"tgt_nums\"] = scraped_df[\"tgt\"].apply(extract_numbers)\n\nscraped2_df = scraped_df[\n    scraped_df[\"src_nums\"] == scraped_df[\"tgt_nums\"]\n]\nlen(scraped2_df)","metadata":{"_uuid":"ba47a7eb-cf77-4d6b-ad7d-59d8d91642aa","_cell_guid":"04700bad-f932-44a1-8530-dd72e22042df","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:45.919637Z","iopub.execute_input":"2025-12-22T20:41:45.91989Z","iopub.status.idle":"2025-12-22T20:41:46.066242Z","shell.execute_reply.started":"2025-12-22T20:41:45.919844Z","shell.execute_reply":"2025-12-22T20:41:46.065679Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BAD_PATTERNS = [\n    \"this line\",\n    \"the text\",\n    \"means that\",\n    \"refers to\",\n    \"uncertain\",\n    \"possibly\",\n    \"translation:\",\n    \"note:\",\n]\n\ndef looks_like_commentary(s: str) -> bool:\n    s = s.lower()\n    return any(p in s for p in BAD_PATTERNS)\n\nscraped_df = scraped_df[~scraped_df[\"tgt\"].apply(looks_like_commentary)]","metadata":{"_uuid":"96b48a28-a801-4f8f-8bea-266b7c855ba0","_cell_guid":"541063c5-93f7-4ded-85ee-62b80e5fa0e6","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:41:46.06721Z","iopub.execute_input":"2025-12-22T20:41:46.06752Z","iopub.status.idle":"2025-12-22T20:41:46.105467Z","shell.execute_reply.started":"2025-12-22T20:41:46.067486Z","shell.execute_reply":"2025-12-22T20:41:46.105013Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SPECIALS = [\n    \"<gap>\", \"<big_gap>\",\n    \"<DET:ki>\", \"<DET:d>\", \"<DET:lu2>\", \"<DET:m>\", \"<DET:mi>\",\n    \"<DET:uru>\", \"<DET:kur>\", \"<DET:e2>\", \"<DET:geš>\",\n    \"<DET:tug2>\", \"<DET:dub>\", \"<DET:id2>\", \"<DET:mušen>\",\n    \"<DET:na4>\", \"<DET:kuš>\", \"<DET:u2>\"\n]\n\n\n# ---------- model & tokenizer (replace existing block) ----------\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nimport torch\n\np = \"google/mt5-small\"   # or your chosen model\ntok = AutoTokenizer.from_pretrained(p)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(p)\n\n# Decide speed vs memory: if GPU total mem > 12GB, prefer no checkpointing (faster)\n_has_big_gpu = False\nif torch.cuda.is_available():\n    try:\n        props = torch.cuda.get_device_properties(0)\n        total_gb = props.total_memory / (1024**3)\n        _has_big_gpu = total_gb >= 12.0\n    except Exception:\n        _has_big_gpu = False\n\nif _has_big_gpu:\n    # FAST mode: no checkpointing (faster per step)\n    # only enable this if you successfully ran once without OOM\n    try:\n        model.gradient_checkpointing_disable()\n    except Exception:\n        pass\n    model.config.use_cache = True\nelse:\n    # MEMORY-SAFE mode: enable checkpointing to avoid OOM (slower)\n    model.gradient_checkpointing_enable()\n    model.config.use_cache = False\n\n# Reduce beams and cap generation length during eval\nmodel.config.num_beams = 2   # 1 is greedy; 2 is light beam\n# we'll set generation max length via training args (below)\n\n\n_new = [t for t in SPECIALS if t not in tok.get_vocab()]\nif _new:\n    tok.add_tokens(_new, special_tokens=True)\n    model.resize_token_embeddings(len(tok))\n\nmodel.to('cuda')\n\nprint(\"Model loaded. Big GPU:\", _has_big_gpu, \"-> checkpointing enabled:\", not _has_big_gpu)\n# -----------------------------------------------------------------","metadata":{"_uuid":"f8c50a63-b71d-45bd-8130-78ec3b172dac","_cell_guid":"7af5b2c5-4033-469a-bf02-fa27d482d8ce","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-12-22T20:42:55.407686Z","iopub.execute_input":"2025-12-22T20:42:55.408035Z","iopub.status.idle":"2025-12-22T20:43:07.720196Z","shell.execute_reply.started":"2025-12-22T20:42:55.408004Z","shell.execute_reply":"2025-12-22T20:43:07.71939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Tokenize scraped_df and paired_df using the existing tokenizer `tok`.\n# Produces columns: src_ids, src_mask, tgt_ids, tgt_mask (lists of ints).\n\nfrom tqdm.auto import tqdm\nimport math\n\n# --- config ---\nbatch_size = 64\nmax_src_len = 192\nmax_tgt_len = 128\n\n\n# add any missing special tokens and resize model embeddings if model object exists\nnew_specials = [t for t in SPECIALS if t not in tok.get_vocab()]\nif new_specials:\n    tok.add_tokens(new_specials, special_tokens=True)\n    try:\n        mdl  # noqa: F821\n        mdl.resize_token_embeddings(len(tok))\n    except NameError:\n        # model not present in environment; that's fine\n        pass\n\n# helper: batch tokenize a list of strings, return list of dicts with input_ids and attention_mask\ndef batch_tokenize_texts(texts, tokenizer, max_length=512, role=\"src\", prefix=\"\"):\n    outs = []\n    for i in range(0, len(texts), batch_size):\n        batch = texts[i:i + batch_size]\n        # optionally prepend a task prefix for mT5-style models (uncomment if you want)\n        # batch = [prefix + t if t is not None else \"\" for t in batch]\n        enc = tokenizer(batch,\n                        padding=False,         # leave variable-length; use DataCollator later\n                        truncation=True,\n                        max_length=max_length,\n                        return_attention_mask=True)\n        # enc[\"input_ids\"] is a list of lists\n        for ids, mask in zip(enc[\"input_ids\"], enc[\"attention_mask\"]):\n            outs.append({\"input_ids\": ids, \"attention_mask\": mask})\n    return outs\n\n# tokenize a DataFrame in-place; df must have src_col and tgt_col\ndef tokenize_dataframe(df, src_col=\"src\", tgt_col=\"tgt\",\n                       tokenizer=tok, max_src=max_src_len, max_tgt=max_tgt_len):\n    # prepare text lists\n    src_texts = df[src_col].fillna(\"\").astype(str).tolist()\n    tgt_texts = df[tgt_col].fillna(\"\").astype(str).tolist()\n\n    # batch tokenize\n    src_tok = batch_tokenize_texts(src_texts, tokenizer, max_length=max_src, role=\"src\")\n    tgt_tok = batch_tokenize_texts(tgt_texts, tokenizer, max_length=max_tgt, role=\"tgt\")\n\n    assert len(src_tok) == len(df) and len(tgt_tok) == len(df)\n\n    # add columns (lists of ints)\n    df = df.copy()\n    df[\"src_ids\"] = [x[\"input_ids\"] for x in src_tok]\n    df[\"src_mask\"] = [x[\"attention_mask\"] for x in src_tok]\n    df[\"tgt_ids\"] = [x[\"input_ids\"] for x in tgt_tok]\n    df[\"tgt_mask\"] = [x[\"attention_mask\"] for x in tgt_tok]\n\n    # For seq2seq training you will want 'labels' where padding tokens are -100.\n    # Since we did not pad here, labels == tgt_ids (collator will pad & convert pad_token_id -> -100).\n    df[\"labels\"] = df[\"tgt_ids\"]\n\n    return df\n\n# --- run tokenization on your DataFrames ---\n# expects scraped_df has columns: ['tablet','src','tgt', ...]\n# expects paired_df has columns: ['src','tgt', ...]\nscraped_df_tok = tokenize_dataframe(scraped_df, src_col=\"src\", tgt_col=\"tgt\")\npairs_df_tok  = tokenize_dataframe(pairs_df,  src_col=\"src\", tgt_col=\"tgt\")\n\n# quick sanity prints\nprint(\"scraped_df_tok:\", len(scraped_df_tok), \"rows; example src_ids length:\", len(scraped_df_tok.loc[0,'src_ids']))\nprint(\"paired_df_tok: \", len(pairs_df_tok),  \"rows; example tgt_ids length:\", len(pairs_df_tok.loc[0,'tgt_ids']))","metadata":{"_uuid":"f4fb98f9-3462-47e8-868e-275b63ce1076","_cell_guid":"c52f6130-5d91-407a-9c1b-e833ca94c868","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-12-22T20:43:07.721548Z","iopub.execute_input":"2025-12-22T20:43:07.721789Z","iopub.status.idle":"2025-12-22T20:43:13.399169Z","shell.execute_reply.started":"2025-12-22T20:43:07.721765Z","shell.execute_reply":"2025-12-22T20:43:13.398398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pairs_df_tok.head()","metadata":{"_uuid":"428f713c-e2a7-4727-9e0a-06592f69e8c3","_cell_guid":"6de34dac-21d2-403d-bff6-660efdc3bd37","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:43:13.400043Z","iopub.execute_input":"2025-12-22T20:43:13.4003Z","iopub.status.idle":"2025-12-22T20:43:13.418698Z","shell.execute_reply.started":"2025-12-22T20:43:13.400257Z","shell.execute_reply":"2025-12-22T20:43:13.418019Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pairs_df_tok[\"quality\"] = \"high\"\nscraped_df_tok[\"quality\"] = \"low\"\nratio = 2  # scraped per paired\n\nn_high = len(pairs_df_tok)\nn_low  = min(len(scraped_df_tok), n_high * ratio)\n\nscraped_sample = scraped_df_tok.sample(\n    n=n_low,\n    random_state=42,\n    replace=False\n)\nmixed_df = pd.concat(\n    [pairs_df_tok, scraped_sample],\n    ignore_index=True\n)\nmixed_df = mixed_df.sample(frac=1.0, random_state=42).reset_index(drop=True)\nmixed_df = mixed_df.dropna(how='all')\nprint(len(mixed_df))\nmixed_df.head()","metadata":{"_uuid":"aa78f007-2b32-4a4c-b276-a616f92d951f","_cell_guid":"bc3887b4-af4b-4863-b36f-4e880d2af806","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:43:13.420305Z","iopub.execute_input":"2025-12-22T20:43:13.420793Z","iopub.status.idle":"2025-12-22T20:43:13.467616Z","shell.execute_reply.started":"2025-12-22T20:43:13.420753Z","shell.execute_reply":"2025-12-22T20:43:13.46707Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import random, numpy as np, torch\n\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", DEVICE)","metadata":{"_uuid":"b7125801-8dea-444d-aa6c-503bfc9e6c9a","_cell_guid":"e33c14f6-0815-466b-b2ca-33dbbc2f08fa","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:43:13.468389Z","iopub.execute_input":"2025-12-22T20:43:13.468655Z","iopub.status.idle":"2025-12-22T20:43:13.477307Z","shell.execute_reply.started":"2025-12-22T20:43:13.468631Z","shell.execute_reply":"2025-12-22T20:43:13.476582Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def split_clean_train_val(df, val_frac=0.08, seed=SEED):\n    df = df.sample(frac=1.0, random_state=seed).reset_index(drop=True)\n    n_val = max(1, int(len(df) * val_frac))\n    val_df = df.iloc[:n_val].reset_index(drop=True)\n    train_df = df.iloc[n_val:].reset_index(drop=True)\n    return train_df, val_df\n\npaired_train_df, paired_val_df = split_clean_train_val(pairs_df_tok)\nprint(\"paired train:\", len(paired_train_df))\nprint(\"paired val:\", len(paired_val_df))","metadata":{"_uuid":"55a79e7c-9034-4e92-b01d-ed800f596522","_cell_guid":"2245ff54-e39c-4171-a0c6-145b2ec24677","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:43:13.478222Z","iopub.execute_input":"2025-12-22T20:43:13.478582Z","iopub.status.idle":"2025-12-22T20:43:13.48845Z","shell.execute_reply.started":"2025-12-22T20:43:13.478546Z","shell.execute_reply":"2025-12-22T20:43:13.487912Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import Dataset\n\nclass TokMTDataset(Dataset):\n    def __init__(self, df):\n        self.df = df.reset_index(drop=True)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        return {\n            \"input_ids\": torch.tensor(row[\"src_ids\"], dtype=torch.long),\n            \"attention_mask\": torch.tensor(row[\"src_mask\"], dtype=torch.long),\n            \"labels\": torch.tensor(row[\"labels\"], dtype=torch.long),\n        }","metadata":{"_uuid":"e5fcb3ff-73a2-48ec-882e-44b65d82e948","_cell_guid":"ea83b99f-ce6c-4f44-9ac0-43d68ef37554","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:43:13.489746Z","iopub.execute_input":"2025-12-22T20:43:13.490066Z","iopub.status.idle":"2025-12-22T20:43:13.500646Z","shell.execute_reply.started":"2025-12-22T20:43:13.490043Z","shell.execute_reply":"2025-12-22T20:43:13.500046Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import (\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments,\n    DataCollatorForSeq2Seq,\n)\n\ndef run_phase(\n    model,\n    tokenizer,\n    train_df,\n    val_df,\n    *,\n    phase_name,\n    epochs,\n    lr,\n    batch_size,\n    output_dir\n):\n    train_ds = TokMTDataset(train_df)\n    val_ds   = TokMTDataset(val_df)\n\n    data_collator = DataCollatorForSeq2Seq(\n        tokenizer=tokenizer,\n        model=model,\n        padding=True\n    )\n\n    # ---------- Replace args = Seq2SeqTrainingArguments(...) in run_phase ----------\n    args = Seq2SeqTrainingArguments(\n        output_dir=output_dir,\n        # less frequent / cheaper eval\n        eval_strategy=\"steps\",\n        eval_steps=500,                # fewer evals -> much faster\n        logging_steps=100,\n        save_steps=1000,\n        save_total_limit=2,\n    \n        # batching: try to increase GPU utilization\n        learning_rate=lr,\n        per_device_train_batch_size=2,         # increase from 1 -> 4 (if OOM, drop to 2)\n        per_device_eval_batch_size=1,          # must be <= train batch when generating\n        gradient_accumulation_steps=8,         # effective batch = 4*2 = 8\n    \n        # training length/precision\n        num_train_epochs=epochs,\n        predict_with_generate=False,           # IMPORTANT: avoid generation during training\n        fp16=(DEVICE==\"cuda\"),\n    \n        # generation control (used only if predict_with_generate=True or manual generate)\n        generation_max_length=128,\n    \n        # speedups\n        dataloader_num_workers=4,              # faster data loading\n        remove_unused_columns=False,\n        report_to=\"none\",\n        seed=SEED,\n    )\n    # -----------------------------------------------------------------\n\n\n    trainer = Seq2SeqTrainer(\n        model=model,\n        args=args,\n        train_dataset=train_ds,\n        eval_dataset=val_ds,\n        data_collator=data_collator,\n        tokenizer=tokenizer,\n    )\n\n    print(f\"\\n=== Training {phase_name} ===\")\n    trainer.train()\n    trainer.save_model(output_dir + \"/final\")","metadata":{"_uuid":"b9e0ea51-63c2-4006-ba2d-9d969fa4797e","_cell_guid":"076ba751-8ed0-4f88-ae54-44238a141ba0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:43:15.929076Z","iopub.execute_input":"2025-12-22T20:43:15.929381Z","iopub.status.idle":"2025-12-22T20:43:18.391257Z","shell.execute_reply.started":"2025-12-22T20:43:15.929354Z","shell.execute_reply":"2025-12-22T20:43:18.390654Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc, torch\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"_uuid":"af15b7a8-a6a6-4fc2-a8ca-07dad214ff86","_cell_guid":"fd160aaa-b373-44ef-8f69-da6dfb1fa593","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:43:18.603948Z","iopub.execute_input":"2025-12-22T20:43:18.604523Z","iopub.status.idle":"2025-12-22T20:43:18.996786Z","shell.execute_reply.started":"2025-12-22T20:43:18.604496Z","shell.execute_reply":"2025-12-22T20:43:18.996042Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\ngc.collect()\nrun_phase(\n    model=model,\n    tokenizer=tok,\n    train_df=paired_train_df,\n    val_df=paired_val_df,\n    phase_name=\"Phase A (paired only)\",\n    epochs=4,\n    lr=3e-4,\n    batch_size=1 if DEVICE==\"cuda\" else 2,\n    output_dir=\"phaseA_paired\"\n)","metadata":{"_uuid":"d947a095-528b-43ec-bcdc-4a1aea873ac8","_cell_guid":"d1afe300-2297-44dd-bf85-387e0f9dcbdf","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-22T20:43:26.10231Z","iopub.execute_input":"2025-12-22T20:43:26.102615Z","iopub.status.idle":"2025-12-22T20:52:52.826908Z","shell.execute_reply.started":"2025-12-22T20:43:26.102587Z","shell.execute_reply":"2025-12-22T20:52:52.825939Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_one_sentence(en: str) -> str:\n    en = clean_ws(en)\n    if not en:\n        return \"...\"\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [p.strip() for p in parts if p.strip()]\n    # remove ending punct to avoid multiple sentences, then join\n    parts = [re.sub(r\"[.!?]+\\s*$\", \"\", p).strip() for p in parts]\n    parts = [p for p in parts if p]\n    out = \"; \".join(parts).strip()\n    if not out.endswith(\".\"):\n        out += \".\"\n    return out\n\nmodel.eval()\ntest_df  = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/test.csv\")\ndef generate_translation(src_text: str):\n    enc = tok(\n        clean_ws(src_text),\n        return_tensors=\"pt\",\n        max_length=192,\n        truncation=True,\n    ).to(DEVICE)\n\n    with torch.no_grad():\n        out_ids = model.generate(\n            **enc,\n            num_beams=4,\n            max_new_tokens=128,\n            length_penalty=1.0,\n            early_stopping=True,\n        )\n    pred = tok.decode(out_ids[0], skip_special_tokens=True)\n    return make_one_sentence(pred)\n\npreds = [generate_translation(t) for t in test_df[\"transliteration\"].tolist()]\n\nsub = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": preds})\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"Wrote submission.csv\")\nsub.head()","metadata":{"_uuid":"3255360f-65a5-4d2d-aed6-88a56f78347d","_cell_guid":"efba31e5-15a9-4926-9f12-064cfb93d403","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-12-22T20:54:24.115395Z","iopub.execute_input":"2025-12-22T20:54:24.115707Z","iopub.status.idle":"2025-12-22T20:54:31.684901Z","shell.execute_reply.started":"2025-12-22T20:54:24.115673Z","shell.execute_reply":"2025-12-22T20:54:31.684156Z"}},"outputs":[],"execution_count":null}]}