{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":14429337,"datasetId":9216317,"databundleVersionId":15247107}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#  Project: Deep Past — Sumerian/Akkadian Translation\n\n### Author: Teresa Fitzgerald  \n**Date:** March 2026  \n\n\n---\n\n##  Notebook Strategy & \"Gold Logic\"\n\nThis notebook follows a strict pipeline to handle the high-complexity Unicode and long-sequence transliterations found in the Deep Past competition.\n\n---\n\n## 1️⃣ Model Architecture\n\n- **Core Model:** ByT5-small  \n  - Byte-based tokenizer (handles 34+ non-ASCII Unicode characters)\n- **Optimization:** Gradient Accumulation (steps = 8)  \n  - Enables long-sequence training without OOM errors\n\n---\n\n## 2️⃣ Master Config (Locked)\n\n| Parameter              | Value | Reason |\n|------------------------|-------|--------|\n| Max Source Length      | 1024  | Covers 100% of training transliterations |\n| Max Target Length      | 4096  | Accommodates the 3,995-character translation outlier |\n| Epochs                 | 15    | Required for stable character-map learning |\n| Learning Rate          | 5e-5  | Fine-tuning stability |\n\n---\n\n##  Critical Guards\n\n     **Step 12.2 Guard:** Assertion ensures normalization does not wipe out rows  \n-  **Step 15.1 Guard:** Regex-based punctuation-only filter  \n-  **Empty Mask Guard:**  \n  `sub_df.loc[empty_mask, \"translation\"] = \"Unclear.\"`  \n  Protects against NaNs or whitespace-only results  \n\n\n","metadata":{}},{"cell_type":"markdown","source":"## Step 2 — Inspect + Export Snapshots (downloadable)\n\nshow null/empty counts\nshow basic length stats for text columns\nexport small CSV snapshots to /kaggle/working/ so you can download and inspect","metadata":{}},{"cell_type":"code","source":"# Step  \nimport sys, platform, torch, transformers, safetensors, os\n\nprint(\"Python:\", sys.version)\nprint(\"Platform:\", platform.platform())\nprint(\"torch:\", torch.__version__, \"| cuda:\", torch.version.cuda, \"| cuda available:\", torch.cuda.is_available())\nprint(\"transformers:\", transformers.__version__)\nprint(\"safetensors:\", safetensors.__version__)\nprint(\"Disk /kaggle/working:\", os.popen(\"df -h /kaggle/working | tail -n 1\").read().strip())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:01.501706Z","iopub.execute_input":"2026-02-18T02:25:01.502439Z","iopub.status.idle":"2026-02-18T02:25:03.686324Z","shell.execute_reply.started":"2026-02-18T02:25:01.502411Z","shell.execute_reply":"2026-02-18T02:25:03.685602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Step 1 — Load Data (canonical)\n# =========================\nimport os\nimport pandas as pd\n\nDATA_DIR = \"//kaggle/input/competitions/deep-past-initiative-machine-translation\"  # change if needed\n\ntrain_path  = os.path.join(DATA_DIR, \"train.csv\")\ntest_path   = os.path.join(DATA_DIR, \"test.csv\")\nsample_path = os.path.join(DATA_DIR, \"sample_submission.csv\")\n\n# Load\ntrain_df = pd.read_csv(train_path)\ntest_df = pd.read_csv(test_path)\nsample_sub = pd.read_csv(sample_path)\n\n# Normalize train id column to match test/sample\n# train has 'oare_id' but the rest of the pipeline expects 'id'\nif \"id\" not in train_df.columns and \"oare_id\" in train_df.columns:\n    train_df = train_df.rename(columns={\"oare_id\": \"id\"})\n\nprint(\"train :\", train_df.shape)\nprint(\"test  :\", test_df.shape)\nprint(\"sample:\", sample_sub.shape)\n\nprint(\"\\nTrain columns:\", list(train_df.columns))\nprint(\"Test columns :\", list(test_df.columns))\nprint(\"Sample columns:\", list(sample_sub.columns))\n\ndisplay(train_df.head(3))\ndisplay(test_df.head(3))\ndisplay(sample_sub.head(3))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:03.692294Z","iopub.execute_input":"2026-02-18T02:25:03.692551Z","iopub.status.idle":"2026-02-18T02:25:04.198654Z","shell.execute_reply.started":"2026-02-18T02:25:03.692505Z","shell.execute_reply":"2026-02-18T02:25:04.198071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Step 2 — Inspect + Export Snapshots\n# =========================\nimport os\nimport pandas as pd\n\ndef basic_profile(df, name):\n    print(f\"\\n===== {name} =====\")\n    print(\"shape:\", df.shape)\n    print(\"columns:\", list(df.columns))\n\n    # Nulls\n    null_counts = df.isna().sum()\n\n    # Empty strings only (ignore NaNs)\n    empty_counts = (\n        df.apply(lambda col: col.astype(\"string\").fillna(\"\").str.len().eq(0))\n          .sum()\n    )\n\n    prof = pd.DataFrame({\n        \"nulls\": null_counts,\n        \"empty_strings\": empty_counts\n    }).sort_values([\"nulls\", \"empty_strings\"], ascending=False)\n\n    display(prof.head(50))\n\nbasic_profile(train_df, \"train_df\")\nbasic_profile(test_df, \"test_df\")\nbasic_profile(sample_sub, \"sample_sub\")\n\n# --- Identify likely text columns (object or string dtype), excluding IDs ---\ndef get_text_cols(df):\n    text_cols = []\n    for c in df.columns:\n        if c.lower() == \"id\":\n            continue\n        if pd.api.types.is_object_dtype(df[c]) or pd.api.types.is_string_dtype(df[c]):\n            text_cols.append(c)\n    return text_cols\n\ntrain_text_cols = get_text_cols(train_df)\ntest_text_cols  = get_text_cols(test_df)\n\nprint(\"\\nTrain text columns:\", train_text_cols)\nprint(\"Test text columns :\", test_text_cols)\n\n# --- Length stats for each text col ---\ndef length_stats(df, cols, name):\n    print(f\"\\n--- Length stats: {name} ---\")\n    for c in cols:\n        lens = df[c].astype(\"string\").fillna(\"\").str.len()\n        print(\n            f\"{c:25s}  min={int(lens.min()):>4}  \"\n            f\"p50={int(lens.median()):>4}  \"\n            f\"p95={int(lens.quantile(0.95)):>4}  \"\n            f\"max={int(lens.max()):>6}\"\n        )\n\nlength_stats(train_df, train_text_cols, \"train_df\")\nlength_stats(test_df,  test_text_cols,  \"test_df\")\n\n# --- Export small snapshots for eyeballing ---\nOUT_DIR = \"/kaggle/working\"\ntrain_snap_path = os.path.join(OUT_DIR, \"train_snapshot_200.csv\")\ntest_snap_path  = os.path.join(OUT_DIR, \"test_snapshot_200.csv\")\nsub_snap_path   = os.path.join(OUT_DIR, \"sample_submission_snapshot_50.csv\")\n\ntrain_df.head(200).to_csv(train_snap_path, index=False)\ntest_df.head(200).to_csv(test_snap_path, index=False)\nsample_sub.head(50).to_csv(sub_snap_path, index=False)\n\nprint(\"\\n✅ Wrote snapshots:\")\nprint(\" -\", train_snap_path)\nprint(\" -\", test_snap_path)\nprint(\" -\", sub_snap_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.199649Z","iopub.execute_input":"2026-02-18T02:25:04.200272Z","iopub.status.idle":"2026-02-18T02:25:04.268577Z","shell.execute_reply.started":"2026-02-18T02:25:04.200246Z","shell.execute_reply":"2026-02-18T02:25:04.267904Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Input → Output mapping is confirmed\ntrain.transliteration  →  train.translation\ntest.transliteration   →  submission.translation\nDo NOT\nlowercase\nremove diacritics\nstrip punctuation\ntokenize aggressively\nAllowed (later, carefully)\nnormalize whitespace only\nnormalize Unicode spacing\npreserve newlines & punctuation\nDataset size reality\n1561 rows = too small for seq2seq from scratch\nRetrieval / hybrid methods are realistic\nBLEU/CHRF reward surface similarity\n","metadata":{}},{"cell_type":"markdown","source":"Step 3 — Length & Structure Analysis (ONLY)\nPurpose:\nVerify whether test transliterations fall inside the train distribution.\nThis tells us whether retrieval-style baselines are even viable.","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 3 — Length & Structure Analysis\n# =========================\nimport pandas as pd\n\ndef length_summary(df, col, name):\n    lengths = df[col].astype(\"string\").fillna(\"\").str.len()\n    summary = {\n        \"count\": len(lengths),\n        \"min\": int(lengths.min()),\n        \"p10\": int(lengths.quantile(0.10)),\n        \"p25\": int(lengths.quantile(0.25)),\n        \"p50\": int(lengths.quantile(0.50)),\n        \"p75\": int(lengths.quantile(0.75)),\n        \"p90\": int(lengths.quantile(0.90)),\n        \"max\": int(lengths.max()),\n        \"mean\": round(lengths.mean(), 2),\n    }\n    print(f\"\\n{name}\")\n    for k, v in summary.items():\n        print(f\"{k:>6}: {v}\")\n    return lengths\n\n# Train transliteration lengths\ntrain_lens = length_summary(\n    train_df,\n    \"transliteration\",\n    \"TRAIN transliteration length stats\"\n)\n\n# Train translation lengths\n_ = length_summary(\n    train_df,\n    \"translation\",\n    \"TRAIN translation length stats\"\n)\n\n# Test transliteration lengths\ntest_lens = length_summary(\n    test_df,\n    \"transliteration\",\n    \"TEST transliteration length stats\"\n)\n\n# Compare test vs train coverage\nprint(\"\\nTEST lengths vs TRAIN range:\")\nfor i, l in enumerate(test_lens.tolist()):\n    status = \"INSIDE\" if (l >= train_lens.min() and l <= train_lens.max()) else \"OUTSIDE\"\n    print(f\"test row {i}: length={l} → {status}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.27037Z","iopub.execute_input":"2026-02-18T02:25:04.270633Z","iopub.status.idle":"2026-02-18T02:25:04.291108Z","shell.execute_reply.started":"2026-02-18T02:25:04.270611Z","shell.execute_reply":"2026-02-18T02:25:04.290449Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"observation\nmin  = 21\np50  = 365\nmean = 426\np90  = 880\nmax  = 932\nInterpretation\nTraining inputs range from very short (21 chars) to very long (~930 chars).\nThe median (365) and mean (426) are close → distribution is fairly balanced.\nThis suggests:\nSome rows are single lines / phrases\nOthers are full multi-line scholarly transliterations\n✅ Important:\nThe model (or retrieval system) has s\n","metadata":{}},{"cell_type":"markdown","source":"min  = 6\np50  = 383\nmean = 499\np90  = 1040\nmax  = 3895   ← very large\nInterpretation\nTranslations are often longer than transliterations\nThis is expected for:\nscholarly translations\nexplanatory parentheses\nexpanded English phrasing\nThe huge max (3895) indicates:\nsome rows translate entire passages or paragraphs\noutput length can vary wildly even for similar inputs\n🔴 Critical implication\nAny fixed-length generation (e.g., truncation) would be dangerous\nOutput length cannot be inferred reliably from input length\n","metadata":{}},{"cell_type":"markdown","source":"min  = 129\np50  = 139\nmax  = 267\nmean = 168\nInterpretation\nALL test inputs are:\nshort to medium\ntightly clustered\nmuch shorter than the train median\nCompare:\nTrain median ≈ 365\nTest mean ≈ 169\n➡️ Test inputs are about half the size of a typical train input.\n","metadata":{}},{"cell_type":"markdown","source":"test row 0 → INSIDE\ntest row 1 → INSIDE\ntest row 2 → INSIDE\ntest row 3 → INSIDE\nEvery test example is fully inside the training length range\nNo extrapolation is required\nNo chunking or padding is needed\n✅ This is ideal for retrieval-based baseline","metadata":{}},{"cell_type":"markdown","source":"Step 4 — Whitespace & Unicode Structure Audit\nPurpose:\nDetect hidden characters that can silently hurt BLEU / CHRF:\nnon-breaking spaces\nthin spaces\ntabs\nmultiple spaces\nnewlines\nunusual Unicode symbols","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 4 — Whitespace & Unicode Audit\n# =========================\nimport collections\nimport unicodedata\n\ndef whitespace_audit(series, name):\n    s = series.dropna().astype(str)\n    print(f\"\\n=== Whitespace audit: {name} ===\")\n    print(\"Leading space % :\", round((s.str.startswith(\" \")).mean() * 100, 2))\n    print(\"Trailing space %:\", round((s.str.endswith(\" \")).mean() * 100, 2))\n    print(\"Double spaces % :\", round((s.str.contains(\"  \")).mean() * 100, 2))\n    print(\"Tabs %          :\", round((s.str.contains(r\"\\t\")).mean() * 100, 2))\n    print(\"Newlines %      :\", round((s.str.contains(r\"\\n\")).mean() * 100, 2))\n\ndef unicode_char_audit(series, name, topn=20):\n    sample = \"\\n\".join(series.dropna().astype(str).head(2000).tolist())\n    counter = collections.Counter(sample)\n\n    suspicious = []\n    for ch, ct in counter.items():\n        if ord(ch) > 127 or ch in [\"\\t\", \"\\u00a0\", \"\\u2009\", \"\\u200a\", \"\\u202f\"]:\n            suspicious.append((ch, ct))\n\n    suspicious = sorted(suspicious, key=lambda x: x[1], reverse=True)\n\n    print(f\"\\n=== Unicode audit: {name} ===\")\n    print(f\"Found {len(suspicious)} non-ASCII / special chars (top shown):\")\n    for ch, ct in suspicious[:topn]:\n        code = f\"U+{ord(ch):04X}\"\n        name_uc = unicodedata.name(ch, \"UNKNOWN\")\n        print(f\"{code:8s} {repr(ch):8s} count={ct:<6} {name_uc}\")\n\n# --- Run audits ---\n\nwhitespace_audit(train_df[\"transliteration\"], \"train.transliteration\")\nwhitespace_audit(train_df[\"translation\"],     \"train.translation\")\nwhitespace_audit(test_df[\"transliteration\"],  \"test.transliteration\")\n\nunicode_char_audit(train_df[\"transliteration\"], \"train.transliteration\")\nunicode_char_audit(train_df[\"translation\"],     \"train.translation\")\nunicode_char_audit(test_df[\"transliteration\"],  \"test.transliteration\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.292103Z","iopub.execute_input":"2026-02-18T02:25:04.292589Z","iopub.status.idle":"2026-02-18T02:25:04.419098Z","shell.execute_reply.started":"2026-02-18T02:25:04.292565Z","shell.execute_reply":"2026-02-18T02:25:04.418246Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 4 — Interpretation & Observations\n1️⃣ Whitespace audit — what it tells us\n🔹 train.transliteration\nTrailing space %: 2.31\nEverything else: 0.0\nInterpretation\nTransliteration is very clean\nNo tabs, no double spaces, no newlines\nOnly issue: ~2.3% have trailing spaces\n✅ Trailing spaces are unintentional noise\n❌ They carry no linguistic meaning","metadata":{}},{"cell_type":"markdown","source":"train.translation\nTrailing space %: 10.57\nDouble spaces % : 8.97\nEverything else: 0.0\nInterpretation\nTranslations are clean structurally\nBut:\n~10% end with a trailing space\n~9% contain double spaces\n⚠️ This is very important:\nBLEU and CHRF treat \"word word\" ≠ \"word word\"\nThese are almost certainly copy-editing artifacts","metadata":{}},{"cell_type":"markdown","source":"test.transliteration\nAll whitespace metrics: 0.0\nInterpretation\nTest data is pristine\nThis is good, but also dangerous:\nIf we don’t normalize train data similarly, mismatch occurs","metadata":{}},{"cell_type":"markdown","source":"Unicode audit — transliteration (train)\nš, ṣ, ṭ, ḫ\naccented vowels\nsubscripts ₄, ₅\nellipsis …\nInterpretation\nThese are core transliteration symbols\nThey appear frequently and consistently\nThey encode phonetic and grammatical meaning","metadata":{}},{"cell_type":"markdown","source":"Unicode audit — translation (train)\nNotable characters:\nmacrons: ā ī ē ū\ndiacritics: ṣ ṭ\nscholarly markers: ʾ\nfractions: ½ ⅓ ⅚\npunctuation variants: — ’\nInterpretation\nThis is scholarly English\nThese characters are intentional\nRemoving them would damage correctness","metadata":{}},{"cell_type":"markdown","source":"Unicode audit — test transliteration\nNotable new symbol:\nU+201E  „  DOUBLE LOW-9 QUOTATION MARK\nInterpretation\nThis is a quotation style used in European typography\nIt appears only in test, but:\nIt is valid Unicode punctuation\nIt visually differs from standard \"\n","metadata":{}},{"cell_type":"markdown","source":"Step 5 — Apply minimal, safe normalization Strip trailing and leading spaces\nConvert tabs → space (just in case)\nCollapse 2+ spaces → 1 space\nLeave all Unicode characters and punctuation untouched (diacritics, quotes, ellipses stay)","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 5 — Minimal Whitespace Normalization\n# =========================\nimport re\nimport pandas as pd\n\ndef normalize_whitespace(s: str) -> str:\n    if pd.isna(s):\n        return \"\"\n    s = str(s)\n\n    # Convert tabs to spaces (safe)\n    s = s.replace(\"\\t\", \" \")\n\n    # Collapse multiple spaces into single space (safe)\n    s = re.sub(r\" {2,}\", \" \", s)\n\n    # Strip leading/trailing spaces (safe)\n    s = s.strip()\n\n    return s\n\n# --- Apply normalization ---\ntrain_df[\"transliteration_norm\"] = train_df[\"transliteration\"].map(normalize_whitespace)\ntrain_df[\"translation_norm\"]     = train_df[\"translation\"].map(normalize_whitespace)\n\ntest_df[\"transliteration_norm\"]  = test_df[\"transliteration\"].map(normalize_whitespace)\n\nprint(\"✅ Step 5: normalization columns created\")\n\n# --- Measure how many rows changed ---\ntrain_trans_changed = (train_df[\"transliteration\"] != train_df[\"transliteration_norm\"]).sum()\ntrain_tran_changed  = (train_df[\"translation\"]     != train_df[\"translation_norm\"]).sum()\ntest_trans_changed  = (test_df[\"transliteration\"]  != test_df[\"transliteration_norm\"]).sum()\n\nprint(\"\\nRows changed:\")\nprint(\" - train transliteration:\", train_trans_changed, \"/\", len(train_df))\nprint(\" - train translation    :\", train_tran_changed,  \"/\", len(train_df))\nprint(\" - test  transliteration:\", test_trans_changed,  \"/\", len(test_df))\n\n# --- Show a few examples of changes (if any) ---\ndef show_changes(df, orig_col, norm_col, n=10):\n    diff = df[df[orig_col] != df[norm_col]][[orig_col, norm_col]].head(n)\n    if len(diff) == 0:\n        print(f\"\\nNo differences found for {orig_col} → {norm_col}\")\n    else:\n        print(f\"\\nExamples of differences for {orig_col} → {norm_col}:\")\n        display(diff)\n\nshow_changes(train_df, \"transliteration\", \"transliteration_norm\", n=5)\nshow_changes(train_df, \"translation\",     \"translation_norm\",     n=5)\nshow_changes(test_df,  \"transliteration\", \"transliteration_norm\", n=5)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.420185Z","iopub.execute_input":"2026-02-18T02:25:04.420907Z","iopub.status.idle":"2026-02-18T02:25:04.476755Z","shell.execute_reply.started":"2026-02-18T02:25:04.42088Z","shell.execute_reply":"2026-02-18T02:25:04.476102Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Interpretation\nTrain transliteration\nVery few rows changed\nThis matches Step 4 findings: transliteration was already clean\nTrain translation\n~19% changed — expected and good\nThis aligns perfectly with:\n~10.6% trailing spaces\n~9% double spaces\nTest transliteration\n0 changes → pristine\nConfirms our normalization did not overreach\n✅ This is a healthy signal — normalization corrected noise, not content.","metadata":{}},{"cell_type":"markdown","source":"## Step 6 — Valid Baseline Submission (Format Check)\n","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 6 — Create a VALID baseline submission (format-only)\n# =========================\nimport pandas as pd\n\n# Reference dtype from sample submission\nsample_id_dtype = sample_sub[\"id\"].dtype\n\n# Build submission df with required columns (format-only placeholders)\nsubmission_df = pd.DataFrame({\n    \"id\": test_df[\"id\"].astype(sample_id_dtype),\n    \"translation\": [\"...\"] * len(test_df)\n})\n\n# Validate columns/order match sample submission exactly\nexpected_cols = list(sample_sub.columns)\nactual_cols = list(submission_df.columns)\n\nprint(\"Expected columns:\", expected_cols)\nprint(\"Actual columns  :\", actual_cols)\n\nassert actual_cols == expected_cols, \"❌ Column order mismatch vs sample_sub\"\n\n# Validate row count matches\nassert len(submission_df) == len(sample_sub), \"❌ Row count mismatch vs sample_sub\"\n\n# Validate id coverage (compare using sample dtype)\nmissing_ids = set(sample_sub[\"id\"].astype(sample_id_dtype)) - set(submission_df[\"id\"])\nextra_ids   = set(submission_df[\"id\"]) - set(sample_sub[\"id\"].astype(sample_id_dtype))\n\nprint(\"Missing ids:\", missing_ids)\nprint(\"Extra ids  :\", extra_ids)\n\nassert len(missing_ids) == 0 and len(extra_ids) == 0, \"❌ id mismatch vs sample_sub\"\n\n# Save\nout_path = \"/kaggle/working/submission.csv\"\nsubmission_df.to_csv(out_path, index=False)\n\nprint(\"✅ Step 6 complete — wrote:\", out_path)\ndisplay(submission_df)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.4776Z","iopub.execute_input":"2026-02-18T02:25:04.477909Z","iopub.status.idle":"2026-02-18T02:25:04.490404Z","shell.execute_reply.started":"2026-02-18T02:25:04.477887Z","shell.execute_reply":"2026-02-18T02:25:04.489728Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 7 — First Real Baseline (Retrieval / Nearest Neighbor)\nGoal of Step 7 (and ONLY Step 7):\nUse training data to generate real translations\nReplace the \"...\" placeholder\nKeep everything else (EDA, normalization, submission contract) untouched","metadata":{}},{"cell_type":"code","source":"# ==================================\n# Step 7 — Character Inventory (RAW)\n# ==================================\nfrom collections import Counter\n\n# Updated for robustness (using explicit names instead of indices)\nSRC_COL = \"transliteration\"\nTGT_COL = \"translation\"\n\n# Guard check to ensure columns exist before inventory\nassert SRC_COL in train_df.columns, f\"Column {SRC_COL} not found!\"\nassert TGT_COL in train_df.columns, f\"Column {TGT_COL} not found!\"\ndef char_inventory(series):\n    c = Counter()\n    for s in series.dropna().astype(str):\n        c.update(s)\n    return c\n\nsrc_chars = char_inventory(train_df[SRC_COL])\ntgt_chars = char_inventory(train_df[TGT_COL])\n\nprint(\"Top 60 SOURCE characters:\")\nfor ch, cnt in src_chars.most_common(60):\n    print(f\"{repr(ch):>6} : {cnt}\")\n\nprint(\"\\nTop 60 TARGET characters:\")\nfor ch, cnt in tgt_chars.most_common(60):\n    print(f\"{repr(ch):>6} : {cnt}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.500746Z","iopub.execute_input":"2026-02-18T02:25:04.501Z","iopub.status.idle":"2026-02-18T02:25:04.614867Z","shell.execute_reply.started":"2026-02-18T02:25:04.500978Z","shell.execute_reply":"2026-02-18T02:25:04.614008Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Observation step 7.1\nHyphens (-) are the most frequent character\nConfirms syllabic transliteration structure (e.g., a-na, i-na).\nIndicates that word boundaries are often encoded through hyphenation, not spaces.\nUppercase letters are common and meaningful\nReflect Sumerian logograms written in ALL CAPS.\nCapitalization in source text carries semantic information and is not stylistic noise.\nRich use of diacritics and special consonants\nCharacters such as š, ṣ, ṭ, ḫ and accented vowels (á, à, é, í, ú, ù) appear frequently.\nThese characters encode phonological distinctions and must be treated as linguistically meaningful.\nSubscripts and numeric markers are present\nUnicode subscripts (e.g., ₄, ₅) and digits (1–6) occur regularly.\nThese indicate distinct signs or readings and are not interchangeable.\nModern editorial symbols exist but are secondary\nPeriods, parentheses, and ellipses (…) appear but are less frequent than linguistic symbols.\nThese likely reflect modern scholarly notation rather than original text content.\n2. Target text (English translations)\nStandard English alphabet and punctuation dominate\nConfirms that translations are mostly conventional English prose.\nAkkadian characters persist in English\nCharacters like š, Š, ā, ī, ē, ū appear in translations.\nIndicates that translators preserve Akkadian spellings for names and technical terms.\nPunctuation and numbers are common\nCommas, periods, colons, semicolons, hyphens, parentheses, and digits appear frequently.\nSuggests structured explanatory or referential translation style.\nApostrophes are present but appear consistent\nOnly one apostrophe form is prominent at this stage.\nNo evidence yet of mixed Unicode apostrophe variants.\n3. Cross-cutting observations\nThe dataset is Unicode-heavy and linguistically precise, not noisy free text.\nCapitalization, diacritics, hyphenation, and subscripts are semantically meaningful.\nEditorial notation exists but does not dominate the signal.\nCharacter-level fidelity will matter strongly for evaluation (especially chrF++).","metadata":{}},{"cell_type":"markdown","source":"## Step 7.2 — Whitespace & Invisible Character Audit\nPurpose (observation only)\nIdentify:\ndouble / inconsistent spaces\ntabs or newlines\nleading / trailing spaces\nnon-breaking spaces (NBSP)","metadata":{}},{"cell_type":"code","source":"# ==================================\n# Step 7.2a — Whitespace Diagnostics (FIXED: no positional indexing)\n# ==================================\ndef whitespace_report(series):\n    series = series.astype(str)\n    return {\n        \"double_space_%\": (series.str.contains(\"  \")).mean(),\n        \"tab_%\": (series.str.contains(\"\\t\")).mean(),\n        \"newline_%\": (series.str.contains(\"\\n\")).mean(),\n        \"leading_space_%\": (series.str.startswith(\" \")).mean(),\n        \"trailing_space_%\": (series.str.endswith(\" \")).mean(),\n        \"nbsp_%\": (series.str.contains(\"\\u00A0\")).mean(),\n    }\n\n# FIX: Use explicit column names (stable even after adding new columns)\nSRC_COL = \"transliteration\"\nTGT_COL = \"translation\"\n\nprint(\"SOURCE whitespace report:\")\nprint(whitespace_report(train_df[SRC_COL]))\n\nprint(\"\\nTARGET whitespace report:\")\nprint(whitespace_report(train_df[TGT_COL]))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.61611Z","iopub.execute_input":"2026-02-18T02:25:04.616485Z","iopub.status.idle":"2026-02-18T02:25:04.642455Z","shell.execute_reply.started":"2026-02-18T02:25:04.616447Z","shell.execute_reply":"2026-02-18T02:25:04.641491Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Observation Step 7.2\nStep 7.2 \\\nWhitespace noise exists primarily in the target (English) text, not the source.\nThe issue is moderate but consistent enough to be relevant for evaluation metrics.","metadata":{}},{"cell_type":"markdown","source":"## Step 8.1 — Cleaning Rules v1 (Decision + Implementation)","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 8.1 — Cleaning v1 (FIXED blockers + tokenizer-ready useful columns)\n# =========================\nimport re\nimport pandas as pd\n\n# FIX: explicit, stable column names (no positional indexing)\nSRC_COL = \"transliteration\"\nTGT_COL = \"translation\"\nTEST_SRC_COL = \"transliteration\"\n\n_ws_re = re.compile(r\"\\s+\")\n\ndef clean_src_v1(s: str) -> str:\n    if pd.isna(s):\n        return \"\"\n    s = str(s)\n    # hygiene only (source is already clean)\n    s = s.strip()                  # removes trailing/leading spaces\n    s = _ws_re.sub(\" \", s)         # collapse any whitespace runs\n    return s\n\ndef clean_tgt_v1(s: str) -> str:\n    if pd.isna(s):\n        return \"\"\n    s = str(s)\n    # confirmed noise fixes\n    s = s.strip()\n    s = _ws_re.sub(\" \", s)\n    return s\n\n# Copy to avoid accidental chained modification issues\ntrain_df = train_df.copy()\ntest_df = test_df.copy()\n\n# Apply cleaning\ntrain_df[\"src_clean_v1\"] = train_df[SRC_COL].map(clean_src_v1)\ntrain_df[\"tgt_clean_v1\"] = train_df[TGT_COL].map(clean_tgt_v1)\ntest_df[\"src_clean_v1\"]  = test_df[TEST_SRC_COL].map(clean_src_v1)\n\n# Quick verification on whitespace issues we observed\ndef whitespace_report(series):\n    series = series.astype(str)\n    return {\n        \"double_space_%\": (series.str.contains(\"  \")).mean(),\n        \"leading_space_%\": (series.str.startswith(\" \")).mean(),\n        \"trailing_space_%\": (series.str.endswith(\" \")).mean(),\n    }\n\nprint(\"SOURCE raw whitespace:\", whitespace_report(train_df[SRC_COL]))\nprint(\"SOURCE clean_v1 whitespace:\", whitespace_report(train_df[\"src_clean_v1\"]))\n\nprint(\"\\nTARGET raw whitespace:\", whitespace_report(train_df[TGT_COL]))\nprint(\"TARGET clean_v1 whitespace:\", whitespace_report(train_df[\"tgt_clean_v1\"]))\n\ndisplay(train_df[[SRC_COL, \"src_clean_v1\", TGT_COL, \"tgt_clean_v1\"]].head(5))\n\n# ============================================================\n# NEW: Tokenizer-ready views (useful columns only, canonical)\n# ============================================================\ntrain_tok_df = train_df[\n    [\"id\", \"src_clean_v1\", \"tgt_clean_v1\"]\n].rename(columns={\n    \"src_clean_v1\": \"source_text\",\n    \"tgt_clean_v1\": \"target_text\"\n})\n\ntest_tok_df = test_df[\n    [\"id\", \"src_clean_v1\"]\n].rename(columns={\n    \"src_clean_v1\": \"source_text\"\n})\n\nprint(\"\\n✅ Tokenizer-ready views created\")\nprint(\"TRAIN tokenizer columns:\", list(train_tok_df.columns))\nprint(\"TEST tokenizer columns :\", list(test_tok_df.columns))\n\ndisplay(train_tok_df.head(3))\ndisplay(test_tok_df.head(3))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.643594Z","iopub.execute_input":"2026-02-18T02:25:04.643939Z","iopub.status.idle":"2026-02-18T02:25:04.801302Z","shell.execute_reply.started":"2026-02-18T02:25:04.643903Z","shell.execute_reply":"2026-02-18T02:25:04.800768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 9 — Create Train/Validation split (baseline)\nGoal: set up a reproducible split so we can score locally before generating submissions.","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 9.2 — Train/Val Split (after model reload)\n# =========================\nfrom sklearn.model_selection import train_test_split\n\n# Use cleaned v1 columns\nSRC = \"src_clean_v1\"\nTGT = \"tgt_clean_v1\"\n\n# Keep only rows with non-empty src/tgt\nmt_df = train_df[[SRC, TGT]].copy()\nmt_df = mt_df[\n    (mt_df[SRC].str.len() > 0) &\n    (mt_df[TGT].str.len() > 0)\n].reset_index(drop=True)\n\ntrain_texts, val_texts = train_test_split(\n    mt_df,\n    test_size=0.10,\n    random_state=42\n)\n\nprint(\"MT df:\", mt_df.shape)\nprint(\"Train split:\", train_texts.shape)\nprint(\"Val split  :\", val_texts.shape)\n\ndisplay(train_texts.head(3))\ndisplay(val_texts.head(3))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:04.809634Z","iopub.execute_input":"2026-02-18T02:25:04.809986Z","iopub.status.idle":"2026-02-18T02:25:05.478935Z","shell.execute_reply.started":"2026-02-18T02:25:04.809959Z","shell.execute_reply":"2026-02-18T02:25:05.478303Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Observation:\nStep 9 — Train / Validation Split: Observations\n1. Dataset size and integrity\nTotal usable MT rows: 1,561\nAll rows have:\nnon-empty cleaned source (src_clean_v1)\nnon-empty cleaned target (tgt_clean_v1)\nNo rows were dropped unexpectedly during filtering.\nObservation:\nThe dataset is small but fully usable, which confirms this is a low-resource MT task and reinforces the need for careful validation and regularization later.\n2. Split proportions\nTraining set: 1,404 rows (~90%)\nValidation set: 157 rows (~10%)\nObservation:\nThe split is:\nstandard\nbalanced\nreproducible (fixed random seed)\n3. Content distribution (spot checks)\nFrom the sampled rows:\nSource texts vary in length and structure\nshort transactional records (weights, silver amounts)\nformal letters (“Say to X, thus Y”)\nseals and administrative formulas (“KIŠIB … DUMU …”)\nTargets are fluent, human-written English\nconsistent narrative style\nstandard punctuation\ndomain-specific vocabulary (silver, minas, shekels, seals, letters)","metadata":{}},{"cell_type":"markdown","source":"## tep 10 — Baseline Model Selection & Setup (v0)","metadata":{}},{"cell_type":"markdown","source":"Step 10.1 — Code: Load model/tokenizer + quick sanity checks\n","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 10.1 — Load ByT5 baseline (CANONICAL LOAD after NaN reset)\n# =========================\nimport os\nimport torch\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# --------------------------------------------------\n# OFFLINE GUARDS (Kaggle-safe)\n# --------------------------------------------------\nos.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\nos.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\n\n# --------------------------------------------------\n# CANONICAL MODEL PATH\n# --------------------------------------------------\nMODEL_PATH = \"/kaggle/input/byt5-kaggle-dataset/byt5_kaggle_dataset/byt5-small-hf\"\n\nprint(f\"Checking path: {MODEL_PATH}\")\nif not os.path.exists(MODEL_PATH):\n    raise FileNotFoundError(\n        \"❌ Path not found. Open Kaggle > Data sidebar > click your dataset, \"\n        \"and copy the exact folder path shown there.\"\n    )\n\n# --------------------------------------------------\n# CANONICAL LOAD (this is the ACTIVE model instance)\n# --------------------------------------------------\ntokenizer = AutoTokenizer.from_pretrained(\n    MODEL_PATH,\n    local_files_only=True\n)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    MODEL_PATH,\n    local_files_only=True\n)\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\nprint(\"✅ Model Loaded Successfully (canonical instance)\")\nprint(\"Device:\", device)\nprint(\"Tokenizer pad_token_id:\", tokenizer.pad_token_id)\nprint(\"Model config pad_token_id:\", model.config.pad_token_id)\nprint(\"Model params (M):\", f\"{sum(p.numel() for p in model.parameters()) / 1e6:.1f}\")\n\n# --------------------------------------------------\n# SANITY CHECK — SMOKE TEST ONLY\n# --------------------------------------------------\n# IMPORTANT:\n# - This does NOT define training truncation.\n# - Real truncation will be defined in the dataset/tokenizer step.\n# --------------------------------------------------\nSMOKE_MAX_LEN = 256\nprint(f\"\\n[Smoke test only] Tokenizing with max_length={SMOKE_MAX_LEN}\")\nprint(\"NOTE: This is NOT the training truncation policy.\")\n\nex_src = train_texts[\"src_clean_v1\"].iloc[0]\nex_tgt = train_texts[\"tgt_clean_v1\"].iloc[0]\n\nenc = tokenizer(\n    ex_src,\n    truncation=True,\n    max_length=SMOKE_MAX_LEN,\n    return_tensors=\"pt\"\n)\nenc = {k: v.to(device) for k, v in enc.items()}\ndec = tokenizer.decode(enc[\"input_ids\"][0], skip_special_tokens=True)\n\nprint(\"\\n--- Example source (raw) ---\")\nprint(ex_src[:300])\n\nprint(\"\\n--- Decoded (sanity check; minor differences are expected) ---\")\nprint(dec[:300])\n\nprint(\"\\n--- Example target (raw) ---\")\nprint(ex_tgt[:300])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:05.47982Z","iopub.execute_input":"2026-02-18T02:25:05.480218Z","iopub.status.idle":"2026-02-18T02:25:36.410851Z","shell.execute_reply.started":"2026-02-18T02:25:05.480192Z","shell.execute_reply":"2026-02-18T02:25:36.410089Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 11 — Training configuration (baseline)\nGoal: pick max lengths + batch size + grad accumulation that fit the 9-hour limit and avoid OOM, while capturing most sequences.\n","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 11.1 — Token length stats (FIXED: uses canonical tokenizer views)\n# =========================\nimport numpy as np\n\n# Guard: require canonical tokenizer views\nif \"train_tok_df\" not in globals():\n    raise NameError(\n        \"❌ train_tok_df not found. Run Step 8.1 (with tokenizer-ready views) first \"\n        \"so we tokenize only the canonical columns.\"\n    )\n\ndef token_lens_batch(texts, batch_size=64):\n    \"\"\"\n    Compute token lengths using batch tokenization (faster than per-row calls).\n    No truncation here — we want the true token length distribution.\n    \"\"\"\n    lens = []\n    texts = list(texts)\n    for i in range(0, len(texts), batch_size):\n        batch = texts[i:i + batch_size]\n        enc = tokenizer(batch, add_special_tokens=True, truncation=False)\n        lens.extend([len(ids) for ids in enc[\"input_ids\"]])\n    return np.array(lens, dtype=np.int32)\n\n# Random sample to keep it fast and representative\nsample_n = min(500, len(train_tok_df))\nsample_df = train_tok_df.sample(n=sample_n, random_state=42)\n\nsrc_lens = token_lens_batch(sample_df[\"source_text\"].tolist(), batch_size=64)\ntgt_lens = token_lens_batch(sample_df[\"target_text\"].tolist(), batch_size=64)\n\nprint(\"Token length stats (random sample):\")\nprint(\"SRC percentiles [50,80,90,95,99]:\", np.percentile(src_lens, [50, 80, 90, 95, 99]).astype(int))\nprint(\"TGT percentiles [50,80,90,95,99]:\", np.percentile(tgt_lens, [50, 80, 90, 95, 99]).astype(int))\nprint(\"Max SRC in sample:\", int(src_lens.max()))\nprint(\"Max TGT in sample:\", int(tgt_lens.max()))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:36.412003Z","iopub.execute_input":"2026-02-18T02:25:36.412632Z","iopub.status.idle":"2026-02-18T02:25:36.820593Z","shell.execute_reply.started":"2026-02-18T02:25:36.412595Z","shell.execute_reply":"2026-02-18T02:25:36.819792Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Step 11.2 — Lockable training configuration (ByT5 v0)\nWe’ll implement a safe baseline given your token stats:\nmax_source_length = 1024 (matches observed SRC cap)\nmax_target_length = 512 (keeps training feasible; we can try 768 later as an experiment)","metadata":{}},{"cell_type":"code","source":"# ============================================================\n# Step 11.2 — Master Training Config (🔒 CONFIG_LOCK Applied)\n# FT-1: only change = NUM_EPOCHS 9 -> 11\n# ============================================================\nimport torch\n\n# 1. PRE-CONFIG GUARD\nassert (train_df[\"src_clean_v1\"].str.len() == 0).sum() == 0, \\\n    \"Warning: Some transliterations became empty after normalization.\"\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Optional parity toggle\nPARITY_SCORE14 = False\n\n# 2. SEQUENCE LENGTHS\nMAX_SOURCE_LENGTH = 1024\nMAX_TARGET_LENGTH = 2048\n\nif PARITY_SCORE14:\n    MAX_TARGET_LENGTH = 768\n\n# 3. TRAINING HYPERPARAMETERS\nNUM_EPOCHS = 17         \nLEARNING_RATE = 5e-5\nWEIGHT_DECAY = 0.005\n\nif PARITY_SCORE14:\n    NUM_EPOCHS = 8\n    LEARNING_RATE = 3e-4\n\n# 4. MEMORY MANAGEMENT\nPER_DEVICE_TRAIN_BATCH_SIZE = 1\nPER_DEVICE_EVAL_BATCH_SIZE = 1\nGRADIENT_ACCUMULATION_STEPS = 8\n\n# Stability lock\nFP16 = False\n\n# 5. INFERENCE CONFIG\nGEN_MAX_LENGTH = 1024\nGEN_NUM_BEAMS = 4\n\n# Generation knobs used by inference\nGEN_MAX_NEW_TOKENS = 512\nGEN_MIN_NEW_TOKENS = 32\nGEN_EARLY_STOPPING = True\nGEN_LENGTH_PENALTY = 1.10\nNO_REPEAT_NGRAM_SIZE = 4\nREPETITION_PENALTY = 1.15\n\nprint(\"=== Step 11.2 Config (LOCKED & VERIFIED) ===\")\nprint(f\"Device: {DEVICE}\")\nprint(f\"Epochs: {NUM_EPOCHS} | LR: {LEARNING_RATE}\")\nprint(f\"Lengths: Source={MAX_SOURCE_LENGTH}, Target={MAX_TARGET_LENGTH}\")\nprint(f\"Inference: Beams={GEN_NUM_BEAMS}, Max_Gen={GEN_MAX_LENGTH}\")\nprint(\"Gen max_new_tokens:\", GEN_MAX_NEW_TOKENS, \"| min_new_tokens:\", GEN_MIN_NEW_TOKENS)\nprint(\"No-repeat ngram:\", NO_REPEAT_NGRAM_SIZE, \"| repetition_penalty:\", REPETITION_PENALTY)\nprint(\"Parity mode:\", PARITY_SCORE14)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:36.821665Z","iopub.execute_input":"2026-02-18T02:25:36.821972Z","iopub.status.idle":"2026-02-18T02:25:36.831304Z","shell.execute_reply.started":"2026-02-18T02:25:36.821938Z","shell.execute_reply":"2026-02-18T02:25:36.830537Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 12 (tokenization + dataset objects).","metadata":{}},{"cell_type":"code","source":"\n# =========================\n# Step 12.1 — Build Datasets (useful columns only + non-empty guard)\n# =========================\nfrom datasets import Dataset\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\n\n# Guard: require canonical tokenizer views\nif \"train_tok_df\" not in globals():\n    raise NameError(\n        \"❌ train_tok_df not found. Run Step 8.1 (with tokenizer-ready views) first.\"\n    )\n\n# 1) Safety cleaning: drop NaN + empty strings (true \"non-empty after cleaning\")\ndf = train_tok_df.copy()\n\ndf = df.dropna(subset=[\"source_text\", \"target_text\"])\ndf[\"source_text\"] = df[\"source_text\"].astype(str)\ndf[\"target_text\"] = df[\"target_text\"].astype(str)\n\ndf = df[(df[\"source_text\"].str.len() > 0) & (df[\"target_text\"].str.len() > 0)].reset_index(drop=True)\n\n# 2) Split (same policy as earlier: 10% val, deterministic)\ntrain_df_clean, val_df_clean = train_test_split(\n    df,\n    test_size=0.10,\n    random_state=42\n)\n\ntrain_df_clean = train_df_clean.reset_index(drop=True)\nval_df_clean   = val_df_clean.reset_index(drop=True)\n\n# 3) Keep only useful columns (lock schema for tokenization)\ntrain_df_clean = train_df_clean[[\"source_text\", \"target_text\"]]\nval_df_clean   = val_df_clean[[\"source_text\", \"target_text\"]]\n\n# 4) Build HF Datasets\ntrain_ds = Dataset.from_pandas(train_df_clean, preserve_index=False)\nval_ds   = Dataset.from_pandas(val_df_clean, preserve_index=False)\n\nprint(\"✅ Dataset Creation Successful\")\nprint(\"Total cleaned rows:\", len(df))\nprint(\"Total Training Rows:\", len(train_ds))\nprint(\"Total Validation Rows:\", len(val_ds))\nprint(\"-\" * 30)\n\n# 5) Quick peek (clean + readable)\nexample = train_ds[0]\nprint(\"🔍 DATASET PEEK:\")\nfor col in train_ds.column_names:\n    content = example[col]\n    if isinstance(content, str):\n        content = content[:150] + \"...\" if len(content) > 150 else content\n    print(f\"[{col}]: {content}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:36.851657Z","iopub.execute_input":"2026-02-18T02:25:36.85186Z","iopub.status.idle":"2026-02-18T02:25:37.620859Z","shell.execute_reply.started":"2026-02-18T02:25:36.85184Z","shell.execute_reply":"2026-02-18T02:25:37.620026Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 12.1 — Observations\n1. Dataset sizes are correct and consistent\ntrain_ds: 1,404 rows\nval_ds: 157 rows\nThese match exactly the split from Step 9, which confirms:\nno rows were dropped during conversion\nno duplication occurred\ntrain/validation integrity is preserved\n2. Feature schema is minimal and clean\nfeatures: ['src_clean_v1', 'tgt_clean_v1']\nObservation:\nOnly the two required text fields are present\nNo stray columns (indices, IDs, or metadata)","metadata":{}},{"cell_type":"code","source":"# # =========================\n# # Downloadable copy of sample submission (for visual inspection)\n# # =========================\n# import pandas as pd\n# import os\n\n# SAMPLE_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/sample_submission.csv\"\n# OUT_PATH = \"/kaggle/working/sample_submission_copy.csv\"\n\n# sample_df = pd.read_csv(SAMPLE_PATH)\n\n# print(\"Sample shape:\", sample_df.shape)\n# print(\"Sample columns:\", list(sample_df.columns))\n# display(sample_df.head(5))\n\n# # Hard validation of expected schema\n# expected_cols = [\"id\", \"translation\"]\n# if list(sample_df.columns) != expected_cols:\n#     raise ValueError(f\"Sample submission columns mismatch. Expected {expected_cols}, got {list(sample_df.columns)}\")\n\n# # Write a copy to working dir (Kaggle lets you download this)\n# sample_df.to_csv(OUT_PATH, index=False)\n# print(\"✅ Wrote downloadable copy to:\", OUT_PATH)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:37.621834Z","iopub.execute_input":"2026-02-18T02:25:37.622385Z","iopub.status.idle":"2026-02-18T02:25:37.625956Z","shell.execute_reply.started":"2026-02-18T02:25:37.622358Z","shell.execute_reply":"2026-02-18T02:25:37.625338Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 12.2 — Tokenize (ByT5) with locked max lengths","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 12.2 — Tokenize (CANONICAL)\n# - source_text + target_text\n# - lengths: src=MAX_SOURCE_LENGTH, tgt=MAX_TARGET_LENGTH (from Step 11.2)\n# - mask label padding to -100\n# - include truncation diagnostics\n# =========================\n\nif \"train_ds\" not in globals() or \"val_ds\" not in globals():\n    raise NameError(\"❌ train_ds/val_ds not found. Run canonical Step 12.1 first.\")\n\nrequired_cols = set([\"source_text\", \"target_text\"])\nif set(train_ds.column_names) != required_cols or set(val_ds.column_names) != required_cols:\n    raise ValueError(\n        f\"❌ Dataset schema mismatch.\\n\"\n        f\"Expected columns: {sorted(required_cols)}\\n\"\n        f\"train_ds columns: {train_ds.column_names}\\n\"\n        f\"val_ds columns  : {val_ds.column_names}\\n\"\n        f\"Do NOT tokenize src_clean_v1/tgt_clean_v1 directly; use canonical datasets.\"\n    )\n\ndef tokenize_batch(batch):\n    # 1) Tokenize SOURCE\n    src = tokenizer(\n        batch[\"source_text\"],\n        max_length=MAX_SOURCE_LENGTH,\n        truncation=True,\n        padding=\"max_length\",\n    )\n\n    # 2) Tokenize TARGET\n    tgt = tokenizer(\n        batch[\"target_text\"],\n        max_length=MAX_TARGET_LENGTH,\n        truncation=True,\n        padding=\"max_length\",\n    )\n\n    # 3) Mask TARGET padding so loss ignores it\n    pad_id = tokenizer.pad_token_id\n    labels = tgt[\"input_ids\"]\n    labels = [\n        [(tok if tok != pad_id else -100) for tok in seq]\n        for seq in labels\n    ]\n\n    # 4) Add labels\n    src[\"labels\"] = labels\n\n    # 5) Truncation flags (diagnostics)\n    # If attention_mask sums to max_length, it likely hit the limit (not perfect, but useful)\n    src[\"src_len\"] = [int(sum(m)) for m in src[\"attention_mask\"]]\n    # For targets, count non-pad tokens (pad_id present in tgt input_ids)\n    src[\"tgt_len\"] = [int(sum(1 for tok in seq if tok != pad_id)) for seq in tgt[\"input_ids\"]]\n\n    return src\n\n# Tokenize datasets\ntokenized_train = train_ds.map(\n    tokenize_batch,\n    batched=True,\n    remove_columns=train_ds.column_names\n)\ntokenized_val = val_ds.map(\n    tokenize_batch,\n    batched=True,\n    remove_columns=val_ds.column_names\n)\n\nprint(\"tokenized_train:\", tokenized_train)\nprint(\"tokenized_val  :\", tokenized_val)\n\n# -------------------------\n# Diagnostics: truncation rate\n# -------------------------\ntrain_src_trunc = sum(1 for l in tokenized_train[\"src_len\"] if l >= MAX_SOURCE_LENGTH)\ntrain_tgt_trunc = sum(1 for l in tokenized_train[\"tgt_len\"] if l >= MAX_TARGET_LENGTH)\n\nval_src_trunc = sum(1 for l in tokenized_val[\"src_len\"] if l >= MAX_SOURCE_LENGTH)\nval_tgt_trunc = sum(1 for l in tokenized_val[\"tgt_len\"] if l >= MAX_TARGET_LENGTH)\n\nprint(\"\\n--- Truncation diagnostics (heuristic) ---\")\nprint(f\"Train src at max_len ({MAX_SOURCE_LENGTH}): {train_src_trunc} / {len(tokenized_train)}\")\nprint(f\"Train tgt at max_len ({MAX_TARGET_LENGTH}): {train_tgt_trunc} / {len(tokenized_train)}\")\nprint(f\"Val   src at max_len ({MAX_SOURCE_LENGTH}): {val_src_trunc} / {len(tokenized_val)}\")\nprint(f\"Val   tgt at max_len ({MAX_TARGET_LENGTH}): {val_tgt_trunc} / {len(tokenized_val)}\")\n\n# -------------------------\n# Sanity checks on one example\n# -------------------------\nex = tokenized_train[0]\n\nprint(\"\\nSanity lengths (train[0]):\")\nprint(\"input_ids len      :\", len(ex[\"input_ids\"]))        # expect MAX_SOURCE_LENGTH\nprint(\"attention_mask len :\", len(ex[\"attention_mask\"]))   # expect MAX_SOURCE_LENGTH\nprint(\"labels len         :\", len(ex[\"labels\"]))           # expect MAX_TARGET_LENGTH\n\nneg100_count = sum(1 for t in ex[\"labels\"] if t == -100)\npad_in_labels = sum(1 for t in ex[\"labels\"] if t == tokenizer.pad_token_id)\n\nprint(\"\\nSanity label masking (train[0]):\")\nprint(\"labels -100 count   :\", neg100_count)               # should be > 0 for padded examples (often)\nprint(\"labels pad_id count :\", pad_in_labels)              # MUST be 0\n\nprint(\"\\nLast 30 label tokens (train[0]):\")\nprint(ex[\"labels\"][-30:])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:37.62681Z","iopub.execute_input":"2026-02-18T02:25:37.627085Z","iopub.status.idle":"2026-02-18T02:25:40.201626Z","shell.execute_reply.started":"2026-02-18T02:25:37.627055Z","shell.execute_reply":"2026-02-18T02:25:40.200909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Step 12.3 — Prepare MODEL-READY tokenized datasets (drop src_len/tgt_len)\n# =========================\n\nif \"tokenized_train\" not in globals() or \"tokenized_val\" not in globals():\n    raise NameError(\"❌ tokenized_train/tokenized_val not found. Run Step 12.2 first.\")\n\nmodel_cols = [\"input_ids\", \"attention_mask\", \"labels\"]\ndrop_train = [c for c in tokenized_train.column_names if c not in model_cols]\ndrop_val   = [c for c in tokenized_val.column_names if c not in model_cols]\n\nprint(\"Tokenized train columns:\", tokenized_train.column_names)\nprint(\"Tokenized val columns  :\", tokenized_val.column_names)\nprint(\"Dropping (train):\", drop_train)\nprint(\"Dropping (val)  :\", drop_val)\n\ntokenized_train_model = tokenized_train.remove_columns(drop_train) if drop_train else tokenized_train\ntokenized_val_model   = tokenized_val.remove_columns(drop_val) if drop_val else tokenized_val\n\nprint(\"\\n✅ Model-ready datasets created\")\nprint(\"train_model columns:\", tokenized_train_model.column_names)\nprint(\"val_model columns  :\", tokenized_val_model.column_names)\n\nassert set(tokenized_train_model.column_names) == set(model_cols), \"❌ train_model schema wrong\"\nassert set(tokenized_val_model.column_names) == set(model_cols), \"❌ val_model schema wrong\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:40.20266Z","iopub.execute_input":"2026-02-18T02:25:40.202965Z","iopub.status.idle":"2026-02-18T02:25:40.213539Z","shell.execute_reply.started":"2026-02-18T02:25:40.20294Z","shell.execute_reply":"2026-02-18T02:25:40.212745Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n### Step 12.1 — Tokenization\n\nIn this step we tokenize the source and target texts **separately** to ensure:\n- Source sequences are padded/truncated to `MAX_SOURCE_LENGTH = 1024`\n- Target sequences are padded/truncated to `MAX_TARGET_LENGTH = 512`\n\nCrucially, **all target padding tokens are masked to `-100`** so they are ignored by the loss function during training.\n\nSanity checks confirm:\n- Correct sequence lengths\n- Presence of `-100` in labels\n- Absence of `pad_token_id` in labels\n\nThis step fixes the NaN loss issue observed during initial training.\n","metadata":{}},{"cell_type":"markdown","source":"Step 13.1 — Trainer setup (args + metric)","metadata":{}},{"cell_type":"markdown","source":"### TRaining setup","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 13.1 — Trainer setup (SAFE SAVE FIX + keep best model + minimal NaN patch)\n# =========================\nimport os\nimport inspect\nimport torch\nfrom transformers import DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer\n\nrequired = [\n    \"tokenizer\", \"model\",\n    \"tokenized_train_model\", \"tokenized_val_model\",\n    \"NUM_EPOCHS\", \"WEIGHT_DECAY\",\n    \"PER_DEVICE_TRAIN_BATCH_SIZE\", \"PER_DEVICE_EVAL_BATCH_SIZE\",\n    \"GRADIENT_ACCUMULATION_STEPS\",\n    \"GEN_MAX_LENGTH\", \"GEN_NUM_BEAMS\",\n]\nmissing = [x for x in required if x not in globals()]\nif missing:\n    raise NameError(f\"❌ Missing required objects: {missing}\")\n\nOUTPUT_DIR = \"/kaggle/working/byt5_baseline_v0\"\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(DEVICE)\n\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model,\n    label_pad_token_id=-100\n)\n\n# ✅ Minimal NaN patch\nLR_SAFE = 5e-5  # lowered from 5e-4 / 1e-4\nMAX_GRAD_NORM = 1.0\n\ncandidate_args = dict(\n    output_dir=OUTPUT_DIR,\n    overwrite_output_dir=True,\n\n    num_train_epochs=NUM_EPOCHS,\n    learning_rate=LR_SAFE,\n    weight_decay=WEIGHT_DECAY,\n    per_device_train_batch_size=PER_DEVICE_TRAIN_BATCH_SIZE,\n    per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH_SIZE,\n    gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,\n\n    # ✅ Force off (stable now + future-proof for GPU)\n    fp16=False,\n\n    # ✅ keep Adafactor if you’re already using it; otherwise Trainer defaults to adamw_torch\n    optim=\"adafactor\",\n\n    max_grad_norm=MAX_GRAD_NORM,\n\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    save_total_limit=1,\n    save_only_model=True,\n    save_safetensors=True,\n\n    load_best_model_at_end=True,\n    metric_for_best_model=\"eval_loss\",\n    greater_is_better=False,\n\n    predict_with_generate=True,\n    generation_max_length=GEN_MAX_LENGTH,\n    generation_num_beams=GEN_NUM_BEAMS,\n\n    logging_steps=10,\n    report_to=\"none\",\n    remove_unused_columns=True,\n)\n\n# Version-safe filtering (no assumptions)\nsig = inspect.signature(Seq2SeqTrainingArguments.__init__)\nvalid_params = set(sig.parameters.keys())\n\nfiltered_args, dropped_args = {}, {}\nfor k, v in candidate_args.items():\n    if k in valid_params:\n        filtered_args[k] = v\n    else:\n        dropped_args[k] = v\n\ntraining_args = Seq2SeqTrainingArguments(**filtered_args)\n\nprint(\"✅ TrainingArguments created.\")\nprint(\"DEVICE:\", DEVICE)\nprint(\"fp16:\", getattr(training_args, \"fp16\", None))\nprint(\"learning_rate:\", getattr(training_args, \"learning_rate\", None))\nprint(\"optim:\", getattr(training_args, \"optim\", None))\nprint(\"max_grad_norm:\", getattr(training_args, \"max_grad_norm\", None))\nif dropped_args:\n    print(\"\\n⚠️ Dropped unsupported keys:\")\n    for k in sorted(dropped_args.keys()):\n        print(\" -\", k)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_train_model,\n    eval_dataset=tokenized_val_model,\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n)\n\nprint(\"\\n✅ Step 13.1 complete — Trainer initialized (minimal NaN patch).\")\nprint(\"Train cols:\", trainer.train_dataset.column_names)\nprint(\"Eval cols :\", trainer.eval_dataset.column_names)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:40.214744Z","iopub.execute_input":"2026-02-18T02:25:40.215396Z","iopub.status.idle":"2026-02-18T02:25:42.064793Z","shell.execute_reply.started":"2026-02-18T02:25:40.215371Z","shell.execute_reply":"2026-02-18T02:25:42.064057Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Step 13.2 — Train (same as before","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 13.2A — NaN diagnostic (forward + backward on a few batches)\n# =========================\nimport torch\nimport numpy as np\n\n# Guards\nif \"tokenized_train_model\" not in globals():\n    raise NameError(\"❌ tokenized_train_model not found. Run Step 12.3 first.\")\nif \"data_collator\" not in globals():\n    raise NameError(\"❌ data_collator not found. Run Step 13.1 first.\")\nif \"model\" not in globals():\n    raise NameError(\"❌ model not found.\")\n\ndevice = next(model.parameters()).device\nmodel.train()\n\ndef batch_stats(batch):\n    # Count how many label tokens are NOT -100\n    labels = batch[\"labels\"]\n    valid = (labels != -100).sum().item()\n    total = labels.numel()\n    return valid, total\n\n# Try a few batches to see if NaN appears even before optimizer steps\nbad = False\nfor start in [0, 8, 16]:\n    examples = [tokenized_train_model[i] for i in range(start, min(start + 2, len(tokenized_train_model)))]\n    collated = data_collator(examples)\n\n    # Move to device\n    collated = {k: v.to(device) for k, v in collated.items() if torch.is_tensor(v)}\n\n    valid, total = batch_stats(collated)\n    print(f\"\\nBatch starting at {start}: valid label tokens = {valid} / {total}\")\n\n    # Forward\n    out = model(**collated)\n    loss = out.loss\n\n    print(\"loss:\", float(loss.detach().cpu()), \"| finite:\", bool(torch.isfinite(loss).item()))\n    if not torch.isfinite(loss).item():\n        bad = True\n        break\n\n    # Backward (no optimizer step)\n    model.zero_grad(set_to_none=True)\n    loss.backward()\n\n    # Check grads\n    any_bad_grad = False\n    for n, p in model.named_parameters():\n        if p.grad is not None and (not torch.isfinite(p.grad).all()):\n            print(\"❌ Non-finite grad in:\", n)\n            any_bad_grad = True\n            break\n\n    print(\"grads finite:\", (not any_bad_grad))\n    if any_bad_grad:\n        bad = True\n        break\n\nprint(\"\\n✅ Step 13.2A done.\")\nif bad:\n    print(\"❌ NaN/Inf detected BEFORE optimizer step → data/batch issue.\")\nelse:\n    print(\"✅ Forward+backward are stable → optimizer/LR is the likely NaN source.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:42.06566Z","iopub.execute_input":"2026-02-18T02:25:42.065961Z","iopub.status.idle":"2026-02-18T02:25:45.392799Z","shell.execute_reply.started":"2026-02-18T02:25:42.065936Z","shell.execute_reply":"2026-02-18T02:25:45.391996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Step 13.2 — Train + Eval (with NaN guards)\n# =========================\nimport math\n\nif \"trainer\" not in globals():\n    raise NameError(\"❌ trainer not found. Run Step 13.1 first.\")\n\n# Hard guard: schema must be model-only (Trainer may add decoder_* at collate time, that's fine)\nexpected_cols = {\"input_ids\", \"attention_mask\", \"labels\"}\ntrain_cols = set(trainer.train_dataset.column_names)\neval_cols  = set(trainer.eval_dataset.column_names)\n\nif train_cols != expected_cols or eval_cols != expected_cols:\n    raise ValueError(\n        \"❌ Trainer dataset schema is not model-ready.\\n\"\n        f\"Expected: {sorted(expected_cols)}\\n\"\n        f\"Train cols: {sorted(train_cols)}\\n\"\n        f\"Eval cols : {sorted(eval_cols)}\\n\"\n        \"Fix: ensure you are using tokenized_train_model/tokenized_val_model (Step 12.3).\"\n    )\n\n# Train\ntrain_result = trainer.train()\nprint(\"✅ Training complete.\")\n\n# NaN/Inf guard\nlosses = [x.get(\"loss\") for x in trainer.state.log_history if \"loss\" in x]\nif losses:\n    last_loss = losses[-1]\n    print(\"Last logged loss:\", last_loss)\n    if last_loss is None or (isinstance(last_loss, float) and (math.isnan(last_loss) or math.isinf(last_loss))):\n        raise ValueError(\"❌ Detected NaN/Inf loss during training.\")\nelse:\n    print(\"⚠️ No loss entries found in log_history (not fatal, but unusual).\")\n\n# Eval\neval_result = trainer.evaluate()\nprint(\"✅ Eval complete.\")\nprint(eval_result)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-18T02:25:45.393913Z","iopub.execute_input":"2026-02-18T02:25:45.394215Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Step 13.1A — Trainer/model sanity check (MODEL-READY, FIXED)\n# =========================\nimport numpy as np\nimport torch\n\n# Guards\nif \"trainer\" not in globals():\n    raise NameError(\"❌ trainer not found. Run Step 13.1 first.\")\nif \"tokenized_train_model\" not in globals():\n    raise NameError(\"❌ tokenized_train_model not found. Run Step 12.3 first (drop src_len/tgt_len).\")\nif \"data_collator\" not in globals():\n    raise NameError(\"❌ data_collator not found. Run Step 13.1 first.\")\n\nprint(\"Trainer model is model:\", trainer.model is model)\nprint(\"Trainer device:\", next(trainer.model.parameters()).device)\n\n# Build a tiny batch from MODEL-READY tokenized data\nbatch = [tokenized_train_model[i] for i in range(2)]\ncollated = data_collator(batch)\n\nprint(\"\\nCollated keys:\", sorted(collated.keys()))\n\n# Allow keys that the seq2seq collator may add\nallowed_keys = {\n    \"input_ids\",\n    \"attention_mask\",\n    \"labels\",\n    \"decoder_input_ids\",      # ✅ collator may add this\n    \"decoder_attention_mask\", # ✅ sometimes present\n}\n\nextra_keys = set(collated.keys()) - allowed_keys\nif extra_keys:\n    raise ValueError(\n        f\"❌ Collated batch has unexpected keys: {sorted(extra_keys)}\\n\"\n        \"If you see src_len/tgt_len here, the diagnostic columns leaked into training again.\"\n    )\n\n# Move tensors to correct device\ndevice = next(trainer.model.parameters()).device\ncollated = {k: v.to(device) for k, v in collated.items()}\n\ntrainer.model.eval()\nwith torch.no_grad():\n    out = trainer.model(**collated)\n\nloss = out.loss\nlogits = out.logits.detach().float().cpu().numpy()\n\nprint(\"\\nloss finite:\", bool(torch.isfinite(loss).item()), \"| loss:\", float(loss.detach().cpu()))\nprint(\"logits finite %:\", float(np.isfinite(logits).mean()) * 100)\nprint(\"logits min/max:\", float(np.nanmin(logits)), float(np.nanmax(logits)))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Step 13.D4a — Collator check (post-fix)\n# =========================\nimport numpy as np\n\nbatch = [tokenized_train[i] for i in range(4)]\ncollated = data_collator(batch)\nlabels = collated[\"labels\"].detach().cpu().numpy()\n\nprint(\"labels shape:\", labels.shape)\nprint(\"Count -100:\", int((labels == -100).sum()))\nprint(\"Count pad_id:\", int((labels == tokenizer.pad_token_id).sum()))\nprint(\"labels min/max:\", int(labels.min()), int(labels.max()))\nprint(\"Last 20 labels ex0:\", labels[0, -20:])\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Step 13.D4b — Count all-ignored label rows\n# =========================\ndef all_ignored(example):\n    return all(t == -100 for t in example[\"labels\"])\n\nn_all_ignored = sum(all_ignored(tokenized_train[i]) for i in range(len(tokenized_train)))\nprint(\"Train rows where ALL labels == -100:\", n_all_ignored, \"/\", len(tokenized_train))\n\nn_all_ignored_val = sum(all_ignored(tokenized_val[i]) for i in range(len(tokenized_val)))\nprint(\"Val rows where ALL labels == -100:\", n_all_ignored_val, \"/\", len(tokenized_val))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Step 13.D4c — Forward-pass NaN test (fp32 vs fp16) — FIXED (model-ready dataset)\n# =========================\nimport torch\nimport numpy as np\n\n# ---- Guards ----\nif \"model\" not in globals():\n    raise NameError(\"❌ model not found.\")\nif \"data_collator\" not in globals():\n    raise NameError(\"❌ data_collator not found.\")\nif \"tokenized_train_model\" not in globals():\n    raise NameError(\"❌ tokenized_train_model not found. Run Step 12.3 (drop src_len/tgt_len).\")\n\ndevice = next(model.parameters()).device\n\n# ✅ Use MODEL-READY tokenized dataset (NO src_len/tgt_len)\nbatch = [tokenized_train_model[i] for i in range(2)]\ncollated = data_collator(batch)\n\n# ✅ Keep only keys the model can accept\nallowed_keys = {\"input_ids\", \"attention_mask\", \"labels\", \"decoder_input_ids\", \"decoder_attention_mask\"}\ncollated = {k: v for k, v in collated.items() if k in allowed_keys}\n\n# Move to device\nbatch_on_device = {k: v.to(device) for k, v in collated.items()}\n\ndef forward_test(use_fp16: bool):\n    model.eval()\n    with torch.no_grad():\n        if use_fp16 and device.type == \"cuda\":\n            with torch.autocast(device_type=\"cuda\", dtype=torch.float16):\n                out = model(**batch_on_device)\n        else:\n            out = model(**batch_on_device)\n\n    loss = out.loss\n    logits = out.logits.detach().float().cpu().numpy()\n\n    print(\"\\n--- Forward test ---\")\n    print(\"device:\", device)\n    print(\"use_fp16:\", use_fp16)\n    print(\"loss is finite:\", bool(torch.isfinite(loss).item()), \"| loss:\", float(loss.detach().cpu()) if torch.isfinite(loss) else loss)\n    print(\"logits finite %:\", float(np.isfinite(logits).mean()) * 100)\n    print(\"logits min/max:\", float(np.nanmin(logits)), float(np.nanmax(logits)))\n\nforward_test(use_fp16=False)  # fp32 path\nforward_test(use_fp16=True)   # fp16 path (only runs autocast on CUDA)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Debug #5.A — Measure true token lengths (no truncation)\n# =========================\nimport numpy as np\nimport pandas as pd\n\n# Assumes you still have train_texts with columns: src_clean_v1, tgt_clean_v1\n# If not, build it from mt_df/train split: train_texts = train_split_df (whatever you named it)\n\nSRC_COL = \"src_clean_v1\"\nTGT_COL = \"tgt_clean_v1\"\n\ndef tok_len(text: str) -> int:\n    return len(tokenizer(text, add_special_tokens=True, truncation=False)[\"input_ids\"])\n\n# Compute lengths (can take a minute on 1404 rows)\nlens = pd.DataFrame({\n    \"src_len\": train_texts[SRC_COL].astype(str).map(tok_len),\n    \"tgt_len\": train_texts[TGT_COL].astype(str).map(tok_len),\n})\n\nprint(lens.describe(percentiles=[0.5, 0.9, 0.95, 0.99]).round(2))\n\n# Top longest targets\ntop = lens.sort_values(\"tgt_len\", ascending=False).head(10)\nprint(\"\\nTop 10 longest target rows (by tgt_len):\")\ndisplay(top)\n\n# Save indices for later\nLONG_TGT_IDXS = top.index.tolist()\nprint(\"LONG_TGT_IDXS:\", LONG_TGT_IDXS)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Step 14. Baseline submision","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 14 — Baseline Submission Generation (t5-small)\n# Produces: /kaggle/working/submission.csv\n# =========================\nimport os\nimport re\nimport numpy as np\nimport pandas as pd\nimport torch\n\n# ---- REQUIRED INPUTS  ----\n# model, tokenizer, device\n# test_df with columns: ['id', ..., 'transliteration']\n# sample_sub with columns: ['id', 'translation']\n\nOUT_DIR = \"/kaggle/working\"\nSUB_PATH = os.path.join(OUT_DIR, \"submission.csv\")\nos.makedirs(OUT_DIR, exist_ok=True)\n\nassert \"id\" in test_df.columns, \"test_df must contain an 'id' column\"\nassert \"transliteration\" in test_df.columns, \"test_df must contain a 'transliteration' column\"\nassert list(sample_sub.columns) == [\"id\", \"translation\"], \"sample_sub must have columns: ['id','translation']\"\n\n# ---- Decode settings (baseline, safe - UPDATED WITH LOCKS) ----\nBATCH_SIZE = 4  # adjust if you hit OOM\nMAX_INPUT_LEN = MAX_SOURCE_LENGTH          # from locked Config\nMAX_NEW_TOKENS = GEN_MAX_NEW_TOKENS        # FIX: Changed from 256 to locked value (e.g. 512/1024)\nNUM_BEAMS = GEN_NUM_BEAMS                  # from locked Config (e.g., 5)\nDO_SAMPLE = False\n\n# ---- Text postprocess (light-touch) ----\n_ws_re = re.compile(r\"\\s+\")\ndef clean_pred_text(s: str) -> str:\n    s = str(s)\n    s = _ws_re.sub(\" \", s).strip()\n    return s\n\n# ---- Ensure model eval + on correct device ----\nmodel.eval()\nmodel.to(device)\n\n# ---- Generate predictions ----\nids = test_df[\"id\"].tolist()\nsrc_texts = test_df[\"transliteration\"].astype(str).tolist()\n\npreds = []\nwith torch.no_grad():\n    for start in range(0, len(src_texts), BATCH_SIZE):\n        batch_texts = src_texts[start:start+BATCH_SIZE]\n\n        enc = tokenizer(\n            batch_texts,\n            truncation=True,\n            padding=True,\n            max_length=MAX_INPUT_LEN,\n            return_tensors=\"pt\",\n        )\n        enc = {k: v.to(device) for k, v in enc.items()}\n\n        # 🔒 GENERATION WITH LOCKED GUARDS\n        gen_ids = model.generate(\n            **enc,\n            max_new_tokens=MAX_NEW_TOKENS,\n            num_beams=NUM_BEAMS,\n            repetition_penalty=REPETITION_PENALTY,     # FIX: Added locked penalty (1.2)\n            no_repeat_ngram_size=NO_REPEAT_NGRAM_SIZE, # FIX: Added locked n-gram (3)\n            do_sample=DO_SAMPLE,\n            early_stopping=True,\n        )\n\n        batch_preds = tokenizer.batch_decode(gen_ids, skip_special_tokens=True)\n        preds.extend([clean_pred_text(p) for p in batch_preds])\n\n# ---- Build submission in sample format ----\nsub_df = pd.DataFrame({\"id\": ids, \"translation\": preds})\n\n# 🔒 THE EMPTY MASK GUARD (Locked per 2026-02-17 instruction)\nempty_mask = sub_df[\"translation\"].isna() | (sub_df[\"translation\"].astype(\"string\").str.strip() == \"\")\nsub_df.loc[empty_mask, \"translation\"] = \"\" # Safeguard empty strings\n\n# Ensure exact id alignment with sample_submission\nsub_df = sub_df.merge(sample_sub[[\"id\"]], on=\"id\", how=\"right\")\nsub_df[\"translation\"] = sub_df[\"translation\"].fillna(\"\")\n\n# Final column order\nsub_df = sub_df[[\"id\", \"translation\"]]\n\n# ---- Write submission.csv ----\nsub_df.to_csv(SUB_PATH, index=False)\nprint(\"✅ Wrote:\", SUB_PATH)\nprint(\"shape:\", sub_df.shape)\ndisplay(sub_df.head(5))\n\n# ---- Kaggle download helper (run in notebook) ----\n# After running this cell, you can download from the \"Output\" panel,\n# or uncomment the next line in Kaggle:\n# from IPython.display import FileLink; FileLink(SUB_PATH)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compliance check\nprint(\"Rows in test_df:\", len(test_df))\nprint(\"Rows in submission:\", len(sub_df))\nprint(\"Missing ids:\", set(test_df[\"id\"]) - set(sub_df[\"id\"]))\nprint(\"Extra ids:\", set(sub_df[\"id\"]) - set(test_df[\"id\"]))\nprint(\"Columns:\", list(sub_df.columns))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 15.1: Improve decoding (generation) without retraining.","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 15.1 — Decode tuning (NO retraining)\n# Goal: reduce repetition + avoid dot-only outputs\n# Output: /kaggle/working/submission.csv (ready to submit)\n# =========================\nimport os\nimport re\nimport pandas as pd\nimport torch\n\nOUTPUT_PATH = \"/kaggle/working/submission.csv\"   # Kaggle expects this name\nDISPLAY_N = 4\n\n# --- Safety: ensure model is in eval mode and on GPU if available ---\nmodel.eval()\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# --- Generation settings (tuned per 2026-02-17 LOCKS) ---\nGEN_CFG = dict(\n    max_new_tokens=GEN_MAX_NEW_TOKENS,   # FIX: Use locked 1024/512 from Step 11.2\n    min_new_tokens=24,                   # avoid ultra-short outputs\n    num_beams=GEN_NUM_BEAMS,             # FIX: Use locked beam count (5)\n    early_stopping=True,\n\n    # Anti-repetition controls (high ROI - UPDATED TO LOCKS)\n    no_repeat_ngram_size=NO_REPEAT_NGRAM_SIZE, # FIX: Use locked 3\n    repetition_penalty=REPETITION_PENALTY,     # FIX: Use locked 1.2\n    \n    # Length behavior\n    length_penalty=1.0,                  \n)\n\n# If you want to try sampling later (optional), keep this False for now.\nUSE_SAMPLING = False\nif USE_SAMPLING:\n    GEN_CFG.update(dict(\n        do_sample=True,\n        top_p=0.92,\n        temperature=0.8,\n        num_beams=1\n    ))\n\n# --- Helper: detect \"dots-only\" / punctuation-only outputs ---\n_punct_only_re = re.compile(r\"^[\\s\\.\\-\\–\\—_,'\\\";:!?()\\[\\]{}]+$\")\n\ndef postprocess_pred(s: str) -> str:\n    s = (s or \"\").strip()\n    # Collapse whitespace\n    s = re.sub(r\"\\s+\", \" \", s).strip()\n\n    # If model outputs only punctuation/dots, replace with minimal fallback.\n    if len(s) == 0 or _punct_only_re.match(s):\n        return \"Unclear.\"\n    return s\n\n# --- Generate for test_df ---\nassert \"id\" in test_df.columns, \"test_df must contain column 'id'\"\nassert \"transliteration\" in test_df.columns, \"test_df must contain column 'transliteration'\"\n\nids = test_df[\"id\"].tolist()\nsrcs = test_df[\"transliteration\"].astype(str).tolist()\n\npreds = []\nBATCH_SIZE = 4  \n\nwith torch.no_grad():\n    for i in range(0, len(srcs), BATCH_SIZE):\n        batch_src = srcs[i:i+BATCH_SIZE]\n\n        enc = tokenizer(\n            batch_src,\n            return_tensors=\"pt\",\n            padding=True,\n            truncation=True,\n            max_length=MAX_SOURCE_LENGTH,   # locked at 1024\n        )\n        enc = {k: v.to(device) for k, v in enc.items()}\n\n        out_ids = model.generate(\n            **enc,\n            **GEN_CFG\n        )\n\n        batch_preds = tokenizer.batch_decode(out_ids, skip_special_tokens=True)\n        batch_preds = [postprocess_pred(p) for p in batch_preds]\n        preds.extend(batch_preds)\n\n# --- Build and save submission ---\nsub_df = pd.DataFrame({\"id\": ids, \"translation\": preds})\n\n# 🔒 THE EMPTY MASK GUARD (Locked per 2026-02-17 instruction)\n# Ensures we don't submit NaNs or whitespace-only strings\nempty_mask = sub_df[\"translation\"].isna() | (sub_df[\"translation\"].astype(\"string\").str.strip() == \"\")\nsub_df.loc[empty_mask, \"translation\"] = \"Unclear.\" \n\nsub_df.to_csv(OUTPUT_PATH, index=False)\n\nprint(\"✅ Step 15.1 complete — wrote:\", OUTPUT_PATH)\nprint(\"shape:\", sub_df.shape)\ndisplay(sub_df.head(DISPLAY_N))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 16","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 16 — Train longer (same pipeline) ✅ FIX: model-ready datasets + fp16 locked OFF\n# =========================\nimport os\nimport inspect\nimport torch\nfrom transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer, DataCollatorForSeq2Seq\n\n# ---- Guards (no assumptions) ----\nrequired = [\n    \"tokenizer\", \"model\",\n    \"tokenized_train_model\", \"tokenized_val_model\",\n    \"PER_DEVICE_TRAIN_BATCH_SIZE\", \"PER_DEVICE_EVAL_BATCH_SIZE\",\n    \"GRADIENT_ACCUMULATION_STEPS\",\n]\nmissing = [x for x in required if x not in globals()]\nif missing:\n    raise NameError(f\"❌ Missing required objects: {missing}\")\n\n# Hard guard: model-ready schema only\nexpected_cols = {\"input_ids\", \"attention_mask\", \"labels\"}\ntrain_cols = set(tokenized_train_model.column_names)\nval_cols   = set(tokenized_val_model.column_names)\nif train_cols != expected_cols or val_cols != expected_cols:\n    raise ValueError(\n        \"❌ tokenized_train_model/tokenized_val_model are not model-ready.\\n\"\n        f\"Expected: {sorted(expected_cols)}\\n\"\n        f\"Train cols: {sorted(train_cols)}\\n\"\n        f\"Val cols  : {sorted(val_cols)}\\n\"\n        \"Fix: re-run Step 12.3 to create model-ready datasets.\"\n    )\n\nOUTPUT_DIR = \"/kaggle/working/t5small_v1_longer\"\n\n# Optional wipe (keep if you want a clean run)\nos.system(f\"rm -rf {OUTPUT_DIR}\")\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\n# ---- Locked training choices for this step ----\nNUM_EPOCHS = 8\nLEARNING_RATE = 3e-4\n\n# Device + fp16 rule\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(DEVICE)\n\n# 🔒 Your rule: fp16 ALWAYS False\nFP16_LOCKED = False\n\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model,\n    label_pad_token_id=-100\n)\n\n# Build args (version-safe: eval_strategy vs evaluation_strategy differences)\ncandidate_args = dict(\n    output_dir=OUTPUT_DIR,\n    overwrite_output_dir=True,\n    num_train_epochs=NUM_EPOCHS,\n    learning_rate=LEARNING_RATE,\n    weight_decay=0.0,\n    per_device_train_batch_size=PER_DEVICE_TRAIN_BATCH_SIZE,\n    per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH_SIZE,\n    gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,\n\n    fp16=FP16_LOCKED,  # 🔒 always off\n\n    # Evaluation + saving\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n\n    # ✅ checkpoint safety\n    save_only_model=True,\n    save_safetensors=True,\n    save_total_limit=1,\n\n    load_best_model_at_end=True,\n    metric_for_best_model=\"eval_loss\",\n    greater_is_better=False,\n\n    # Keep eval stable/fast\n    predict_with_generate=False,\n\n    logging_steps=50,\n    report_to=\"none\",\n\n    # ✅ must be True so trainer doesn't pass non-model keys\n    remove_unused_columns=True,\n)\n\n# Filter args to what this transformers version supports\nsig = inspect.signature(Seq2SeqTrainingArguments.__init__)\nvalid_params = set(sig.parameters.keys())\n\nfiltered_args, dropped = {}, {}\nfor k, v in candidate_args.items():\n    if k in valid_params:\n        filtered_args[k] = v\n    else:\n        dropped[k] = v\n\ntraining_args = Seq2SeqTrainingArguments(**filtered_args)\n\nprint(\"✅ Step 16.1 — TrainingArguments created.\")\nprint(\"DEVICE:\", DEVICE)\nprint(\"fp16:\", getattr(training_args, \"fp16\", None))\nprint(\"learning_rate:\", getattr(training_args, \"learning_rate\", None))\nif dropped:\n    print(\"⚠️ Dropped unsupported args:\", sorted(dropped.keys()))\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_train_model,  # ✅ FIXED\n    eval_dataset=tokenized_val_model,      # ✅ FIXED\n    data_collator=data_collator,\n    tokenizer=tokenizer,\n)\n\nprint(\"✅ Step 16.1 — starting training...\")\ntrainer.train()\nprint(\"✅ Step 16.1 — training complete. Best checkpoint loaded.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Step 16.1 Submission generation with longer time","metadata":{}},{"cell_type":"code","source":"# =========================\n# Step 16.1 — Train longer (same pipeline) ✅ FIX: VERSION 34 PARAMETERS\n# =========================\nimport os\nimport inspect\nimport torch\nfrom transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer, DataCollatorForSeq2Seq\n\n# ... (Keep your Guards and Schema checks here) ...\n\nOUTPUT_DIR = \"/kaggle/working/t5small_v1_longer\"\nos.system(f\"rm -rf {OUTPUT_DIR}\")\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(DEVICE)\nFP16_LOCKED = False\n\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model, label_pad_token_id=-100)\n\ncandidate_args = dict(\n    output_dir=OUTPUT_DIR,\n    overwrite_output_dir=True,\n    num_train_epochs=NUM_EPOCHS,\n    learning_rate=LEARNING_RATE,\n    weight_decay=WEIGHT_DECAY,         \n    per_device_train_batch_size=PER_DEVICE_TRAIN_BATCH_SIZE,\n    per_device_eval_batch_size=PER_DEVICE_EVAL_BATCH_SIZE,\n    gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,\n    fp16=FP16_LOCKED,\n    eval_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    save_only_model=True,\n    save_safetensors=True,\n    save_total_limit=1,\n    load_best_model_at_end=True,\n    metric_for_best_model=\"eval_loss\",\n    greater_is_better=False,\n    predict_with_generate=False,\n    logging_steps=50,\n    report_to=\"none\",\n    remove_unused_columns=True,\n)\n\n# ... (Keep the rest of your filter and trainer logic) ...","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Step 16x","metadata":{}},{"cell_type":"code","source":"# ========================================================\n# STEP 16.X — FULL INFERENCE TEST (Submission Generator) ✅ patched + guarded\n# ========================================================\nimport torch\nimport pandas as pd\nimport re\nfrom tqdm.auto import tqdm\n\nOUTPUT_PATH = \"/kaggle/working/submission.csv\"\n\n# ---- Guards (no assumptions) ----\nrequired = [\"model\", \"tokenizer\", \"test_df\", \"clean_src_v1\", \"MAX_SOURCE_LENGTH\"]\nmissing = [x for x in required if x not in globals()]\nif missing:\n    raise NameError(f\"❌ Missing required objects: {missing}\")\n\nif \"transliteration\" not in test_df.columns:\n    raise KeyError(f\"❌ test_df missing 'transliteration'. Columns: {list(test_df.columns)}\")\nif \"id\" not in test_df.columns:\n    raise KeyError(f\"❌ test_df missing 'id'. Columns: {list(test_df.columns)}\")\n\n# 1) Setup the Model\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\nmodel.eval()\n\n# 2) Extract and Clean the TEST data (explicit column name)\ntest_sentences = (\n    test_df[\"transliteration\"]\n    .fillna(\"\")\n    .astype(str)\n    .map(clean_src_v1)\n    .tolist()\n)\n\nprint(f\"📦 Starting inference on {len(test_sentences)} rows...\")\nprint(\"DEVICE:\", device)\nprint(\"MAX_SOURCE_LENGTH:\", MAX_SOURCE_LENGTH)\n\n# 3) Output post-processing (guards empty / punctuation-only)\n_punct_only_re = re.compile(r\"^[\\s\\.\\-\\–\\—_,'\\\";:!?()\\[\\]{}]+$\")\n\ndef postprocess_pred(s: str) -> str:\n    s = (s or \"\").strip()\n    s = re.sub(r\"\\s+\", \" \", s).strip()\n    if len(s) == 0 or _punct_only_re.match(s):\n        return \"Unclear.\"\n    return s\n\n# 4) Batching Configuration\nBATCH_SIZE = 16  # lower if you hit OOM on GPU\nall_translations = []\n\n# 5) The Loop\nwith torch.no_grad():\n    for i in tqdm(range(0, len(test_sentences), BATCH_SIZE)):\n        batch = test_sentences[i : i + BATCH_SIZE]\n\n        inputs = tokenizer(\n            batch,\n            return_tensors=\"pt\",\n            padding=True,\n            truncation=True,\n            max_length=MAX_SOURCE_LENGTH,\n        )\n        inputs = {k: v.to(device) for k, v in inputs.items()}\n\n        generated_ids = model.generate(\n            input_ids=inputs[\"input_ids\"],\n            attention_mask=inputs[\"attention_mask\"],\n            max_new_tokens=GEN_MAX_NEW_TOKENS ,   # match your Step 16.2 intention\n            min_new_tokens=GEN_MIN_NEW_TOKENS ,\n            num_beams=GEN_NUM_BEAMS,\n            early_stopping=True,\n            no_repeat_ngram_size=NO_REPEAT_NGRAM_SIZE,\n            repetition_penalty=REPETITION_PENALTY,\n            length_penalty=GEN_LENGTH_PENALTY,\n        )\n\n        decoded_batch = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n        decoded_batch = [postprocess_pred(x) for x in decoded_batch]\n        all_translations.extend(decoded_batch)\n\n# ---- Hard guards ----\nif len(all_translations) != len(test_df):\n    raise ValueError(\n        f\"❌ Mismatch: translations={len(all_translations)} vs test_df rows={len(test_df)}\"\n    )\n\n# Placeholder leakage guard (from earlier debug Step 6)\nif any(t.strip() == \"...\" for t in all_translations):\n    raise ValueError(\"❌ Found '...' placeholder in translations — do NOT submit.\")\n\n# 6) Save Submission\nsubmission_df = pd.DataFrame({\n    \"id\": test_df[\"id\"].tolist(),\n    \"translation\": all_translations\n})\n\nsubmission_df.to_csv(OUTPUT_PATH, index=False)\n\nprint(f\"✅ SUCCESS! Generated {len(submission_df)} translations.\")\nprint(\"Saved to:\", OUTPUT_PATH)\ndisplay(submission_df.head(10))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !rm -rf /kaggle/working/t5small_v1_longer\n# !mkdir -p /kaggle/working/t5small_v1_longer\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import gc\n# import torch\n\n# # Clear Python garbage collection\n# gc.collect()\n\n# # Clear CUDA cache\n# if torch.cuda.is_available():\n#     torch.cuda.empty_cache()\n\n# print(\"GPU Memory Cleared.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}