{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"},{"sourceId":7902820,"sourceType":"datasetVersion","datasetId":4641566},{"sourceId":14188526,"sourceType":"datasetVersion","datasetId":9046477},{"sourceId":14197093,"sourceType":"datasetVersion","datasetId":9053011},{"sourceId":14199544,"sourceType":"datasetVersion","datasetId":9055585},{"sourceId":14199278,"sourceType":"datasetVersion","datasetId":9055384},{"sourceId":690127,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":523146,"modelId":537154},{"sourceId":690795,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":523698,"modelId":537719}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Baseline Solution: ByT5 + Sentence-Level Reconstruction\n\nThis notebook builds a **neural machine translation (NMT)** baseline to translate **Old Assyrian transliterations → English**.\n\n### 1) Document → sentence reconstruction\n\nThe provided `train.csv` is aligned at the **document level**. To train a seq2seq model effectively, we convert it into **sentence-level parallel data**:\n- Split transliterations into line-like segments (using `\\n`, then simple fallbacks).\n- Split English translations into sentences using punctuation.\n- Build **monotone alignments** by matching chunks with similar length (allowing 1–2 segment merges).\n\nThis produces clean `(src_sentence, tgt_sentence)` pairs for supervised training.\n\n### 2) Fine-tuning a seq2seq model (ByT5)\n\nWe fine-tune **ByT5** (`AutoModelForSeq2SeqLM`) on the reconstructed pairs:\n- Tokenize the source (transliteration) and target (English).\n- Train with `Seq2SeqTrainer` and `DataCollatorForSeq2Seq`.\n- Use small batches + gradient accumulation to fit GPU memory.\n\n### 3) Inference + submission formatting\n\nFor each test transliteration sentence, we generate an English translation (beam search). Since the competition expects **one sentence per row**, multi-sentence outputs are merged into a single sentence (joined with `;`) and ended with a final period.\n\nOutput: `submission.csv` with columns `id` and `translation`.","metadata":{}},{"cell_type":"code","source":"import os\n# 1. 设置环境变量\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T10:41:44.104621Z","iopub.execute_input":"2025-12-18T10:41:44.105354Z","iopub.status.idle":"2025-12-18T10:41:44.111874Z","shell.execute_reply.started":"2025-12-18T10:41:44.105322Z","shell.execute_reply":"2025-12-18T10:41:44.111168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, re, math, random\nimport numpy as np\nimport pandas as pd\n\nimport torch\nfrom torch.utils.data import Dataset\n\nfrom transformers import (\n    AutoTokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainingArguments,\n    Seq2SeqTrainer,\n)\n\nSEED = 42\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\n\nTRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\nTEST_PATH  = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n\n# Put your local model path here if you added it as a Kaggle Dataset:\n# e.g. \"/kaggle/input/byt5-small\" or \"/kaggle/input/mt5-small\"\nMODEL_PATH_CANDIDATES = [\"/kaggle/input/byt5-small\"]\n\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"Device:\", DEVICE)\n\ntrain_df = pd.read_csv(TRAIN_PATH)\ntest_df  = pd.read_csv(TEST_PATH)\n\nprint(\"train:\", train_df.shape, \"test:\", test_df.shape)\n\n\ndef clean_ws(x: str) -> str:\n    if pd.isna(x): return \"\"\n    return re.sub(r\"\\s+\", \" \", str(x)).strip()\n\ndef split_src_transliteration(doc: str):\n    \"\"\"Split transliteration doc into 'lines' as proxy for sentences.\"\"\"\n    doc = \"\" if pd.isna(doc) else str(doc)\n    # Often transliterations are line-based with \\n\n    parts = re.split(r\"\\n+\", doc)\n    parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    # fallback: if no newlines, split on \" ; \" or long spaces\n    if len(parts) <= 1:\n        parts = re.split(r\"\\s{2,}|;\\s+\", clean_ws(doc))\n        parts = [clean_ws(p) for p in parts if clean_ws(p)]\n    return parts\n\ndef split_tgt_english(doc: str):\n    \"\"\"Split English translation into sentences (simple regex).\"\"\"\n    doc = clean_ws(doc)\n    if not doc:\n        return []\n    # Keep punctuation as boundary\n    sents = re.split(r\"(?<=[.!?])\\s+\", doc)\n    sents = [clean_ws(s) for s in sents if clean_ws(s)]\n    return sents\n\ndef tok_len(s: str) -> int:\n    # length proxy for alignment; byte-level later anyway\n    return max(1, len(s.split()))\n\ndef align_monotone(src_units, tgt_units, max_src_merge=2, max_tgt_merge=2):\n    \"\"\"\n    Monotone greedy alignment by matching length proportions.\n    Produces pairs (src_text, tgt_text), allowing merges 1-2 units.\n    \"\"\"\n    i = j = 0\n    pairs = []\n    while i < len(src_units) and j < len(tgt_units):\n        best = None\n        # try combinations: 1..max merge\n        for si in range(1, max_src_merge+1):\n            for tj in range(1, max_tgt_merge+1):\n                if i+si <= len(src_units) and j+tj <= len(tgt_units):\n                    src_chunk = \" \".join(src_units[i:i+si])\n                    tgt_chunk = \" \".join(tgt_units[j:j+tj])\n                    # objective: minimize normalized length mismatch\n                    ls, lt = tok_len(src_chunk), tok_len(tgt_chunk)\n                    score = abs(math.log(ls/lt))\n                    if (best is None) or (score < best[0]):\n                        best = (score, si, tj, src_chunk, tgt_chunk)\n        if best is None:\n            break\n        _, si, tj, s_chunk, t_chunk = best\n        pairs.append((s_chunk, t_chunk))\n        i += si\n        j += tj\n\n    # If leftovers exist, attach them to last pair (simple & safe)\n    if pairs:\n        if i < len(src_units):\n            pairs[-1] = (pairs[-1][0] + \" \" + \" \".join(src_units[i:]), pairs[-1][1])\n        if j < len(tgt_units):\n            pairs[-1] = (pairs[-1][0], pairs[-1][1] + \" \" + \" \".join(tgt_units[j:]))\n\n    return pairs\n\npairs = []\nfor _, row in train_df.iterrows():\n    src_doc = row[\"transliteration\"]\n    tgt_doc = row[\"translation\"]\n\n    src_lines = split_src_transliteration(src_doc)\n    tgt_sents = split_tgt_english(tgt_doc)\n\n    if len(src_lines) == 0 or len(tgt_sents) == 0:\n        continue\n\n    doc_pairs = align_monotone(src_lines, tgt_sents, max_src_merge=2, max_tgt_merge=2)\n    pairs.extend(doc_pairs)\n\npairs_df = pd.DataFrame(pairs, columns=[\"src\", \"tgt\"])\npairs_df = pairs_df[(pairs_df[\"src\"].str.len() > 0) & (pairs_df[\"tgt\"].str.len() > 0)].drop_duplicates()\n\nprint(\"Sentence-level pairs:\", pairs_df.shape)\npairs_df.head()\n\n\ndef load_model_and_tokenizer():\n    last_err = None\n    for p in MODEL_PATH_CANDIDATES:\n        try:\n            tok = AutoTokenizer.from_pretrained(p)\n            mdl = AutoModelForSeq2SeqLM.from_pretrained(p)\n            return tok, mdl, p\n        except Exception as e:\n            last_err = e\n    raise RuntimeError(f\"Could not load any model path. Last error: {last_err}\")\n\ntokenizer, model, used_path = load_model_and_tokenizer()\nprint(\"Loaded:\", used_path)\n\nmodel.to(DEVICE)\n\n\nMAX_SRC = 256\nMAX_TGT = 128\n\nclass MTDataset(Dataset):\n    def __init__(self, df):\n        self.src = df[\"src\"].tolist()\n        self.tgt = df[\"tgt\"].tolist()\n\n    def __len__(self):\n        return len(self.src)\n\n    def __getitem__(self, idx):\n        x = tokenizer(\n            self.src[idx],\n            max_length=MAX_SRC,\n            truncation=True,\n        )\n        y = tokenizer(\n            text_target=self.tgt[idx],\n            max_length=MAX_TGT,\n            truncation=True,\n        )\n        x[\"labels\"] = y[\"input_ids\"]\n        return x\n    \ndef preprocess(batch):\n    x = tokenizer(\n        batch[\"src\"],\n        max_length=MAX_SRC,\n        truncation=True,\n    )\n    y = tokenizer(\n        text_target=batch[\"tgt\"],\n        max_length=MAX_TGT,\n        truncation=True,\n    )\n    x[\"labels\"] = y[\"input_ids\"]\n    return x\n\n\n# simple split\nval_frac = 0.08\npairs_df = pairs_df.sample(frac=1.0, random_state=SEED).reset_index(drop=True)\nn_val = max(1, int(len(pairs_df) * val_frac))\nval_df = pairs_df.iloc[:n_val].reset_index(drop=True)\ntrn_df = pairs_df.iloc[n_val:].reset_index(drop=True)\n\ntrain_ds = MTDataset(trn_df)\nval_ds   = MTDataset(val_df)\n\ndef collate_fn(features):\n    # map preprocess then pad\n    proc = [preprocess(f) for f in features]\n    return data_collator(proc)\n\ndata_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)\n\nargs = Seq2SeqTrainingArguments(\n    output_dir=\"byt5_deeppast\",\n    eval_strategy=\"steps\",\n    eval_steps=100,\n    save_steps=400,\n    logging_steps=50,\n    learning_rate=3e-4,\n    per_device_train_batch_size=8 if DEVICE==\"cuda\" else 2,\n    per_device_eval_batch_size=8 if DEVICE==\"cuda\" else 2,\n    gradient_accumulation_steps=2 if DEVICE==\"cuda\" else 1,\n    num_train_epochs=30,          # start small; increase later (4-6)\n    predict_with_generate=True, \n    fp16=(DEVICE==\"cuda\"),\n    save_total_limit=2,\n    report_to=\"none\",\n    remove_unused_columns=False, \n)\n\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=args,\n    train_dataset=train_ds,\n    eval_dataset=val_ds,\n    data_collator=data_collator,\n)\ntrainer.train()\n\n\ndef make_one_sentence(en: str) -> str:\n    en = clean_ws(en)\n    if not en:\n        return \"...\"\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [p.strip() for p in parts if p.strip()]\n    # remove ending punct to avoid multiple sentences, then join\n    parts = [re.sub(r\"[.!?]+\\s*$\", \"\", p).strip() for p in parts]\n    parts = [p for p in parts if p]\n    out = \"; \".join(parts).strip()\n    if not out.endswith(\".\"):\n        out += \".\"\n    return out\n\nmodel.eval()\n\ndef generate_translation(src_text: str):\n    enc = tokenizer(\n        clean_ws(src_text),\n        return_tensors=\"pt\",\n        max_length=MAX_SRC,\n        truncation=True,\n    ).to(DEVICE)\n\n    with torch.no_grad():\n        out_ids = model.generate(\n            **enc,\n            num_beams=4,\n            max_new_tokens=MAX_TGT,\n            length_penalty=1.0,\n            early_stopping=True,\n        )\n    pred = tokenizer.decode(out_ids[0], skip_special_tokens=True)\n    return make_one_sentence(pred)\n\npreds = [generate_translation(t) for t in test_df[\"transliteration\"].tolist()]\n\nsub = pd.DataFrame({\"id\": test_df[\"id\"], \"translation\": preds})\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"Wrote submission.csv\")\nsub.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T10:41:44.117489Z","iopub.execute_input":"2025-12-18T10:41:44.117712Z","iopub.status.idle":"2025-12-18T11:47:54.197113Z","shell.execute_reply.started":"2025-12-18T10:41:44.117690Z","shell.execute_reply":"2025-12-18T11:47:54.196285Z"}},"outputs":[],"execution_count":null}]}