{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ===============================\n# 1) Imports\n# ===============================\nimport os\nimport gc\nimport random\nimport re\nimport unicodedata\n\nimport numpy as np\nimport pandas as pd\nimport torch\n\nfrom datasets import Dataset\nfrom transformers import (\n    ByT5Tokenizer,\n    AutoModelForSeq2SeqLM,\n    DataCollatorForSeq2Seq,\n    Seq2SeqTrainer,\n    Seq2SeqTrainingArguments\n)\n\n# ===============================\n# 2) Paths\n# ===============================\nINPUT_DIR = \"/kaggle/input/competitions/deep-past-initiative-machine-translation\"\nPUBLISHED_PATH = f\"{INPUT_DIR}/published_texts.csv\"\n\nOUTPUT_MODEL_DIR = \"/kaggle/working/byt5_dapt\"\n\n# ===============================\n# 3) Cleaning\n# ===============================\ndef clean_akkadian(text):\n\n    if not isinstance(text, str):\n        return \"\"\n\n    text = unicodedata.normalize(\"NFC\", text)\n\n    text = text.replace(\"…\", \" \")\n\n    text = re.sub(r'\\((\\w)\\)(\\w+)', r'{\\1}\\2', text)\n    text = re.sub(r'\\(.*?\\)', ' ', text)\n\n    text = text.replace(\"[\",\"\").replace(\"]\",\"\")\n\n    text = re.sub(r'<.*?>', '<gap>', text)\n\n    subs = {\n        \"₀\":\"0\",\"₁\":\"1\",\"₂\":\"2\",\"₃\":\"3\",\"₄\":\"4\",\n        \"₅\":\"5\",\"₆\":\"6\",\"₇\":\"7\",\"₈\":\"8\",\"₉\":\"9\"\n    }\n\n    for k,v in subs.items():\n        text = text.replace(k,v)\n\n    text = re.sub(r'[!?/]', '', text)\n    text = \" \".join(text.split())\n\n    return text\n\n# ===============================\n# 4) Load + concatenate corpus\n# ===============================\ndf = pd.read_csv(PUBLISHED_PATH)\ntexts = df[\"transliteration\"].dropna().tolist()\n\ntexts = [clean_akkadian(t) for t in texts if len(t) > 10]\n\n# 🔥 CRITICAL: concatenate into one stream\nfull_text = \" \".join(texts)\n\nwords = full_text.split()\n\n# ===============================\n# 5) Chunk into long sequences\n# ===============================\nCHUNK_SIZE = 512\n\nchunks = [\n    \" \".join(words[i:i+CHUNK_SIZE])\n    for i in range(0, len(words), CHUNK_SIZE)\n    if len(words[i:i+CHUNK_SIZE]) > 20\n]\n\nprint(\"Chunks:\", len(chunks))\n\n# ===============================\n# 6) Proper T5 span masking\n# ===============================\ndef mask_spans(text, mask_ratio=0.15):\n\n    words = text.split()\n    n = len(words)\n\n    if n < 5:\n        return None\n\n    n_to_mask = max(1, int(n * mask_ratio))\n\n    spans = []\n    used = set()\n\n    while n_to_mask > 0:\n        span_len = min(random.randint(1, 3), n_to_mask)\n        start = random.randint(0, n - span_len)\n\n        if any(i in used for i in range(start, start+span_len)):\n            continue\n\n        for i in range(start, start+span_len):\n            used.add(i)\n\n        spans.append((start, start + span_len))\n        n_to_mask -= span_len\n\n    spans = sorted(spans)\n\n    input_tokens = []\n    target_tokens = []\n\n    current_mask = 0\n    i = 0\n\n    while i < n:\n        span = next((s for s in spans if s[0] == i), None)\n\n        if span:\n            sentinel = f\"<extra_id_{current_mask}>\"\n\n            input_tokens.append(sentinel)\n            target_tokens.append(sentinel)\n\n            for j in range(span[0], span[1]):\n                target_tokens.append(words[j])\n\n            i = span[1]\n            current_mask += 1\n        else:\n            input_tokens.append(words[i])\n            i += 1\n\n    target_tokens.append(f\"<extra_id_{current_mask}>\")\n\n    return \" \".join(input_tokens), \" \".join(target_tokens)\n\n# ===============================\n# 7) Build dataset\n# ===============================\nexamples = []\n\nfor text in chunks:\n    res = mask_spans(text)\n    if res is None:\n        continue\n\n    inp, tgt = res\n\n    examples.append({\n        \"input\": inp,\n        \"target\": tgt\n    })\n\nprint(\"Training examples:\", len(examples))\n\ndataset = Dataset.from_pandas(pd.DataFrame(examples))\n\n# ===============================\n# 8) Load model\n# ===============================\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\ntokenizer = ByT5Tokenizer.from_pretrained(\"google/byt5-small\")\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\n    \"google/byt5-small\"\n).to(device)\n\nmodel.gradient_checkpointing_enable()\n\n# ===============================\n# 9) Tokenization\n# ===============================\nMAX_LENGTH = 512\n\ndef preprocess(examples):\n\n    model_inputs = tokenizer(\n        examples[\"input\"],\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n\n    labels = tokenizer(\n        text_target=examples[\"target\"],\n        max_length=MAX_LENGTH,\n        truncation=True\n    )\n\n    model_inputs[\"labels\"] = labels[\"input_ids\"]\n\n    return model_inputs\n\ndataset = dataset.map(preprocess, batched=True)\n\n# ===============================\n# 10) Data collator\n# ===============================\ndata_collator = DataCollatorForSeq2Seq(\n    tokenizer=tokenizer,\n    model=model\n)\n\n# ===============================\n# 11) Training args\n# ===============================\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=OUTPUT_MODEL_DIR,\n    learning_rate=5e-5,\n    per_device_train_batch_size=4,\n    num_train_epochs=3,\n    logging_steps=100,\n    save_strategy=\"epoch\",\n    fp16=torch.cuda.is_available(),\n    report_to=\"none\"\n)\n\n# ===============================\n# 12) Trainer\n# ===============================\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=dataset,\n    data_collator=data_collator\n)\n\n# ===============================\n# 13) Train\n# ===============================\ntrainer.train()\n\n# ===============================\n# 14) Save (SAFE)\n# ===============================\ntrainer.save_model(OUTPUT_MODEL_DIR)\ntokenizer.save_pretrained(OUTPUT_MODEL_DIR)\n\nprint(\"Saved to:\", OUTPUT_MODEL_DIR)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-16T10:24:36.120728Z","iopub.execute_input":"2026-03-16T10:24:36.121375Z","iopub.status.idle":"2026-03-16T11:38:16.835374Z","shell.execute_reply.started":"2026-03-16T10:24:36.121347Z","shell.execute_reply":"2026-03-16T11:38:16.833960Z"}},"outputs":[],"execution_count":null}]}