{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inference ByT5: Translation Akkadian to English\n\n#### This is the final notebook in the following succession of notebooks:\n\n### 1. [Clean Oracc & CDLI Corpus for ByT5 Span Corruption](https://www.kaggle.com/code/anthonytoffetti/clean-oracc-cdli-corpus-byt5-format)\n\n### 2. [Pre-train ByT5 Span Corruption](https://www.kaggle.com/code/anthonytoffetti/pre-train-byt5-span-corruption)\n\n### 3. [Clean Deep Past Data: For Translation](https://www.kaggle.com/code/anthonytoffetti/clean-deep-past-data-for-translation)\n\n### 4. [ByT5 Fine Tune: Akkadian to English](https://www.kaggle.com/code/anthonytoffetti/byt5-fine-tune-akkadian-to-english)","metadata":{}},{"cell_type":"code","source":"\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')\n\ndf_test = pd.read_csv(\"/kaggle/input/competitions/deep-past-initiative-machine-translation/test.csv\")\ndf_test.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:47:56.224071Z","iopub.execute_input":"2026-06-09T05:47:56.224421Z","iopub.status.idle":"2026-06-09T05:47:56.898029Z","shell.execute_reply.started":"2026-06-09T05:47:56.224392Z","shell.execute_reply":"2026-06-09T05:47:56.897116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef fine_tune_akkadian_cleaner(text):\n    if not isinstance(text, str) or text.strip() == \"\":\n        return \"\"\n    \n    # -------------------------------------------------------------\n    # 1. REMOVE scribal notations & PRESERVE required string shapes\n    # -------------------------------------------------------------\n    text = re.sub(r'<(.*?)>', r'\\1', text)\n    text = re.sub(r'\\[([a-zA-Z0-9_.\\- {d}]+)\\]', r'\\1', text)\n    text = text.replace('˹', '').replace('˺', '').replace('⸢', '').replace('⸣', '')\n    text = text.replace('!', '').replace('?', '')\n    \n    # -------------------------------------------------------------\n    # 2. CDLI / ORACC / UNICODE Character Normalizations (MOVED UP)\n    # -------------------------------------------------------------\n    char_mappings = {\n        'á': 'a2', 'à': 'a3', 'é': 'e2', 'è': 'e3', 'í': 'i2', 'ì': 'i3', 'ú': 'u2', 'ù': 'u3',\n        'sz': 'š', 'SZ': 'Š', 's,': 'ṣ', 'S,': 'Ṣ', 't,': 'ṭ', 'T,': 'Ṭ', '‘': 'ʾ'\n    }\n    for old, new in char_mappings.items():\n        text = text.replace(old, new)\n        \n    text = text.replace('ḫ', 'h').replace('Ḫ', 'H')\n\n    # -------------------------------------------------------------\n    # 3. REPLACE breaks, gaps, superscripts, subscripts\n    # -------------------------------------------------------------\n    text = re.sub(r'\\[x\\]', '<gap>', text)\n    text = text.replace('…', '<big_gap>')\n    text = re.sub(r'\\[\\s*\\.\\.\\.\\s*\\.\\.\\.\\s*\\]', '<big_gap>', text)\n    text = text.replace('[...]', '<big_gap>').replace('...', '<big_gap>')\n    \n    subscript_digits = {'0':'₀', '1':'₁', '2':'₂', '3':'₃', '4':'₄', '5':'₅', '6':'₆', '7':'₇', '8':'₈', '9':'₉'}\n    \n    def to_subscript(match):\n        word_part = match.group(1)\n        number_part = match.group(2)\n        subscript_part = \"\".join(subscript_digits[d] for d in number_part)\n        return f\"{word_part}{subscript_part}\"\n    \n    # This now catches qa2, qe2, and the newly generated a2, e2, u3 perfectly!\n    text = re.sub(r'([a-zA-ZšŠṣṢṭṬḫḪʾ]+)(\\d+)', to_subscript, text)\n\n    # -------------------------------------------------------------\n    # 4. AKKADIAN DETERMINATIVES (Strip curly braces {ki} -> ki)\n    # -------------------------------------------------------------\n    determinatives = [\n        'd', 'mul', 'ki', 'lu₂', 'e₂', 'É', 'uru', 'kur', 'mi', 'munus', 'm', \n        'geš', 'ĝeš', 'GIŠ', 'tug₂', 'TÚG', 'dub', 'DUB', 'id₂', 'ÍD', 'mušen', \n        'MUŠEN', 'na₄', 'na4', 'kuš', 'u₂', 'Ú'\n    ]\n    for det in determinatives:\n        pattern = r'\\{' + re.escape(det) + r'\\}'\n        text = re.sub(pattern, det, text)\n        \n    text = re.sub(r'\\{([a-zA-Z0-9_₂]+)\\}', r'\\1', text)\n\n    # -------------------------------------------------------------\n    # 5. LINE & WORD DIVIDERS / STRUCTURAL CLEANUP\n    # -------------------------------------------------------------\n    text = re.sub(r'[/:.]', ' ', text)\n    text = re.sub(r'-+', '-', text)\n    text = re.sub(r'(^| )-|-( |$)', ' ', text)\n    \n    return re.sub(r'\\s+', ' ', text).strip()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:47:56.900369Z","iopub.execute_input":"2026-06-09T05:47:56.900736Z","iopub.status.idle":"2026-06-09T05:47:56.911638Z","shell.execute_reply.started":"2026-06-09T05:47:56.900710Z","shell.execute_reply":"2026-06-09T05:47:56.910648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import datasets\n\ndef process_corpus_for_finetuning(df, src_col='transliteration'):\n    \n    data = df.copy()\n    \n    # 1. Clean the Akkadian Source text using the non-destructive cleaner\n    data[src_col] = data[src_col].apply(fine_tune_akkadian_cleaner)\n    \n    \n    return data\n\ndf_test = process_corpus_for_finetuning(df_test)\ndf_test['transliteration'][0]\nhf_dataset = datasets.Dataset.from_pandas(df_test)\n\nprint(hf_dataset)\n\nprint(hf_dataset['transliteration'][0])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:47:56.912518Z","iopub.execute_input":"2026-06-09T05:47:56.912789Z","iopub.status.idle":"2026-06-09T05:47:57.656197Z","shell.execute_reply.started":"2026-06-09T05:47:56.912764Z","shell.execute_reply":"2026-06-09T05:47:57.655259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\nfrom peft import PeftModel\n\n\n# official_base = AutoModelForSeq2SeqLM.from_pretrained(\"/kaggle/input/datasets/anthonytoffetti/byt5-pre-trained-oracccdli/checkpoint-1353\")\nofficial_base = AutoModelForSeq2SeqLM.from_pretrained(\"/kaggle/input/datasets/anthonytoffetti/byt5-pre-trained-oracccdli/checkpoint-1353\")\ntokenizer = AutoTokenizer.from_pretrained(\"/kaggle/input/datasets/anthonytoffetti/byt5-pre-trained-oracccdli/checkpoint-1353\")\n\n# 2. Pull your local Kaggle adapter checkpoint dataset folder\nADAPTER_PATH = \"/kaggle/input/datasets/anthonytoffetti/byt5-finetuned-akkadian-english/kaggle/working/byt5_finetuned\"\nprint(\"Overlaying your fine-tuned LoRA adapters...\")\nlora_model = PeftModel.from_pretrained(official_base, ADAPTER_PATH)\n\n# 3. Permanently fuse the weights together into a standard model structure\nprint(\"Baking adapter weights directly into the base model parameters...\")\nmodel = lora_model.merge_and_unload()\n\n# 4. Save everything to /kaggle/working (This is local notebook storage)\nOUTPUT_DIR = \"/kaggle/working/fused_akkadian_byt5_base\"\nmodel.save_pretrained(OUTPUT_DIR)\ntokenizer.save_pretrained(OUTPUT_DIR)\n\nprint(f\"🚀 SUCCESS! Standalone un-wrapped model is compiled at: {OUTPUT_DIR}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:47:57.658040Z","iopub.execute_input":"2026-06-09T05:47:57.659151Z","iopub.status.idle":"2026-06-09T05:48:15.938415Z","shell.execute_reply.started":"2026-06-09T05:47:57.659111Z","shell.execute_reply":"2026-06-09T05:48:15.937578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tokenize_for_byt5(examples):\n    # Add the prefix to the Akkadian text column\n    prefix = \"translate Akkadian to English: \"\n    inputs = [prefix + str(text) for text in examples[\"transliteration\"]]\n    \n    # Tokenize Source (Akkadian bytes)\n    model_inputs = tokenizer(\n        inputs, \n        max_length=512, \n        truncation=True, \n        padding='max_length'\n    )\n    \n    \n    return model_inputs\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:48:15.939617Z","iopub.execute_input":"2026-06-09T05:48:15.940300Z","iopub.status.idle":"2026-06-09T05:48:15.944725Z","shell.execute_reply.started":"2026-06-09T05:48:15.940271Z","shell.execute_reply":"2026-06-09T05:48:15.944061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tokenized_test_data = hf_dataset.map(tokenize_for_byt5, batched=True)\nprint(tokenized_test_data)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:48:15.945845Z","iopub.execute_input":"2026-06-09T05:48:15.946309Z","iopub.status.idle":"2026-06-09T05:48:16.017253Z","shell.execute_reply.started":"2026-06-09T05:48:15.946272Z","shell.execute_reply":"2026-06-09T05:48:16.016383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport torch\nimport gc\nimport re\nfrom collections import Counter\nfrom tqdm import tqdm\nimport os\nfrom transformers import GenerationConfig\n\ngc.collect()\ntorch.cuda.empty_cache()\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel.to(device)\n\n\n\n\n# 1. Lock down strict offline mode to prevent cloud lookup crashes\nos.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\nos.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\n\n# 2. Strict Overfitting Local Beam Search Config\nmodel.generation_config = GenerationConfig(\n    max_length=256,\n    \n    # CRITICAL TRIGGER: Turn off sampling, turn on strict beam tracking\n    do_sample=False,             # Disables creativity; forces strict data matching\n    num_beams=4,                 # Tracks the top 4 most confident character paths\n    \n    # Strict anti-loop penalties to replace your MBR filters\n    repetition_penalty=1.55,     # Heavily suppresses \"the City\" word stacking\n    no_repeat_ngram_size=25,     # Hard character-phrase safety barrier\n    length_penalty=0.6,          # Forces clean sentence endings\n    early_stopping=True,\n    \n    decoder_start_token_id=tokenizer.pad_token_id,\n    eos_token_id=tokenizer.eos_token_id,\n    pad_token_id=tokenizer.pad_token_id,\n    local_files_only=True        # Bypasses the cloud completely\n)\n\n# 3. Force properties onto the GPU device\n\nif hasattr(model, 'generation_config') and model.generation_config is not None:\n    for attr in ['bos_token_id', 'eos_token_id', 'pad_token_id', 'decoder_start_token_id']:\n        val = getattr(model.generation_config, attr, None)\n        if isinstance(val, torch.Tensor):\n            setattr(model.generation_config, attr, val.to(device))\n\nprint(\"🔒 Strict Matching Beam Search configuration locked down!\")\n\nmodel.eval()\n\n\ndef fast_mbr_utility(candidate, pool):\n    cand_chars = list(candidate)\n    if not cand_chars: return 0\n    total_intersection = 0\n    cand_counter = Counter(cand_chars)\n    for ref in pool:\n        ref_counter = Counter(list(ref))\n        intersection = sum((cand_counter & ref_counter).values())\n        total_intersection += intersection / len(cand_chars)\n    return total_intersection / len(pool)\n\ndef clean_duplicated_words(text):\n    return re.sub(re.compile(r'\\b(\\w+)\\s+\\1\\b', re.IGNORECASE), r'\\1', text)\n\ndef remove_sentence_level_loops(text):\n    sentences = re.split(r'(?<=[.!?])\\s+', text.strip())\n    cleaned_sentences = []\n    for sentence in sentences:\n        normalized = re.sub(r'[^\\w\\s]', '', sentence).lower().strip()\n        if cleaned_sentences:\n            prev_normalized = re.sub(r'[^\\w\\s]', '', cleaned_sentences[-1]).lower().strip()\n            if normalized == prev_normalized: continue\n        cleaned_sentences.append(sentence)\n    return \" \".join(cleaned_sentences)\n\n# 2. Optimized Batch MBR Loop\nfinal_predictions = []\nnum_rows = len(tokenized_test_data)\n\n# A safe batch size of 4 maximizes GPU parallelism without risking a padding OOM crash\nBATCH_SIZE = 4 \n\nprint(f\"🚀 Running fast BATCH MBR loop over {num_rows} pre-tokenized rows...\")\nwith torch.no_grad():\n    for i in tqdm(range(0, num_rows, BATCH_SIZE)):\n        # Extract a mini-batch of rows\n        batch_slice = tokenized_test_data[i : i + BATCH_SIZE]\n        \n        # Extract features and dynamically pad them to the batch max length\n        input_ids_list = [torch.tensor(row) for row in batch_slice[\"input_ids\"]]\n        attention_mask_list = [torch.tensor(row) for row in batch_slice[\"attention_mask\"]]\n        \n        inputs = {\n            \"input_ids\": torch.nn.utils.rnn.pad_sequence(input_ids_list, batch_first=True, padding_value=tokenizer.pad_token_id).to(device),\n            \"attention_mask\": torch.nn.utils.rnn.pad_sequence(attention_mask_list, batch_first=True, padding_value=0).to(device)\n        }\n        \n        current_batch_len = len(input_ids_list)\n        \n        # Generate 8 variations for the entire batch in a single GPU operation!\n        outputs = model.generate(\n            **inputs,\n            generation_config=model.generation_config\n        )\n        \n        # 3. Decode the exact batch array output directly (1 output per row)\n        candidates = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        \n        # 4. Clean and append directly with an absolute fallback safeguard\n        for cand in candidates:\n            clean = \" \".join(cand.strip().split())\n            clean = clean_duplicated_words(clean)       # Cleans \"Kanesh Kanesh\"\n            clean = remove_sentence_level_loops(clean)  # Cleans sentence stutters\n            \n            # CRITICAL FIX: If the prediction is completely empty, \n            # provide a generic baseline token so Kaggle doesn't get a blank row\n            if not clean.strip():\n                clean = \"The tablet from the City.\"\n                \n            final_predictions.append(clean)\n\n# 5. Compile the exact submission framework\nsubmission_df = pd.DataFrame({\n    \"id\": tokenized_test_data[\"id\"], \n    \"translation\": final_predictions\n})\n\nsubmission_df.to_csv(\"submission.csv\", index=False)\nprint(f\"🏁 SUCCESS! Batch MBR 'submission.csv' compiled with {len(submission_df)} rows.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:48:16.018306Z","iopub.execute_input":"2026-06-09T05:48:16.018626Z","iopub.status.idle":"2026-06-09T05:48:27.179413Z","shell.execute_reply.started":"2026-06-09T05:48:16.018589Z","shell.execute_reply":"2026-06-09T05:48:27.178668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer, GenerationConfig\n\n# device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n# model.to(device)\n# model.eval()\n\n# anti_loop_config = GenerationConfig(\n#     max_length=256,              \n#     num_beams=4,                 # Standard beam search to look ahead\n    \n#     # CRITICAL BYT5 CHARACTER-LEVEL FIXES:\n#     # A 25-character window blocks phrases like \", son of Kaniya\" from repeating,\n#     # but allows normal short Akkadian/English words to generate cleanly.\n#     no_repeat_ngram_size=25,     \n#     repetition_penalty=1.4,      # Strong discouragement of any repetitive sequence\n#     length_penalty=0.6,          # Encourages shorter, more concise structural text\n    \n#     decoder_start_token_id=tokenizer.pad_token_id, \n#     eos_token_id=tokenizer.eos_token_id,\n#     pad_token_id=tokenizer.pad_token_id,\n#     early_stopping=True \n# )\n\n# model.generation_config = anti_loop_config\n\n# # 5. Define minimal Arguments required for Inference\n# training_args = Seq2SeqTrainingArguments(\n#     output_dir=\"./inference_results\",\n#     per_device_eval_batch_size=32,      # Adjust based on GPU memory\n#     predict_with_generate=True,         # Enables generation instead of raw loss calculation\n#     generation_max_length=512,          # Match your target max length\n#     generation_num_beams=4,             # Beam search settings\n#     fp16=torch.cuda.is_available(),     # Speed up inference if supported by GPU\n#     report_to=\"none\"                    # Disable logging overhead during prediction\n# )\n\n# # 6. Initialize the Seq2SeqTrainer\n# trainer = Seq2SeqTrainer(\n#     model=model,\n#     args=training_args,\n#     processing_class=tokenizer,\n#     eval_dataset=tokenized_test_data\n# )\n# # 7. Execute Predict Loop\n# print(\"Running prediction via HF Trainer...\")\n# predictions = trainer.predict(tokenized_test_data)\n\n# # 8. Decode the prediction tokens back to clean text\n# # trainer.predict outputs a named tuple; predictions.predictions holds the generated token IDs\n# decoded_predictions = tokenizer.batch_decode(predictions.predictions, skip_special_tokens=True)\n# clean_predictions = [\" \".join(pred.strip().split()) for pred in decoded_predictions]\n# clean_predictions = [clean_duplicated_words(pred) for pred in clean_predictions] # From previous step\n# final_kaggle_predictions = [remove_sentence_level_loops(pred) for pred in clean_predictions]\n\n\n# print(f\"🚀 Success! Successfully generated {len(final_kaggle_predictions)} translations.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:48:27.180298Z","iopub.execute_input":"2026-06-09T05:48:27.181117Z","iopub.status.idle":"2026-06-09T05:48:27.185763Z","shell.execute_reply.started":"2026-06-09T05:48:27.181087Z","shell.execute_reply":"2026-06-09T05:48:27.185050Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(f\"🚀 Successfully generated {len(final_kaggle_predictions)} translations!\")\n\n# submission_df = pd.DataFrame(\n#     {\"id\": df_test[\"id\"].values, \"translation\": final_kaggle_predictions}\n# )\n\n# # 2. DEFENSIVE CHECK: Replace any empty strings or failed generations with a placeholder\n# # (Kaggle will reject empty strings. A simple dot \".\" or \"the\" preserves row structure)\n# submission_df[\"translation\"] = submission_df[\"translation\"].fillna(\".\")\n# submission_df[\"translation\"] = submission_df[\"translation\"].apply(lambda x: \".\" if str(x).strip() == \"\" else x)\n\n# # 3. FORCE DATA TYPES\n# submission_df[\"id\"] = submission_df[\"id\"].astype(str) # Adjust to int if the competition ids are pure integers\n# submission_df[\"translation\"] = submission_df[\"translation\"].astype(str)\n\n# # 4. SAVE FILE CLEANLY\n# submission_df.to_csv(\"submission.csv\", index=False)\n# print(\"Defensive submission file generated successfully!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:48:27.187494Z","iopub.execute_input":"2026-06-09T05:48:27.188293Z","iopub.status.idle":"2026-06-09T05:48:27.202260Z","shell.execute_reply.started":"2026-06-09T05:48:27.188258Z","shell.execute_reply":"2026-06-09T05:48:27.201445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-09T05:48:27.203292Z","iopub.execute_input":"2026-06-09T05:48:27.203881Z","iopub.status.idle":"2026-06-09T05:48:27.219766Z","shell.execute_reply.started":"2026-06-09T05:48:27.203851Z","shell.execute_reply":"2026-06-09T05:48:27.219012Z"}},"outputs":[],"execution_count":null}]}