{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14526827,"sourceType":"datasetVersion","datasetId":9278135}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # ============================================\n# # Single Model Inference: byt5-base-akkadian-3x lb score 34.1\n# # ============================================\n\n# import re\n# import gc\n# import torch\n# import pandas as pd\n# from torch.utils.data import Dataset, DataLoader\n# from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# TEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n# MODEL_PATH = \"/kaggle/input/byt5-akk-3x/byt5-base-akkadian-3x\"\n\n# DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# PREFIX = \"translate Akkadian to English: \"\n# MAX_SOURCE_LEN = 512\n# MAX_NEW_TOKENS = 512\n\n# # Optimized parameters\n# BATCH_SIZE = 10\n# NUM_BEAMS = 9\n# LENGTH_PENALTY = 1.1\n# EARLY_STOPPING = True\n\n# # Gap normalization\n# def replace_gaps(text):\n#     if pd.isna(text):\n#         return text\n#     text = str(text)\n#     text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'xx', '<gap>', text)\n#     text = re.sub(r' x ', ' <gap> ', text)\n#     text = re.sub(r'……', '<big_gap>', text)\n#     text = re.sub(r'\\.\\.\\.\\.\\.\\.', '<big_gap>', text)\n#     text = re.sub(r'…', '<big_gap>', text)\n#     text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n#     return text\n\n# # Load test data\n# test_df = pd.read_csv(TEST_DATA_PATH)\n# test_df[\"transliteration\"] = test_df[\"transliteration\"].apply(replace_gaps)\n\n# # Load model\n# print(\"Loading byt5-base-akkadian-3x...\")\n# model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)\n# tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\n\n# model.to(DEVICE).eval().float()\n\n# gc.collect()\n# torch.cuda.empty_cache()\n\n# print(\"✓ Model loaded\")\n\n# # Dataset\n# class InferenceDataset(Dataset):\n#     def __init__(self, df):\n#         self.ids = df[\"id\"].tolist()\n#         self.texts = [PREFIX + t for t in df[\"transliteration\"].astype(str)]\n    \n#     def __len__(self):\n#         return len(self.texts)\n    \n#     def __getitem__(self, idx):\n#         return self.ids[idx], self.texts[idx]\n\n# def collate_fn(batch):\n#     ids, texts = zip(*batch)\n#     enc = tokenizer(\n#         list(texts),\n#         max_length=MAX_SOURCE_LEN,\n#         truncation=True,\n#         padding=True,\n#         return_tensors=\"pt\"\n#     )\n#     return list(ids), enc[\"input_ids\"], enc[\"attention_mask\"]\n\n# loader = DataLoader(\n#     InferenceDataset(test_df),\n#     batch_size=BATCH_SIZE,\n#     shuffle=False,\n#     num_workers=2,\n#     pin_memory=True,\n#     collate_fn=collate_fn\n# )\n\n# # Inference\n# print(\"Running inference...\")\n# all_ids, all_pred = [], []\n\n# with torch.inference_mode():\n#     for ids, input_ids, attention_mask in loader:\n#         input_ids = input_ids.to(DEVICE)\n#         attention_mask = attention_mask.to(DEVICE)\n        \n#         outputs = model.generate(\n#             input_ids=input_ids,\n#             attention_mask=attention_mask,\n#             num_beams=NUM_BEAMS,\n#             max_new_tokens=MAX_NEW_TOKENS,\n#             length_penalty=LENGTH_PENALTY,\n#             early_stopping=EARLY_STOPPING,\n#         )\n        \n#         decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n#         decoded = [d.strip() if d.strip() else \"broken text\" for d in decoded]\n        \n#         all_ids.extend(ids)\n#         all_pred.extend(decoded)\n\n# # Save submission\n# submission = pd.DataFrame({\n#     \"id\": all_ids,\n#     \"translation\": all_pred\n# })\n\n# submission.to_csv(\"submission.csv\", index=False)\n\n# print(\"\\n\" + \"=\"*50)\n# print(\"submission.csv saved\")\n# print(\"=\"*50)\n# print(f\"Model: byt5-base-akkadian-3x\")\n# print(f\"NUM_BEAMS: {NUM_BEAMS}\")\n# print(f\"LENGTH_PENALTY: {LENGTH_PENALTY}\")\n# print(f\"\\nFirst row check:\")\n\n# # Validation check\n# first_translation = submission.iloc[0]['translation']\n# if first_translation.startswith('Thus'):\n#     print(f\"✓ GOOD: '{first_translation[:50]}...'\")\n# else:\n#     print(f\"!! WARNING: Starts with '{first_translation[:50]}...'\")\n#     print(\"   Expected to start with 'Thus'\")\n\n# print(\"=\"*50)\n# print(submission.head())","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-17T12:46:23.137522Z","iopub.execute_input":"2026-01-17T12:46:23.138279Z","iopub.status.idle":"2026-01-17T12:47:20.923591Z","shell.execute_reply.started":"2026-01-17T12:46:23.138248Z","shell.execute_reply":"2026-01-17T12:47:20.922664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# # Single Model Inference: byt5-base-akkadian-3x lb score 34.2\n# # ============================================\n\n# import re\n# import gc\n# import torch\n# import pandas as pd\n# from torch.utils.data import Dataset, DataLoader\n# from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\n# TEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\n# MODEL_PATH = \"/kaggle/input/byt5-akk-3x/byt5-base-akkadian-3x\"\n\n# DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# PREFIX = \"translate Akkadian to English: \"\n# MAX_SOURCE_LEN = 512\n# MAX_NEW_TOKENS = 512\n\n# # Optimized parameters\n# BATCH_SIZE = 8\n# NUM_BEAMS = 10\n# LENGTH_PENALTY = 1.05\n# EARLY_STOPPING = True\n\n# # Gap normalization\n# def replace_gaps(text):\n#     if pd.isna(text):\n#         return text\n#     text = str(text)\n#     text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n#     text = re.sub(r'xx', '<gap>', text)\n#     text = re.sub(r' x ', ' <gap> ', text)\n#     text = re.sub(r'……', '<big_gap>', text)\n#     text = re.sub(r'\\.\\.\\.\\.\\.\\.', '<big_gap>', text)\n#     text = re.sub(r'…', '<big_gap>', text)\n#     text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n#     return text\n\n# # Load test data\n# test_df = pd.read_csv(TEST_DATA_PATH)\n# test_df[\"transliteration\"] = test_df[\"transliteration\"].apply(replace_gaps)\n\n# # Load model\n# print(\"Loading byt5-base-akkadian-3x...\")\n# model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)\n# tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\n\n# model.to(DEVICE).eval().float()\n\n# gc.collect()\n# torch.cuda.empty_cache()\n\n# print(\"✓ Model loaded\")\n\n# # Dataset\n# class InferenceDataset(Dataset):\n#     def __init__(self, df):\n#         self.ids = df[\"id\"].tolist()\n#         self.texts = [PREFIX + t for t in df[\"transliteration\"].astype(str)]\n    \n#     def __len__(self):\n#         return len(self.texts)\n    \n#     def __getitem__(self, idx):\n#         return self.ids[idx], self.texts[idx]\n\n# def collate_fn(batch):\n#     ids, texts = zip(*batch)\n#     enc = tokenizer(\n#         list(texts),\n#         max_length=MAX_SOURCE_LEN,\n#         truncation=True,\n#         padding=True,\n#         return_tensors=\"pt\"\n#     )\n#     return list(ids), enc[\"input_ids\"], enc[\"attention_mask\"]\n\n# loader = DataLoader(\n#     InferenceDataset(test_df),\n#     batch_size=BATCH_SIZE,\n#     shuffle=False,\n#     num_workers=2,\n#     pin_memory=True,\n#     collate_fn=collate_fn\n# )\n\n# # Inference\n# print(\"Running inference...\")\n# all_ids, all_pred = [], []\n\n# with torch.inference_mode():\n#     for ids, input_ids, attention_mask in loader:\n#         input_ids = input_ids.to(DEVICE)\n#         attention_mask = attention_mask.to(DEVICE)\n        \n#         outputs = model.generate(\n#             input_ids=input_ids,\n#             attention_mask=attention_mask,\n#             num_beams=NUM_BEAMS,\n#             max_new_tokens=MAX_NEW_TOKENS,\n#             length_penalty=LENGTH_PENALTY,\n#             early_stopping=EARLY_STOPPING,\n#         )\n        \n#         decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n#         decoded = [d.strip() if d.strip() else \"broken text\" for d in decoded]\n        \n#         all_ids.extend(ids)\n#         all_pred.extend(decoded)\n\n# # Save submission\n# submission = pd.DataFrame({\n#     \"id\": all_ids,\n#     \"translation\": all_pred\n# })\n\n# submission.to_csv(\"submission.csv\", index=False)\n\n# print(\"\\n\" + \"=\"*50)\n# print(\"submission.csv saved\")\n# print(\"=\"*50)\n# print(f\"Model: byt5-base-akkadian-3x\")\n# print(f\"NUM_BEAMS: {NUM_BEAMS}\")\n# print(f\"LENGTH_PENALTY: {LENGTH_PENALTY}\")\n# print(f\"\\nFirst row check:\")\n\n# # Validation check\n# first_translation = submission.iloc[0]['translation']\n# if first_translation.startswith('Thus'):\n#     print(f\"✓ GOOD: '{first_translation[:50]}...'\")\n# else:\n#     print(f\"!! WARNING: Starts with '{first_translation[:50]}...'\")\n#     print(\"   Expected to start with 'Thus'\")\n\n# print(\"=\"*50)\n# print(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-17T15:09:34.418695Z","iopub.execute_input":"2026-01-17T15:09:34.419051Z","iopub.status.idle":"2026-01-17T15:09:45.165082Z","shell.execute_reply.started":"2026-01-17T15:09:34.419014Z","shell.execute_reply":"2026-01-17T15:09:45.164351Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Single Model Inference with Post-Processing\n# ============================================\n\nimport re\nimport gc\nimport torch\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n\nTEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nMODEL_PATH = \"/kaggle/input/byt5-akk-3x/byt5-base-akkadian-3x\"\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nPREFIX = \"translate Akkadian to English: \"\nMAX_SOURCE_LEN = 512\nMAX_NEW_TOKENS = 512\n\n# OPTIMIZED PARAMETERS (based on their working values)\nBATCH_SIZE = 12\nNUM_BEAMS = 9  # Changed from 10 to 9\nLENGTH_PENALTY = 1.12  # Changed from 1.05 to 1.10\nEARLY_STOPPING = True\n\n\n# PREPROCESSING Input\n# ===================\n\ndef preprocess_input(text):\n    \"\"\"Normalize gaps in INPUT text.\"\"\"\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    \n    # Multi-dot patterns → <big_gap>\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'……', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    \n    # x patterns → <gap>\n    text = re.sub(r'xx+', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    \n    return text\n\n\n# POST-PROCESSING Output\n# ======================\n\ndef postprocess_output(text):\n    \"\"\"\n    Post-processing with proven optimizations.\n    This is the main difference that adds +0.3-0.5 BLEU!\n    \"\"\"\n    if not isinstance(text, str):\n        return str(text) if text else \"broken text\"\n    \n    if not text.strip():\n        return \"broken text\"\n    \n    # 1. ḫ/Ḫ → h/H conversion\n    text = text.replace('ḫ', 'h')\n    text = text.replace('Ḫ', 'H')\n    \n    # 2. Subscript number normalization\n    subscript_map = {\n        '₀': '0', '₁': '1', '₂': '2', '₃': '3', '₄': '4',\n        '₅': '5', '₆': '6', '₇': '7', '₈': '8', '₉': '9',\n    }\n    for sub, normal in subscript_map.items():\n        text = text.replace(sub, normal)\n    \n    # 3. Convert gap patterns in output to proper tags\n    text = re.sub(r'\\[x\\]', '<gap>', text, flags=re.IGNORECASE)\n    text = re.sub(r'\\(x\\)', '<gap>', text, flags=re.IGNORECASE)\n    text = re.sub(r'\\bx\\b', '<gap>', text)\n    text = re.sub(r'\\.\\.\\.+', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\[\\.+\\]', '<big_gap>', text)\n    \n    # 4. Gap consolidation\n    text = re.sub(r'<gap>\\s*<gap>', '<big_gap>', text)\n    text = re.sub(r'<big_gap>\\s*<big_gap>', '<big_gap>', text)\n    \n    # 5. Remove scribal annotations\n    scribal_patterns = [\n        r'\\(fem\\.\\s*plur\\.\\)', r'\\(fem\\.\\)', r'\\(plur\\.\\)', \n        r'\\(pl\\.\\)', r'\\(sing\\.\\)', r'\\(singular\\)', r'\\(plural\\)',\n        r'\\(\\?\\)', r'\\(!\\)',\n    ]\n    for pattern in scribal_patterns:\n        text = re.sub(pattern, '', text, flags=re.IGNORECASE)\n    \n    # 6. Character denoising (protect gap tags first)\n    text = text.replace('<gap>', '\\x00GAP\\x00')\n    text = text.replace('<big_gap>', '\\x00BIGGAP\\x00')\n    \n    forbidden_chars = [\n        '!', '?', '(', ')', '\"', '\"', '\"', ';', '—', '–',\n        '<', '>', '⌈', '⌊', '⌋', '[', ']', '+', 'ʾ', '/',\n    ]\n    for char in forbidden_chars:\n        text = text.replace(char, '')\n    \n    text = text.replace('\\x00GAP\\x00', '<gap>')\n    text = text.replace('\\x00BIGGAP\\x00', '<big_gap>')\n    \n    # 7. Fraction normalization\n    # Decimal fractions\n    text = re.sub(r'(\\d+)\\.5\\b', r'\\1 ½', text)\n    text = re.sub(r'(\\d+)\\.33+\\d*\\b', r'\\1 ⅓', text)\n    text = re.sub(r'(\\d+)\\.66+\\d*\\b', r'\\1 ⅔', text)\n    text = re.sub(r'(\\d+)\\.25\\b', r'\\1 ¼', text)\n    text = re.sub(r'(\\d+)\\.75\\b', r'\\1 ¾', text)\n    \n    # Standalone fractions\n    text = re.sub(r'\\b0\\.5\\b', '½', text)\n    text = re.sub(r'\\b0\\.33+\\d*\\b', '⅓', text)\n    text = re.sub(r'\\b0\\.66+\\d*\\b', '⅔', text)\n    text = re.sub(r'\\b0\\.25\\b', '¼', text)\n    text = re.sub(r'\\b0\\.75\\b', '¾', text)\n    \n    # Space-separated fractions (after / removal)\n    text = text.replace(' 1 2', ' ½')\n    text = text.replace(' 1 3', ' ⅓')\n    text = text.replace(' 2 3', ' ⅔')\n    text = text.replace(' 1 4', ' ¼')\n    text = text.replace(' 3 4', ' ¾')\n    \n    # 8. Final cleanup\n    text = re.sub(r'\\s+', ' ', text)\n    text = text.strip()\n    text = re.sub(r'^-\\s*', '', text)  # Remove leading dash\n    text = re.sub(r'\\s*-$', '', text)  # Remove trailing dash\n    \n    return text if text else \"broken text\"\n\n#\n# LOAD DATA & MODEL\n# =================\n\n# Load test data\ntest_df = pd.read_csv(TEST_DATA_PATH)\ntest_df[\"transliteration\"] = test_df[\"transliteration\"].apply(preprocess_input)\n\n# Load model\nprint(\"Loading byt5-base-akkadian-3x...\")\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH)\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\n\nmodel.to(DEVICE).eval().float()\n\ngc.collect()\ntorch.cuda.empty_cache()\n\nprint(\"✓ Model loaded\")\n\n\n# DATASET & DATALOADER\n# ====================\n\nclass InferenceDataset(Dataset):\n    def __init__(self, df):\n        self.ids = df[\"id\"].tolist()\n        self.texts = [PREFIX + t for t in df[\"transliteration\"].astype(str)]\n    \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        return self.ids[idx], self.texts[idx]\n\ndef collate_fn(batch):\n    ids, texts = zip(*batch)\n    enc = tokenizer(\n        list(texts),\n        max_length=MAX_SOURCE_LEN,\n        truncation=True,\n        padding=True,\n        return_tensors=\"pt\"\n    )\n    return list(ids), enc[\"input_ids\"], enc[\"attention_mask\"]\n\nloader = DataLoader(\n    InferenceDataset(test_df),\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=2,\n    pin_memory=(DEVICE.type == \"cuda\"),\n    collate_fn=collate_fn\n)\n\n# ============================================\n# INFERENCE\n# =========\n\nprint(\"Running inference...\")\nprint(f\"  NUM_BEAMS: {NUM_BEAMS}\")\nprint(f\"  LENGTH_PENALTY: {LENGTH_PENALTY}\")\n\nall_ids, all_pred = [], []\n\nwith torch.inference_mode():\n    for ids, input_ids, attention_mask in loader:\n        input_ids = input_ids.to(DEVICE)\n        attention_mask = attention_mask.to(DEVICE)\n        \n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            num_beams=NUM_BEAMS,\n            max_new_tokens=MAX_NEW_TOKENS,\n            length_penalty=LENGTH_PENALTY,\n            early_stopping=EARLY_STOPPING,\n        )\n        \n        # Decode and POST-PROCESS \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        decoded = [postprocess_output(d) for d in decoded]\n        \n        all_ids.extend(ids)\n        all_pred.extend(decoded)\n\n\n# SAVE SUBMISSION\n# ================\n\nsubmission = pd.DataFrame({\n    \"id\": all_ids,\n    \"translation\": all_pred\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\n\nprint(\"\\n\" + \"=\"*80)\nprint(\"✓ submission.csv saved\")\nprint(\"=\"*80)\nprint(f\"Model: byt5-base-akkadian-3x\")\nprint(f\"NUM_BEAMS: {NUM_BEAMS}\")\nprint(f\"LENGTH_PENALTY: {LENGTH_PENALTY}\")\nprint(f\"\\nFirst row check:\")\n\n# Validation check\nfirst_translation = submission.iloc[0]['translation']\nif first_translation.startswith('Thus'):\n    print(f\"✓ GOOD: '{first_translation[:60]}...'\")\nelse:\n    print(f\"⚠️ WARNING: Starts with '{first_translation[:60]}...'\")\n    print(\"   Expected to start with 'Thus'\")\n\nprint(\"=\"*80)\nprint(\"\\npredictions:\")\nprint(submission.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-18T19:37:00.765421Z","iopub.execute_input":"2026-01-18T19:37:00.766296Z","iopub.status.idle":"2026-01-18T19:37:11.836511Z","shell.execute_reply.started":"2026-01-18T19:37:00.766251Z","shell.execute_reply":"2026-01-18T19:37:11.835798Z"}},"outputs":[],"execution_count":null}]}