{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14410665,"sourceType":"datasetVersion","datasetId":9203761},{"sourceId":14661932,"sourceType":"datasetVersion","datasetId":9366631},{"sourceId":282751,"sourceType":"modelInstanceVersion","modelInstanceId":239470,"modelId":222398}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# References\n- [DPC Infer with post-processing by LLM Ver.7](https://www.kaggle.com/code/takamichitoda/dpc-infer-with-post-processing-by-llm?kernelSessionId=290075269)\n    - 入力正規化を追加\n    - Public Score: 30.3\n- [Deep Past Challenge | ByT5 Optimized](https://www.kaggle.com/code/yongsukprasertsuk/deep-past-challenge-byt5-optimized?kernelSessionId=293321073)\n    - PostProcessingをLLMからHeuristicベースに変更\n        - Public Score: 32.8\n    - Add Ensemble\n        - Public Score: 34.4\n- [BYT-Ensemble | Script](https://www.kaggle.com/code/anthonytherrien/byt-ensemble-script?kernelSessionId=295310388)\n    - PreProcessingとPostProcessingを強化","metadata":{}},{"cell_type":"code","source":"import re\nimport gc\nimport torch\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm\nfrom typing import List","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:17.587923Z","iopub.execute_input":"2026-02-14T13:01:17.588545Z","iopub.status.idle":"2026-02-14T13:01:17.592635Z","shell.execute_reply.started":"2026-02-14T13:01:17.588516Z","shell.execute_reply":"2026-02-14T13:01:17.591863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL1_PATH = \"/kaggle/input/datasets/jeanjean111/byt5-base-big-data2\"\nMODEL2_PATH = \"/kaggle/input/datasets/llkh0a/byt5-akkadian-model\"\nMODEL3_PATH = \"/kaggle/input/datasets/qifeihhh666/train-gap-all-2/byt5-base-akkadian_gap_setence2\"\n# MODEL4_PATH = \"/kaggle/input/datasets/assiaben/final-byt5/byt5-akkadian-optimized-34x\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:17.594100Z","iopub.execute_input":"2026-02-14T13:01:17.594363Z","iopub.status.idle":"2026-02-14T13:01:17.606599Z","shell.execute_reply.started":"2026-02-14T13:01:17.594329Z","shell.execute_reply":"2026-02-14T13:01:17.605823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nBATCH_SIZE = 8\nMAX_LENGTH = 512\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:17.607318Z","iopub.execute_input":"2026-02-14T13:01:17.607541Z","iopub.status.idle":"2026-02-14T13:01:17.621330Z","shell.execute_reply.started":"2026-02-14T13:01:17.607521Z","shell.execute_reply":"2026-02-14T13:01:17.620719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Loading models...\")\n\nm1 = AutoModelForSeq2SeqLM.from_pretrained(MODEL1_PATH)\nm2 = AutoModelForSeq2SeqLM.from_pretrained(MODEL2_PATH)\nm3 = AutoModelForSeq2SeqLM.from_pretrained(MODEL3_PATH)\n\nsd1, sd2, sd3 = m1.state_dict(), m2.state_dict(), m3.state_dict()\n\n# =========================\n# Weighted checkpoint averaging\n# =========================\nperf1, perf2, perf3 = 0.98, 1.00, 0.40\ntotal = perf1 + perf2 + perf3\nw1, w2, w3 = perf1/total, perf2/total, perf3/total\n\nprint(f\"Weights → w1={w1:.3f}, w2={w2:.3f}, w3={w3:.3f}\")\n\nfinal_sd = sd2.copy()\nfor k in final_sd:\n    if k in sd1 and k in sd3:\n        final_sd[k] = w1 * sd1[k] + w2 * sd2[k] + w3 * sd3[k]\n    elif k in sd1:\n        final_sd[k] = w1 * sd1[k] + (w2 + w3) * sd2[k]\n    elif k in sd3:\n        final_sd[k] = w3 * sd3[k] + (w1 + w2) * sd2[k]\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL2_PATH)\nmodel.load_state_dict(final_sd)\nmodel.to(DEVICE).eval().float()\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL2_PATH)\n\ndel m1, m2, m3, sd1, sd2, sd3\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:17.622745Z","iopub.execute_input":"2026-02-14T13:01:17.623286Z","iopub.status.idle":"2026-02-14T13:01:22.696002Z","shell.execute_reply.started":"2026-02-14T13:01:17.623248Z","shell.execute_reply":"2026-02-14T13:01:22.695257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def replace_gaps(text):\n    if pd.isna(text):\n        return text\n    text = str(text)\n    text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'xx', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    text = re.sub(r'……', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:22.696993Z","iopub.execute_input":"2026-02-14T13:01:22.697332Z","iopub.status.idle":"2026-02-14T13:01:22.702678Z","shell.execute_reply.started":"2026-02-14T13:01:22.697307Z","shell.execute_reply":"2026-02-14T13:01:22.702045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class OptimizedPreprocessor:\n    # Initialize precompiled regex patterns\n    def __init__(self):\n        # Pre-compile patterns\n        self.patterns = {\n            \"big_gap\": re.compile(r\"(\\.{3,}|…+|……)\"),\n            \"small_gap\": re.compile(r\"(xx+|\\s+x\\s+)\"),\n        }\n\n    # Preprocess a single input text\n    def preprocess_input_text(self, text: str) -> str:\n        # Handle NaN\n        if pd.isna(text):\n            return \"\"\n\n        # Convert to string\n        cleaned_text = str(text)\n\n        # Replace gaps\n        cleaned_text = self.patterns[\"big_gap\"].sub(\"<big_gap>\", cleaned_text)\n        cleaned_text = self.patterns[\"small_gap\"].sub(\"<gap>\", cleaned_text)\n\n        return cleaned_text\n\n    # Preprocess a batch of texts using vectorized pandas ops\n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        # Convert to Series and sanitize\n        s = pd.Series(texts).fillna(\"\")\n        s = s.astype(str)\n\n        # Apply vectorized replacements\n        s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n        s = s.str.replace(self.patterns[\"small_gap\"], \"<gap>\", regex=True)\n\n        return s.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:22.703715Z","iopub.execute_input":"2026-02-14T13:01:22.704315Z","iopub.status.idle":"2026-02-14T13:01:22.719984Z","shell.execute_reply.started":"2026-02-14T13:01:22.704290Z","shell.execute_reply":"2026-02-14T13:01:22.719329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessor = OptimizedPreprocessor()\ntest_df = pd.read_csv(TEST_DATA_PATH)\n\nraw_texts = test_df[\"transliteration\"].tolist()\npreprocessed = preprocessor.preprocess_batch(raw_texts)\n\n# test_df[\"transliteration\"] = test_df[\"transliteration\"].apply(replace_gaps)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:22.721546Z","iopub.execute_input":"2026-02-14T13:01:22.721805Z","iopub.status.idle":"2026-02-14T13:01:22.749736Z","shell.execute_reply.started":"2026-02-14T13:01:22.721784Z","shell.execute_reply":"2026-02-14T13:01:22.749059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PREFIX = \"translate Akkadian to English: \"\n\nclass InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        # self.texts = df['transliteration'].astype(str).tolist()\n        self.texts = [PREFIX + text for text in preprocessed]\n        self.tokenizer = tokenizer\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        text = self.texts[idx]\n        inputs = self.tokenizer(\n            text, \n            max_length=MAX_LENGTH, \n            padding=\"max_length\", \n            truncation=True, \n            return_tensors=\"pt\"\n        )\n        return {\n            \"input_ids\": inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0)\n        }\n\ntest_dataset = InferenceDataset(preprocessed, tokenizer)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\nprint(\"Starting Inference...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:22.750659Z","iopub.execute_input":"2026-02-14T13:01:22.750841Z","iopub.status.idle":"2026-02-14T13:01:22.756806Z","shell.execute_reply.started":"2026-02-14T13:01:22.750823Z","shell.execute_reply":"2026-02-14T13:01:22.756081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class VectorizedPostprocessor:\n    def __init__(self, aggressive=True):\n        self.aggressive = aggressive\n        self.patterns = {\n            \"gap\": re.compile(r\"(\\\\[x\\\\]|\\\\(x\\\\)|\\\\bx\\\\b)\", re.I),\n            \"big_gap\": re.compile(r\"(\\\\.{3,}|…|\\\\[\\\\.+\\\\])\"),\n            \"annotations\": re.compile(r\"\\\\((fem|plur|pl|sing|singular|plural|\\\\?|!)\\\\..\\\\s*\\\\w*\\\\)\", re.I),\n            \"repeated_words\": re.compile(r\"\\\\b(\\\\w+)(?:\\\\s+\\\\1\\\\b)+\"),\n            \"whitespace\": re.compile(r\"\\\\s+\"),\n            \"punct_space\": re.compile(r\"\\\\s+([.,:])\"),\n            \"repeated_punct\": re.compile(r\"([.,])\\\\1+\"),\n        }\n        self.subscript_trans = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n        self.special_chars_trans = str.maketrans(\"ḫḪ\", \"hH\")\n        self.forbidden_trans = str.maketrans(\"\", \"\", '!?()\"——<>⌈⌋⌊[]+ʾ/;')\n    \n    def postprocess_batch(self, translations):\n        s = pd.Series(translations)\n        valid_mask = s.apply(lambda x: isinstance(x, str) and bool(x.strip()))\n        s.loc[~valid_mask] = \"\"\n        s = s.str.translate(self.special_chars_trans).str.translate(self.subscript_trans)\n        s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True).str.strip()\n        if self.aggressive:\n            s = s.str.replace(self.patterns[\"gap\"], \"<gap>\", regex=True)\n            s = s.str.replace(self.patterns[\"big_gap\"], \"<big_gap>\", regex=True)\n            s = s.str.replace(\"<gap> <gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(\"<big_gap> <big_gap>\", \"<big_gap>\", regex=False)\n            s = s.str.replace(self.patterns[\"annotations\"], \"\", regex=True)\n            s = s.str.replace(\"<gap>\", \"\\\\x00G\\\\x00\", regex=False)\n            s = s.str.replace(\"<big_gap>\", \"\\\\x00B\\\\x00\", regex=False)\n            s = s.str.translate(self.forbidden_trans)\n            s = s.str.replace(\"\\\\x00G\\\\x00\", \" <gap> \", regex=False)\n            s = s.str.replace(\"\\\\x00B\\\\x00\", \" <big_gap> \", regex=False)\n            s = s.str.replace(r\"(\\\\d+)\\\\.5\\\\b\", r\"\\\\1½\", regex=True)\n            s = s.str.replace(r\"\\\\b0\\\\.5\\\\b\", \"½\", regex=True)\n            s = s.str.replace(r\"(\\\\d+)\\\\.25\\\\b\", r\"\\\\1¼\", regex=True)\n            s = s.str.replace(r\"\\\\b0\\\\.25\\\\b\", \"¼\", regex=True)\n            s = s.str.replace(r\"(\\\\d+)\\\\.75\\\\b\", r\"\\\\1¾\", regex=True)\n            s = s.str.replace(r\"\\\\b0\\\\.75\\\\b\", \"¾\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_words\"], r\"\\\\1\", regex=True)\n            for n in range(4, 1, -1):\n                s = s.str.replace(r\"\\\\b((?:\\\\w+\\\\s+){\" + str(n-1) + r\"}\\\\w+)(?:\\\\s+\\\\1\\\\b)+\", r\"\\\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"punct_space\"], r\"\\\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"repeated_punct\"], r\"\\\\1\", regex=True)\n            s = s.str.replace(self.patterns[\"whitespace\"], \" \", regex=True)\n            s = s.str.strip().str.strip(\"-\").str.strip()\n        return s.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:22.757767Z","iopub.execute_input":"2026-02-14T13:01:22.758164Z","iopub.status.idle":"2026-02-14T13:01:22.776258Z","shell.execute_reply.started":"2026-02-14T13:01:22.758130Z","shell.execute_reply":"2026-02-14T13:01:22.775596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def hnr(text):\n#     if not isinstance(text, str) or not text.strip(): return \"hnr\"\n    \n#     t = text.replace('ḫ', 'h').replace('Ḫ', 'H')\n#     sub_map = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n#     t = t.translate(sub_map)\n\n#     t = re.sub(r'(\\[x\\]|\\(x\\)|\\bx\\b)', '<gap>', t, flags=re.I)\n#     t = re.sub(r'(\\.{3,}|…|\\[\\.+\\])', '<big_gap>', t)\n    \n#     t = re.sub(r'<gap>\\s*<gap>', ' <big_gap> ', t)\n#     t = re.sub(r'<big_gap>\\s*<big_gap>', ' <big_gap> ', t)\n\n#     t = re.sub(r'\\((fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)', '', t, flags=re.I)\n\n#     t = t.replace('<gap>', '\\x00GAP\\x00').replace('<big_gap>', '\\x00BIG\\x00')\n\n#     bad_chars = '!?()\"—–<>⌈⌋⌊[]+ʾ/;'\n#     t = t.translate(str.maketrans('', '', bad_chars))\n\n#     t = t.replace('\\x00GAP\\x00', ' <gap> ').replace('\\x00BIG\\x00', ' <big_gap> ')\n\n#     frac_map = {\n#         r'\\.5\\b': ' ½', r'\\.25\\b': ' ¼', r'\\.75\\b': ' ¾',\n#         r'\\.33+\\d*\\b': ' ⅓', r'\\.66+\\d*\\b': ' ⅔'\n#     }\n#     for pat, rep in frac_map.items():\n#         t = re.sub(r'(\\d+)' + pat, r'\\1' + rep, t)\n#         t = re.sub(r'\\b0' + pat, rep.strip(), t)\n\n#     # Remove repetitions (words and phrases)\n#     t = re.sub(r'\\b(\\w+)(?:\\s+\\1\\b)+', r'\\1', t)\n#     for n in range(4, 1, -1):\n#         pat = r'\\b((?:\\w+\\s+){' + str(n-1) + r'}\\w+)(?:\\s+\\1\\b)+'\n#         t = re.sub(pat, r'\\1', t)\n\n#     # Additional improvements\n#     # Fix spacing around punctuation\n#     t = re.sub(r'\\s+([.,:])', r'\\1', t)\n    \n#     # Remove duplicate punctuation\n#     t = re.sub(r'([.,])\\1+', r'\\1', t)\n    \n#     # Clean up multiple spaces\n#     t = re.sub(r'\\s+', ' ', t).strip()\n    \n#     # Remove leading/trailing dashes and spaces\n#     t = t.strip().strip('-').strip()\n    \n#     return t","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:22.778201Z","iopub.execute_input":"2026-02-14T13:01:22.778419Z","iopub.status.idle":"2026-02-14T13:01:22.791789Z","shell.execute_reply.started":"2026-02-14T13:01:22.778399Z","shell.execute_reply":"2026-02-14T13:01:22.791070Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_predictions = []\npostProcessor = VectorizedPostprocessor()\nwith torch.no_grad():\n    for batch in tqdm(test_loader):\n        input_ids = batch[\"input_ids\"].to(DEVICE)\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n  \n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            max_length=MAX_LENGTH,\n            num_beams=8,\n            max_new_tokens=512,\n            length_penalty=1.3,\n            early_stopping=True\n        )\n        \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        cleaned = postProcessor.postprocess_batch(decoded)\n\n        all_predictions.extend([c.strip() for c in cleaned])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:22.792718Z","iopub.execute_input":"2026-02-14T13:01:22.792969Z","iopub.status.idle":"2026-02-14T13:01:31.199838Z","shell.execute_reply.started":"2026-02-14T13:01:22.792936Z","shell.execute_reply":"2026-02-14T13:01:31.199059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": all_predictions\n})\n\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission file saved successfully!\")\nsubmission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-14T13:01:31.200832Z","iopub.execute_input":"2026-02-14T13:01:31.201084Z","iopub.status.idle":"2026-02-14T13:01:31.223061Z","shell.execute_reply.started":"2026-02-14T13:01:31.201061Z","shell.execute_reply":"2026-02-14T13:01:31.222485Z"}},"outputs":[],"execution_count":null}]}