{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14374989,"sourceType":"datasetVersion","datasetId":9147887},{"sourceId":14376272,"sourceType":"datasetVersion","datasetId":9181082},{"sourceId":14410665,"sourceType":"datasetVersion","datasetId":9203761},{"sourceId":282751,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":239470,"modelId":222398}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Initiative – Machine Translation (Inference Notebook)\n\nI noticed that the English text after translation sometimes sounded unnatural or awkward.\nTo address this issue, I have shared code that applies LLM-based post-processing to improve the quality of the translated text.\n\nThe current implementation is based on the publicly available Best Score code, with additional post-processing using Gemma-3.\n\n###  Update history:\n\n- [Base Notebook](https://www.kaggle.com/code/takamichitoda/dpc-starter-infer)\n\t- Public Score: 26.6\n- [Version 5](https://www.kaggle.com/code/takamichitoda/dpc-infer-with-post-processing-by-llm?scriptVersionId=287804751): \n\t- Public Score: 28.9\n\t- Use the Best Score model at the time. \n\t- But it have been deleted.\n- [Version 7](https://www.kaggle.com/code/takamichitoda/dpc-infer-with-post-processing-by-llm?scriptVersionId=290075269): \n\t- Public Score: 30.3\n\t- Use the Best Score model at the time.\n\t- model -> [byt5-base-32.6-third](https://www.kaggle.com/datasets/jeanjean111/byt5-base-big-data2)\n- [Version 10](https://www.kaggle.com/code/takamichitoda/dpc-infer-with-post-processing-by-llm?scriptVersionId=290148346)\n\t- Public Score: 32.6\n\t- Fix Prompt.\n- [Version 12](https://www.kaggle.com/code/takamichitoda/dpc-infer-with-post-processing-by-llm?scriptVersionId=292147659)\n\t- Public Score: \n\t- Use the Best Score model at the time.\n\t- weight AVG -> [byt5-base-big-data2](https://www.kaggle.com/datasets/jeanjean111/byt5-base-big-data2), [train-gap-all-2](https://www.kaggle.com/datasets/qifeihhh666/train-gap-all-2), [byt5-akkadian-model](https://www.kaggle.com/datasets/llkh0a/byt5-akkadian-model)\n      - Reference code is [here](https://www.kaggle.com/code/yongsukprasertsuk/deep-past-challenge-weight-averaging).\n    - Shift the post-processing from an LLM-centric approach to a more conservative, consistency-focused approach centered on the dictionary (OA_Lexicon) and train references (translation memory).\n\n\n\nIf you fork this code, don’t forget to upvote the `qifeihhh666`, `jeanjean111`, and `llkh0a` datasets shared by their awesome authors. Let’s support the spirit of contribution!  \n👉 [byt5-base-big-data2](https://www.kaggle.com/datasets/jeanjean111/byt5-base-big-data2)  \n👉 [train-gap-all-2](https://www.kaggle.com/datasets/qifeihhh666/train-gap-all-2)  \n👉 [byt5-akkadian-model](https://www.kaggle.com/datasets/llkh0a/byt5-akkadian-model)   ","metadata":{}},{"cell_type":"code","source":"import re\nimport gc\nimport torch\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:17:12.864714Z","iopub.execute_input":"2026-01-16T05:17:12.864991Z","iopub.status.idle":"2026-01-16T05:17:24.577432Z","shell.execute_reply.started":"2026-01-16T05:17:12.864960Z","shell.execute_reply":"2026-01-16T05:17:24.576844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL1_PATH = \"/kaggle/input/byt5-base-big-data2\"\nMODEL2_PATH = \"/kaggle/input/byt5-akkadian-model\"\nMODEL3_PATH = \"/kaggle/input/train-gap-all-2/byt5-base-akkadian_gap_setence2\"\n\nTEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nBATCH_SIZE = 4\nMAX_LENGTH = 512\nMAX_NEW_TOKENS = 512\nBATCH_SIZE = 8\nNUM_BEAMS = 10\nLENGTH_PENALTY = 1.08\nEARLY_STOPPING = True\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:17:24.578256Z","iopub.execute_input":"2026-01-16T05:17:24.578736Z","iopub.status.idle":"2026-01-16T05:17:24.635064Z","shell.execute_reply.started":"2026-01-16T05:17:24.578708Z","shell.execute_reply":"2026-01-16T05:17:24.634138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Load models\n# =========================\nprint(\"Loading models...\")\n\nm1 = AutoModelForSeq2SeqLM.from_pretrained(MODEL1_PATH)\nm2 = AutoModelForSeq2SeqLM.from_pretrained(MODEL2_PATH)\nm3 = AutoModelForSeq2SeqLM.from_pretrained(MODEL3_PATH)\n\nsd1, sd2, sd3 = m1.state_dict(), m2.state_dict(), m3.state_dict()\n\n# =========================\n# Weighted checkpoint averaging\n# =========================\nperf1, perf2, perf3 = 0.98, 1.00, 0.40\ntotal = perf1 + perf2 + perf3\nw1, w2, w3 = perf1/total, perf2/total, perf3/total\n\nprint(f\"Weights → w1={w1:.3f}, w2={w2:.3f}, w3={w3:.3f}\")\n\nfinal_sd = sd2.copy()\nfor k in final_sd:\n    if k in sd1 and k in sd3:\n        final_sd[k] = w1 * sd1[k] + w2 * sd2[k] + w3 * sd3[k]\n    elif k in sd1:\n        final_sd[k] = w1 * sd1[k] + (w2 + w3) * sd2[k]\n    elif k in sd3:\n        final_sd[k] = w3 * sd3[k] + (w1 + w2) * sd2[k]\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL2_PATH)\nmodel.load_state_dict(final_sd)\nmodel.to(DEVICE).eval().float()\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL2_PATH)\n\ndel m1, m2, m3, sd1, sd2, sd3\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:17:24.637182Z","iopub.execute_input":"2026-01-16T05:17:24.637859Z","iopub.status.idle":"2026-01-16T05:18:20.788228Z","shell.execute_reply.started":"2026-01-16T05:17:24.637807Z","shell.execute_reply":"2026-01-16T05:18:20.787579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================\n# Gap normalization (VERY IMPORTANT)\n# =========================\ndef replace_gaps(text):\n    if pd.isna(text):\n        return text\n    text = str(text)\n    text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.3(?:\\s+\\.3)+\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'\\.{3}(?:\\s+\\.{3})+', '<big_gap>', text)\n    text = re.sub(r'xx', '<gap>', text)\n    text = re.sub(r' x ', ' <gap> ', text)\n    text = re.sub(r'……', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.\\.\\.\\.', '<big_gap>', text)\n    text = re.sub(r'…', '<big_gap>', text)\n    text = re.sub(r'\\.\\.\\.', '<big_gap>', text)\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:18:20.789270Z","iopub.execute_input":"2026-01-16T05:18:20.789985Z","iopub.status.idle":"2026-01-16T05:18:20.795159Z","shell.execute_reply.started":"2026-01-16T05:18:20.789948Z","shell.execute_reply":"2026-01-16T05:18:20.794617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(TEST_DATA_PATH)\ntest_df[\"transliteration\"] = test_df[\"transliteration\"].apply(replace_gaps)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:18:20.796110Z","iopub.execute_input":"2026-01-16T05:18:20.796329Z","iopub.status.idle":"2026-01-16T05:18:20.845394Z","shell.execute_reply.started":"2026-01-16T05:18:20.796307Z","shell.execute_reply":"2026-01-16T05:18:20.844596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PREFIX = \"translate Akkadian to English: \"\n\nclass InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.texts = df['transliteration'].astype(str).tolist()\n        self.texts = [PREFIX + i for i in self.texts]\n        self.tokenizer = tokenizer\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        text = self.texts[idx]\n        inputs = self.tokenizer(\n            text, \n            max_length=MAX_LENGTH, \n            padding=\"max_length\", \n            truncation=True, \n            return_tensors=\"pt\"\n        )\n        return {\n            \"input_ids\": inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0)\n        }\n\ntest_dataset = InferenceDataset(test_df, tokenizer)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:18:20.846400Z","iopub.execute_input":"2026-01-16T05:18:20.846804Z","iopub.status.idle":"2026-01-16T05:18:20.853732Z","shell.execute_reply.started":"2026-01-16T05:18:20.846768Z","shell.execute_reply":"2026-01-16T05:18:20.853013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Starting Inference...\")\nall_predictions = []\n\nwith torch.no_grad():\n    for batch in tqdm(test_loader):\n        input_ids = batch[\"input_ids\"].to(DEVICE)\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n  \n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            #max_length=MAX_LENGTH,\n            num_beams=NUM_BEAMS,\n            max_new_tokens=MAX_NEW_TOKENS,\n            length_penalty=LENGTH_PENALTY,\n            early_stopping=EARLY_STOPPING,\n        )\n        \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_predictions.extend([d.strip() for d in decoded])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:18:20.854788Z","iopub.execute_input":"2026-01-16T05:18:20.855108Z","iopub.status.idle":"2026-01-16T05:18:29.018575Z","shell.execute_reply.started":"2026-01-16T05:18:20.855073Z","shell.execute_reply":"2026-01-16T05:18:29.017865Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Post Processing with LLM","metadata":{}},{"cell_type":"code","source":"# =========================\n# Post-processing config\n# =========================\nUSE_OA_LEXICON = True\nOA_LEXICON_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/OA_Lexicon_eBL.csv\"\nOA_THRESHOLD = 0.92  # higher = safer (less aggressive)\n\n# OA Lexicon tuning (v2: safer)\nOA_USE_TRAIN_SURFACE = True\nOA_TRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\nOA_MIN_SURFACE_FREQ = 3    # only use spellings that appear >= this many times in train\nOA_REQUIRE_PRED_CAPITAL = True  # safest: only normalize tokens starting with uppercase\nOA_ALLOW_NEAR_MATCH = False     # can help (Ashur/Assur), but may hurt if too aggressive\nOA_NEAR_MAX_DIST = 1            # used only when OA_ALLOW_NEAR_MATCH=True\n\n# OA Lexicon tuning (v3+: a bit more coverage, still safe)\nOA_ALLOW_LOWERCASE_IF_TARGET = True   # also fix lowercased proper names if they are in source targets\nOA_LOWER_MIN_LEN = 4\nOA_MIN_SURFACE_FREQ_NAME_TYPES = 2    # for explicit NE types (DN/GN/PN...), allow rarer spellings\nOA_NEAR_MIN_TARGET_FREQ = 10          # near-match only for very frequent names\nOA_NEAR_MIN_LEN = 5\n\n# =========================\n# Translation memory (exact match from train)\n# =========================\n# Very safe if duplicates exist between train/test.\n# If a test transliteration EXACTLY matches a train transliteration (after replace_gaps + optional space normalization),\n# we directly output the most frequent train translation for that source.\nUSE_TRAIN_EXACT_MATCH = True\nTRAIN_MATCH_NORMALIZE_SRC = True   # collapse multiple spaces in transliteration for matching\n\n# Near-duplicate translation memory (optional, higher risk than exact match)\n# Uses char TF-IDF on transliteration; apply only when similarity is extremely high.\nUSE_TRAIN_NEAR_DUP = False\nNEAR_DUP_SIM_THRESHOLD = 0.995\nNEAR_DUP_MIN_SRC_LEN = 20\n\n\n# Optional: LLM post-edit (can be slow / can hurt BLEU if it paraphrases)\nUSE_LLM_POLISH = False  # set True to enable Gemma post-edit","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:18:29.019567Z","iopub.execute_input":"2026-01-16T05:18:29.019964Z","iopub.status.idle":"2026-01-16T05:18:29.025101Z","shell.execute_reply.started":"2026-01-16T05:18:29.019938Z","shell.execute_reply":"2026-01-16T05:18:29.024443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport gc\nimport pandas as pd\nimport re\nimport unicodedata\nfrom collections import defaultdict\nfrom tqdm.auto import tqdm\nfrom transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, AutoProcessor\nfrom collections import Counter, defaultdict","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:18:29.027269Z","iopub.execute_input":"2026-01-16T05:18:29.027482Z","iopub.status.idle":"2026-01-16T05:18:29.177782Z","shell.execute_reply.started":"2026-01-16T05:18:29.027463Z","shell.execute_reply":"2026-01-16T05:18:29.177191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del model\ndel tokenizer\ndel test_loader\ndel test_dataset\n\ngc.collect()\ntorch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-16T05:18:29.178510Z","iopub.execute_input":"2026-01-16T05:18:29.178783Z","iopub.status.idle":"2026-01-16T05:18:29.695646Z","shell.execute_reply.started":"2026-01-16T05:18:29.178750Z","shell.execute_reply":"2026-01-16T05:18:29.694813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------\n# Load OA Lexicon and build token->lexeme index\n# -------------------------\nif USE_OA_LEXICON:\n    print(f\"📚 Loading OA Lexicon: {OA_LEXICON_PATH}\")\n    oa = pd.read_csv(OA_LEXICON_PATH)\n    print(\"OA Lexicon rows:\", len(oa))\n\n    SUB_DIGITS = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n\n    def norm_key_token(s: str) -> str:\n        \"\"\"Key for matching transliteration tokens to lexicon tokens.\"\"\"\n        s = \"\" if s is None else str(s)\n        s = unicodedata.normalize(\"NFKC\", s).translate(SUB_DIGITS).strip()\n        # remove wrapping brackets/quotes\n        s = re.sub(r\"^[\\\"'“”‘’\\(\\)\\[\\]\\{\\}<>]+\", \"\", s)\n        s = re.sub(r\"[\\\"'“”‘’\\(\\)\\[\\]\\{\\}<>]+$\", \"\", s)\n        # trim punctuation at edges\n        s = s.strip(\".,;:!?\")\n        return s.lower()\n\n    token2lexemes = defaultdict(list)  # token_key -> [(lexeme, type), ...]\n\n    for _, r in oa.iterrows():\n        typ = \"\" if pd.isna(r.get(\"type\")) else str(r[\"type\"]).strip()\n        lex = \"\" if pd.isna(r.get(\"lexeme\")) else str(r[\"lexeme\"]).strip()\n        if not lex:\n            continue\n\n        for col in [\"form\", \"norm\", \"Alt_lex\"]:\n            if col not in oa.columns:\n                continue\n            v = r.get(col)\n            if pd.isna(v):\n                continue\n            for tok in str(v).split():\n                k = norm_key_token(tok)\n                if k:\n                    token2lexemes[k].append((lex, typ))\n\n    # de-dup lists (keep order)\n    for k, v in list(token2lexemes.items()):\n        seen = set()\n        uniq = []\n        for lex, typ in v:\n            key = (lex, typ)\n            if key in seen:\n                continue\n            seen.add(key)\n            uniq.append((lex, typ))\n        token2lexemes[k] = uniq\n\n    print(\"OA token keys indexed:\", len(token2lexemes))\nelse:\n    token2lexemes = defaultdict(list)\n\n\n# -------------------------\n# Folding + heuristics\n# -------------------------\n_DIACRITIC_MAP = str.maketrans({\n    \"š\": \"s\", \"Š\": \"s\",\n    \"ṣ\": \"s\", \"Ṣ\": \"s\",\n    \"ṭ\": \"t\", \"Ṭ\": \"t\",\n    \"ḫ\": \"h\", \"Ḫ\": \"h\",\n    \"ā\": \"a\", \"Ā\": \"a\",\n    \"ē\": \"e\", \"Ē\": \"e\",\n    \"ī\": \"i\", \"Ī\": \"i\",\n    \"ū\": \"u\", \"Ū\": \"u\",\n    \"ʾ\": \"\", \"ʼ\": \"\", \"’\": \"\", \"'\": \"\",\n})\n\n_DIACRITIC_CHARS = set([\n    \"š\",\"Š\",\"ṣ\",\"Ṣ\",\"ṭ\",\"Ṭ\",\"ḫ\",\"Ḫ\",\"ā\",\"ē\",\"ī\",\"ū\",\"Ā\",\"Ē\",\"Ī\",\"Ū\"\n])\n\ndef _strip_disambig(s: str) -> str:\n    \"\"\"Remove trailing numeric homograph markers: Inanna2 -> Inanna\"\"\"\n    s = \"\" if s is None else str(s)\n    s = unicodedata.normalize(\"NFKC\", s)\n    s = re.sub(r\"(?<=\\D)\\d+$\", \"\", s)\n    return s\n\n\ndef fold_for_match(s: str) -> str:\n    \"\"\"Aggressive fold for matching name variants (diacritics/digraphs).\"\"\"\n    s = \"\" if s is None else str(s)\n    s = unicodedata.normalize(\"NFKC\", s)\n    s = _strip_disambig(s)\n    s = s.translate(_DIACRITIC_MAP)\n    s = s.lower()\n    # common ASCII digraph variants\n    s = s.replace(\"sh\", \"s\").replace(\"kh\", \"h\")\n    # keep only letters (digits often disambiguators in names)\n    s = re.sub(r\"[^a-z]+\", \"\", s)\n    return s\n\n\ndef looks_like_name(lexeme: str, typ: str) -> bool:\n    if not lexeme:\n        return False\n    t = (typ or \"\").strip().upper()\n\n    # If the lexicon has explicit NE tags, prefer them\n    if t in {\"DN\", \"GN\", \"PN\", \"MN\", \"ON\", \"TN\"}:\n        return True\n\n    # heuristic: lexeme contains uppercase OR Akkadian diacritics\n    if any(ch.isupper() for ch in lexeme):\n        return True\n    if any(ch in _DIACRITIC_CHARS for ch in lexeme):\n        return True\n\n    return False\n\n# -------------------------\n# Extra safety/coverage helpers (v4)\n# -------------------------\n\nEXPLICIT_NE_TYPES = {\"DN\", \"GN\", \"PN\", \"MN\", \"ON\", \"TN\"}\n\ndef is_explicit_ne_type(typ: str) -> bool:\n    t = (typ or \"\").strip().upper()\n    return t in EXPLICIT_NE_TYPES\n\n# stopwords to avoid accidentally uppercasing/rewriting common words when OA_ALLOW_LOWERCASE_IF_TARGET=True\nEN_STOPWORDS = {\n    'the','a','an','and','or','of','to','in','on','at','by','for','from','with','as','but','not','no','nor',\n    'is','are','was','were','be','been','being',\n    'i','you','he','she','it','we','they','me','him','her','us','them','my','your','his','their','our','its',\n    'this','that','these','those','there','here',\n    'who','whom','which','what','when','where','why','how',\n}\n\n\n\n# -------------------------\n# Learn the *surface spelling* from train translations\n# -------------------------\nfold2surface = {}\nfold2freq = {}\n\n\n# -------------------------\n# Translation memory: exact match mapping (train source -> most common train translation)\n# -------------------------\ntrain_exact_map = {}\n\n# -------------------------\n# Near-duplicate TM (char TF-IDF)\n# -------------------------\nnear_dup_vec = None\nnear_dup_nn = None\nnear_dup_keys = None\nnear_dup_tgts = None\n\nif USE_TRAIN_NEAR_DUP and train_exact_map:\n    try:\n        from sklearn.feature_extraction.text import TfidfVectorizer\n        from sklearn.neighbors import NearestNeighbors\n\n        # Use unique sources (train_exact_map keys) to keep it small\n        near_dup_keys = list(train_exact_map.keys())\n        near_dup_tgts = [train_exact_map[k] for k in near_dup_keys]\n\n        near_dup_vec = TfidfVectorizer(\n            analyzer='char',\n            ngram_range=(3, 5),\n            min_df=2,\n            lowercase=True,\n        )\n        X_train = near_dup_vec.fit_transform(near_dup_keys)\n\n        near_dup_nn = NearestNeighbors(n_neighbors=1, metric='cosine', algorithm='brute')\n        near_dup_nn.fit(X_train)\n\n        print(f\"🔁 Near-dup TM ready | train_keys={len(near_dup_keys)}\")\n\n    except Exception as e:\n        print('⚠️ Near-dup TM disabled due to error:', repr(e))\n        near_dup_vec = None\n        near_dup_nn = None\n        near_dup_keys = None\n        near_dup_tgts = None\n\n\ndef norm_src_for_match(s: str) -> str:\n    s = \"\" if s is None else str(s)\n    s = s.strip()\n    if TRAIN_MATCH_NORMALIZE_SRC:\n        s = re.sub(r\"\\s+\", \" \", s)\n    return s\n\nSUB_DIGITS_TM = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n\ndef norm_src_loose(s: str) -> str:\n    \"\"\"Aggressive-ish normalization for TM matching.\n    Safety is enforced elsewhere by requiring a unique strict source per loose key.\n    \"\"\"\n    s = \"\" if s is None else str(s)\n    s = unicodedata.normalize(\"NFKC\", s).translate(SUB_DIGITS_TM)\n    s = s.lower()\n    # keep gap tokens, letters, digits; turn separators/punct into spaces\n    s = re.sub(r\"[^a-z0-9šṣṭḫāēīū<>_]+\", \" \", s)\n    s = re.sub(r\"\\s+\", \" \", s).strip()\n    return s\n\nif USE_TRAIN_EXACT_MATCH:\n    try:\n        print(f\"🧠 Building exact-match TM from train: {OA_TRAIN_PATH}\")\n        _tm_train = pd.read_csv(OA_TRAIN_PATH)\n        if 'transliteration' not in _tm_train.columns:\n            raise ValueError('train.csv missing transliteration column')\n        _tm_train['transliteration'] = _tm_train['transliteration'].apply(replace_gaps)\n\n        tgt_col = 'translation' if 'translation' in _tm_train.columns else _tm_train.columns[-1]\n\n        from collections import Counter, defaultdict\n        tmp = defaultdict(Counter)\n        for src, tgt in zip(_tm_train['transliteration'].astype(str).tolist(), _tm_train[tgt_col].astype(str).tolist()):\n            k = norm_src_for_match(src)\n            if not k:\n                continue\n            tmp[k][tgt] += 1\n        # choose most frequent translation for duplicated sources\n        train_exact_map = {k: c.most_common(1)[0][0] for k, c in tmp.items()}\n        print('TM entries:', len(train_exact_map))\n\n        # --- Loose-exact TM map (format-insensitive, but safety-checked) ---\n        train_loose_map = {}\n        if 'USE_TRAIN_LOOSE_MATCH' in globals() and USE_TRAIN_LOOSE_MATCH:\n            try:\n                from collections import defaultdict\n                loose2strict = defaultdict(set)\n                for _src in _tm_train['transliteration'].astype(str).tolist():\n                    sk = norm_src_for_match(_src)\n                    lk = norm_src_loose(_src)\n                    if lk:\n                        loose2strict[lk].add(sk)\n\n                # Only accept loose keys that map to ONE strict key (very safe)\n                for lk, sset in loose2strict.items():\n                    if len(sset) == 1:\n                        sk = next(iter(sset))\n                        if sk in train_exact_map:\n                            train_loose_map[lk] = train_exact_map[sk]\n\n                print('Loose TM entries:', len(train_loose_map))\n            except Exception as _e:\n                print('⚠️ Loose TM build failed:', repr(_e))\n                train_loose_map = {}\n        else:\n            train_loose_map = {}\n    except Exception as e:\n        print('⚠️ TM build failed:', repr(e))\n        train_exact_map = {}\n\nif USE_OA_LEXICON and OA_USE_TRAIN_SURFACE:\n    try:\n        train_df = pd.read_csv(OA_TRAIN_PATH)\n        if \"translation\" in train_df.columns:\n            col = \"translation\"\n        else:\n            # fallback: last column\n            col = train_df.columns[-1]\n\n        surf_counter = defaultdict(Counter)\n\n        token_re = re.compile(r\"[A-Za-zšṣṭḫāēīūŠṢṬḪĀĒĪŪ'’\\-]+\")\n\n        for text in train_df[col].astype(str).tolist():\n            for tok in token_re.findall(text):\n                if len(tok) < 3:\n                    continue\n\n                # focus on tokens that look like proper nouns in English references\n                if not (tok[:1].isupper() or any(ch in _DIACRITIC_CHARS for ch in tok)):\n                    continue\n\n                f = fold_for_match(tok)\n                if len(f) < 4:\n                    continue\n\n                surf_counter[f][tok] += 1\n\n        for f, counter in surf_counter.items():\n            tok, cnt = counter.most_common(1)[0]\n            fold2surface[f] = tok\n            fold2freq[f] = cnt\n\n        print(f\"🔎 Learned surface forms from train: {len(fold2surface)} folds\")\n\n    except Exception as e:\n        print(\"⚠️ Could not build train surface table:\", repr(e))\n        fold2surface = {}\n        fold2freq = {}\n\n\n# -------------------------\n# Optional: near match (edit distance <= 1), OFF by default\n# -------------------------\n\ndef _levenshtein_leq(a: str, b: str, max_dist: int = 1) -> bool:\n    if a == b:\n        return True\n    if abs(len(a) - len(b)) > max_dist:\n        return False\n\n    # DP with early stop (max_dist small)\n    prev = list(range(len(b) + 1))\n    for i, ca in enumerate(a, 1):\n        cur = [i]\n        min_cur = cur[0]\n        for j, cb in enumerate(b, 1):\n            ins = cur[j-1] + 1\n            dele = prev[j] + 1\n            sub = prev[j-1] + (ca != cb)\n            v = min(ins, dele, sub)\n            cur.append(v)\n            if v < min_cur:\n                min_cur = v\n        if min_cur > max_dist:\n            return False\n        prev = cur\n    return prev[-1] <= max_dist\n\ndef _levenshtein_distance_cap(a: str, b: str, max_dist: int = 1) -> int:\n    \"\"\"Return Levenshtein distance if <= max_dist, else max_dist+1 (early stop).\"\"\"\n    if a == b:\n        return 0\n    if abs(len(a) - len(b)) > max_dist:\n        return max_dist + 1\n\n    prev = list(range(len(b) + 1))\n    for i, ca in enumerate(a, 1):\n        cur = [i]\n        min_cur = cur[0]\n        for j, cb in enumerate(b, 1):\n            ins = cur[j-1] + 1\n            dele = prev[j] + 1\n            sub = prev[j-1] + (ca != cb)\n            v = min(ins, dele, sub)\n            cur.append(v)\n            if v < min_cur:\n                min_cur = v\n        if min_cur > max_dist:\n            return max_dist + 1\n        prev = cur\n    return prev[-1]\n\n\n\n# -------------------------\n# Sentence-level target extraction + substitution\n# -------------------------\n\ndef extract_name_targets(translit: str, max_targets: int = 50):\n    \"\"\"Return a dict: fold_key -> best_surface_token (from train).\"\"\"\n    if not USE_OA_LEXICON:\n        return {}\n\n    translit = \"\" if translit is None else str(translit)\n    targets = {}\n    seen = set()\n\n    for tok in translit.split():\n        k = norm_key_token(tok)\n        for lex, typ in token2lexemes.get(k, []):\n            if lex in seen:\n                continue\n            seen.add(lex)\n\n            if not looks_like_name(lex, typ):\n                continue\n\n            # normalize lexeme and fold\n            lex_clean = _strip_disambig(lex)\n            f = fold_for_match(lex_clean)\n            if len(f) < 4:\n                continue\n\n            # only use spellings that appear in references (train)\n            min_freq = OA_MIN_SURFACE_FREQ_NAME_TYPES if is_explicit_ne_type(typ) else OA_MIN_SURFACE_FREQ\n            if fold2surface and (f in fold2surface) and (fold2freq.get(f, 0) >= min_freq):\n                targets[f] = fold2surface[f]\n\n        if len(targets) >= max_targets:\n            break\n\n    return targets\n\n\ndef lexicon_name_normalize(pred: str, targets: dict) -> str:\n    if not pred or not targets:\n        return pred\n\n    parts = str(pred).split()\n    out = []\n\n    for p in parts:\n        m = re.match(r\"^(\\W*)(.*?)(\\W*)$\", p)\n        pre, core, suf = m.group(1), m.group(2), m.group(3)\n\n        if not core:\n            out.append(p)\n            continue\n\n        # Handle possessive endings: Assur's / Assur’s\n        poss = \"\"\n        core_base = core\n        pm = re.match(r\"^(.*?)(['’]s)$\", core)\n        if pm:\n            core_base = pm.group(1)\n            poss = pm.group(2)\n\n        # Fold for match\n        f = fold_for_match(core_base)\n        if len(f) < 4:\n            out.append(p)\n            continue\n\n        is_cap = core_base[:1].isupper()\n\n        # 1) Exact folded match: apply replacement (optionally even if lowercase)\n        if f in targets:\n            if is_cap or (not OA_REQUIRE_PRED_CAPITAL):\n                out.append(pre + targets[f] + poss + suf)\n                continue\n\n            # Allow lowercased proper names ONLY when they are in the source targets\n            if OA_ALLOW_LOWERCASE_IF_TARGET and len(f) >= OA_LOWER_MIN_LEN and core_base.lower() not in EN_STOPWORDS:\n                out.append(pre + targets[f] + poss + suf)\n                continue\n\n            out.append(p)\n            continue\n\n        # 2) If capital required and token isn't capitalized, do nothing\n        if OA_REQUIRE_PRED_CAPITAL and not is_cap:\n            out.append(p)\n            continue\n\n        # 3) Optional near match (very conservative)\n        if OA_ALLOW_NEAR_MATCH and len(f) >= OA_NEAR_MIN_LEN:\n            best = None\n            best_dist = 999\n            best_freq = -1\n            for tf, surf in targets.items():\n                if len(tf) < OA_NEAR_MIN_LEN:\n                    continue\n                # only try near-match for very frequent canonical spellings\n                if fold2freq.get(tf, 0) < OA_NEAR_MIN_TARGET_FREQ:\n                    continue\n                if abs(len(f) - len(tf)) > OA_NEAR_MAX_DIST:\n                    continue\n                # quick guards\n                if f[0] != tf[0] or f[-1] != tf[-1]:\n                    continue\n\n                dist = _levenshtein_distance_cap(f, tf, max_dist=OA_NEAR_MAX_DIST)\n                if dist <= OA_NEAR_MAX_DIST:\n                    freq = fold2freq.get(tf, 0)\n                    if (dist < best_dist) or (dist == best_dist and freq > best_freq):\n                        best = surf\n                        best_dist = dist\n                        best_freq = freq\n\n            if best is not None:\n                out.append(pre + best + poss + suf)\n                continue\n\n        out.append(p)\n\n    return \" \".join(out)\n\n\n\ndef post_process_with_oa_lexicon(translit: str, pred: str, threshold: float = None) -> str:\n    # `threshold` kept for backward-compatibility (v1 notebooks), but v2 doesn't use it.\n    if not USE_OA_LEXICON:\n        return pred\n    targets = extract_name_targets(translit)\n    return lexicon_name_normalize(pred, targets)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LLM_MODEL_PATH = \"/kaggle/input/gemma-3/transformers/gemma-3-4b-it/1\"\n\n# 4bit量子化でロード (T4 GPU x2環境でも動作可能に)\n# bnb_config = BitsAndBytesConfig(\n#     load_in_4bit=True,\n#     bnb_4bit_quant_type=\"nf4\",\n#     bnb_4bit_compute_dtype=torch.float16,\n#     bnb_4bit_use_double_quant=True,\n# )\n\nif USE_LLM_POLISH:\n    print(f\"🚀 Loading LLM from {LLM_MODEL_PATH}...\")\n    llm_tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_PATH)\n    llm_model = AutoModelForCausalLM.from_pretrained(\n        LLM_MODEL_PATH,\n        # quantization_config=bnb_config,\n        device_map={\"\": 0},\n        torch_dtype=torch.bfloat16,\n    )\nelse:\n    llm_tokenizer = None\n    llm_model = None\n    print(\"⚠️ USE_LLM_POLISH=False -> skip loading LLM.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if USE_LLM_POLISH and llm_model is not None:\n    llm_model.device\nelse:\n    \"LLM disabled\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 方針A: 「単語は変えない」最小編集プロンプト\ndef make_gemma3_prompt(draft_text: str):\n    system_text = \"\"\"You are a deterministic post-editor for MT outputs.\nGoal: maximize exact-match metrics (BLEU/chrF). Therefore NEVER paraphrase.\n\nALLOWED edits (ONLY):\n- whitespace normalization (remove double spaces)\n- spacing around punctuation , . ; : ! ?\n- normalize quotes/dashes to ASCII (' \" -)\n- if there is an unmatched opening '[' or '(' then ONLY add the missing closing bracket ']' or ')' at the END of the text\n- capitalize the first character ONLY if it is a letter AND you do not change any other characters\n\nFORBIDDEN:\n- changing, adding, deleting, or reordering ANY words\n- changing numbers\n- changing proper nouns or names\n- adding explanations\n\nOutput: the corrected text only (single line). If no edits needed, output the input EXACTLY.\"\"\"\n    return [\n        {\"role\": \"system\", \"content\": system_text},\n        {\"role\": \"user\", \"content\": draft_text},\n    ]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport difflib\n\n# ====== ルールベース正規化（LLMより安全） ======\n_DASH_MAP = str.maketrans({\"–\": \"-\", \"—\": \"-\", \"−\": \"-\"})\n_QUOTE_MAP = str.maketrans({\"“\": '\"', \"”\": '\"', \"’\": \"'\", \"‘\": \"'\"})\n\ndef basic_normalize(s: str) -> str:\n    s = str(s)\n    s = s.translate(_DASH_MAP).translate(_QUOTE_MAP)\n    s = re.sub(r\"[ \\t]+\", \" \", s)                      # collapse spaces/tabs\n    s = re.sub(r\"\\s+([,.;:!?])\", r\"\\1\", s)            # no space before punctuation\n    s = re.sub(r\"([,.;:!?])([A-Za-z])\", r\"\\1 \\2\", s)  # ensure a space after punctuation before letters\n    s = s.strip()\n\n    # if bracket is obviously missing, only add closers at the end (metric-safe)\n    if s.count(\"[\") > s.count(\"]\"):\n        s = s + (\"]\" * (s.count(\"[\") - s.count(\"]\")))\n    if s.count(\"(\") > s.count(\")\"):\n        s = s + (\")\" * (s.count(\"(\") - s.count(\")\")))\n\n    return s\n\ndef needs_polish(s: str) -> bool:\n    # 「指標に効きそうな致命傷」だけ\n    if s.count(\"[\") != s.count(\"]\"):\n        return True\n    if s.count(\"(\") != s.count(\")\"):\n        return True\n    # 句読点前スペースが多い/連発などの明確な破綻\n    if re.search(r\"\\s+([,.;:!?])\", s):\n        return True\n    if re.search(r\"([,.;:!?])\\1{1,}\", s):\n        return True\n    return False\n\n\n# ====== 強い安全装置（BLEU破壊を防ぐ） ======\nPRESERVE_TERMS = [\n    \"Seal of\", \"son of\", \"gin\", \"mina\", \"shekel\",\n]\nSIMILARITY_MIN = 0.985   # かなり高め（= 少しでも言い換えたら落とす）\nMAX_ABS_LEN_DELTA = 12   # 末尾に ] ) を足す程度は許容\nMAX_NEW_TOKENS = 128\n\ndef _alpha_tokens_lower(s: str):\n    return [t.lower() for t in re.findall(r\"[A-Za-z]+(?:'[A-Za-z]+)?\", s)]\n\ndef is_safe_edit(orig: str, edited: str) -> bool:\n    orig = basic_normalize(orig)\n    edited = basic_normalize(edited)\n\n    # 0) empty / very short\n    if len(edited) < 3:\n        return False\n\n    # 1) words must be identical (case-insensitive) to avoid BLEU drop\n    if _alpha_tokens_lower(orig) != _alpha_tokens_lower(edited):\n        return False\n\n    # 2) do not change numbers\n    if re.findall(r\"\\d+\", orig) != re.findall(r\"\\d+\", edited):\n        return False\n\n    # 3) bracket safety: allow only adding missing closers at the END\n    if orig.count(\"[\") != edited.count(\"[\"):\n        return False\n    if orig.count(\"(\") != edited.count(\"(\"):\n        return False\n    if edited.count(\"]\") < orig.count(\"]\"):\n        return False\n    if edited.count(\")\") < orig.count(\")\"):\n        return False\n\n    # 4) preserve key terms: if existed, must remain\n    for term in PRESERVE_TERMS:\n        if term in orig and term not in edited:\n            return False\n\n    # 5) string similarity\n    sim = difflib.SequenceMatcher(None, orig, edited).ratio()\n    if sim < SIMILARITY_MIN:\n        return False\n\n    # 6) length delta guard\n    if abs(len(orig) - len(edited)) > MAX_ABS_LEN_DELTA:\n        return False\n\n    return True\n\ndef clean_llm_output(s: str) -> str:\n    s = str(s).strip()\n\n    # remove common preambles\n    s = re.sub(r\"^(Sure|Here(?:'s| is)|Corrected(?: text)?):\\s*\", \"\", s, flags=re.IGNORECASE).strip()\n\n    # code fences\n    if \"```\" in s:\n        s = re.sub(r\"```.*?\\n\", \"\", s, flags=re.DOTALL)\n        s = s.replace(\"```\", \"\").strip()\n\n    # Gemma artifact\n    if \"model\\n\" in s:\n        s = s.split(\"model\\n\")[-1].strip()\n\n    # outer quotes\n    if len(s) >= 2 and s.startswith('\"') and s.endswith('\"'):\n        s = s[1:-1].strip()\n\n    # one-line\n    s = \" \".join(s.splitlines()).strip()\n    return s\n\n\n# ====== 反復・重複の軽い修正（編集しすぎない範囲） ======\n# 目的: BLEU/chrFを壊しやすい \"暴走\"（末尾の繰り返し等）だけを抑える\n# 注意: 触りすぎると逆に落ちるので、デフォは保守的にしてあります。\n\nFUNC_WORDS = {\n    'the','a','an','of','to','and','in','on','for','with','at','by','from','as',\n    'is','are','was','were','be','been','being','that','this','these','those',\n    'it','its','his','her','their','your','my','our','or','not','no'\n}\n\ndef _token_core_lower(t: str) -> str:\n    m = re.match(r\"^(\\W*)(.*?)(\\W*)$\", t)\n    core = m.group(2)\n    return core.lower()\n\ndef dedup_consecutive_tokens(tokens, mode: str = 'function_only'):\n    out = []\n    prev_core = None\n    prev_punct_score = 0\n\n    for t in tokens:\n        m = re.match(r\"^(\\W*)(.*?)(\\W*)$\", t)\n        pre, core, suf = m.group(1), m.group(2), m.group(3)\n        core_l = core.lower()\n        punct_score = len(pre) + len(suf)\n\n        if prev_core is not None and core_l and core_l == prev_core:\n            if (mode == 'all') or (core_l in FUNC_WORDS):\n                # keep the version that has punctuation if useful (e.g., \"the the,\" -> \"the,\")\n                if punct_score > prev_punct_score and out:\n                    out[-1] = t\n                    prev_punct_score = punct_score\n                continue\n\n        out.append(t)\n        prev_core = core_l if core_l else None\n        prev_punct_score = punct_score\n\n    return out\n\ndef remove_repeated_suffix(tokens, min_k=3, max_k=12):\n    # remove duplicated tail segment: ... X Y Z X Y Z\n    changed = False\n    while True:\n        n = len(tokens)\n        found = False\n        max_k_eff = min(max_k, n // 2)\n        for k in range(max_k_eff, min_k - 1, -1):\n            if tokens[n-2*k:n-k] == tokens[n-k:n]:\n                seg = tokens[n-k:n]\n                # require some alphabetic content to avoid stripping punctuation-only repeats\n                if sum(bool(re.search(r\"[A-Za-z]\", x)) for x in seg) >= 2:\n                    tokens = tokens[:n-k]\n                    changed = True\n                    found = True\n                    break\n        if not found:\n            break\n    return tokens, changed\n\ndef repeat_cleanup(s: str):\n    \"\"\"Return (cleaned, changed_flag).\"\"\"\n    if not ('USE_REPEAT_CLEANUP' in globals() and USE_REPEAT_CLEANUP):\n        return s, False\n\n    toks = str(s).split()\n    if len(toks) < 2:\n        return s, False\n\n    mode = DEDUP_MODE if ('DEDUP_MODE' in globals()) else 'function_only'\n    toks2 = dedup_consecutive_tokens(toks, mode=mode)\n    changed = (toks2 != toks)\n\n    if ('REMOVE_REPEATED_SUFFIX' in globals()) and REMOVE_REPEATED_SUFFIX and len(toks2) >= 6:\n        min_k = REPEAT_SUFFIX_MIN_K if ('REPEAT_SUFFIX_MIN_K' in globals()) else 3\n        max_k = REPEAT_SUFFIX_MAX_K if ('REPEAT_SUFFIX_MAX_K' in globals()) else 12\n        toks3, ch = remove_repeated_suffix(toks2, min_k=min_k, max_k=max_k)\n        changed = changed or ch\n    else:\n        toks3 = toks2\n\n    return ' '.join(toks3), changed\n\n# ====== 実行 ======\noriginal_texts = all_predictions\ntranslits = (\n    test_df[\"transliteration\"].astype(str).tolist()\n    if \"transliteration\" in test_df.columns else [\"\"] * len(original_texts)\n)\n\n\n# Precompute near-duplicate TM matches (vectorized)\nnear_dup_best = None\nif USE_TRAIN_NEAR_DUP and (near_dup_vec is not None) and (near_dup_nn is not None):\n    try:\n        test_keys = [norm_src_for_match(s) for s in translits]\n        X_test = near_dup_vec.transform(test_keys)\n        dists, idxs = near_dup_nn.kneighbors(X_test, n_neighbors=1)\n        sims = 1.0 - dists.ravel()\n        idxs = idxs.ravel()\n\n        near_dup_best = []\n        n_ok = 0\n        for s, j, sim in zip(test_keys, idxs, sims):\n            if (len(s) >= NEAR_DUP_MIN_SRC_LEN) and (sim >= NEAR_DUP_SIM_THRESHOLD):\n                near_dup_best.append(near_dup_tgts[int(j)])\n                n_ok += 1\n            else:\n                near_dup_best.append(None)\n        print(f\"🔁 Near-dup matches: {n_ok} / {len(test_keys)} (thr={NEAR_DUP_SIM_THRESHOLD})\")\n\n    except Exception as e:\n        print('⚠️ Near-dup precompute failed:', repr(e))\n        near_dup_best = None\n\npolished_texts = []\ncache = {}  # deterministic decode前提でキャッシュが効く\n\nuse_llm = bool(USE_LLM_POLISH) and (llm_model is not None) and (llm_tokenizer is not None)\n\nprint(f\"🧹 Post-processing {len(original_texts)} sentences | OA Lexicon={USE_OA_LEXICON} | LLM={use_llm}\")\n\nn_tm_exact = 0\nn_tm_loose = 0\nn_tm_near = 0\nn_rep_fix = 0\n\nfor i, (src, text) in enumerate(tqdm(zip(translits, original_texts), total=len(original_texts))):\n    text = str(text)\n\n    # Translation memory exact match override (very safe if duplicates exist)\n    if USE_TRAIN_EXACT_MATCH and train_exact_map:\n        k = norm_src_for_match(src)\n        if k in train_exact_map:\n            polished_texts.append(train_exact_map[k])\n            n_tm_exact += 1\n            continue\n\n    # Translation memory loose-exact match (format-insensitive, safety-checked)\n    if ('USE_TRAIN_LOOSE_MATCH' in globals()) and USE_TRAIN_LOOSE_MATCH and ('train_loose_map' in globals()) and train_loose_map:\n        lk = norm_src_loose(src)\n        if lk in train_loose_map:\n            polished_texts.append(train_loose_map[lk])\n            n_tm_loose += 1\n            continue\n\n    # Near-duplicate TM override (only if exact match did not trigger)\n    if USE_TRAIN_NEAR_DUP and near_dup_best is not None:\n        nd = near_dup_best[i]\n        if nd is not None:\n            polished_texts.append(nd)\n            n_tm_near += 1\n            continue\n\n\n    if len(text) < 5 or text == \"broken text\":\n        polished_texts.append(text)\n        continue\n\n    # まず安全な正規化\n    norm = basic_normalize(text)\n\n    # obvious repetition fixes (very conservative)\n    if ('USE_REPEAT_CLEANUP' in globals()) and USE_REPEAT_CLEANUP:\n        norm2, _changed_rep = repeat_cleanup(norm)\n        norm = norm2\n        if _changed_rep:\n            n_rep_fix += 1\n\n    # OA Lexicon: proper noun normalization (very safe, no paraphrase)\n    if USE_OA_LEXICON:\n        norm = post_process_with_oa_lexicon(src, norm, threshold=OA_THRESHOLD)\n\n    # LLM無効ならここで確定\n    if not use_llm:\n        polished_texts.append(norm)\n        continue\n\n    # LLM不要ならここで確定（事故率と時間を下げる）\n    if not needs_polish(norm):\n        polished_texts.append(norm)\n        continue\n\n    # cache\n    if norm in cache:\n        polished_texts.append(cache[norm])\n        continue\n\n    messages = make_gemma3_prompt(norm)\n\n    prompt_text = llm_tokenizer.apply_chat_template(\n        messages,\n        tokenize=False,\n        add_generation_prompt=True\n    )\n    inputs = llm_tokenizer(prompt_text, return_tensors=\"pt\").to(llm_model.device)\n    input_len = inputs[\"input_ids\"].shape[1]\n\n    with torch.inference_mode():\n        outputs = llm_model.generate(\n            **inputs,\n            max_new_tokens=MAX_NEW_TOKENS,\n            do_sample=False,\n            num_beams=1,\n            repetition_penalty=1.05,\n            eos_token_id=llm_tokenizer.eos_token_id,\n            pad_token_id=llm_tokenizer.eos_token_id,\n        )\n\n    generated_tokens = outputs[0][input_len:]\n    response = llm_tokenizer.decode(generated_tokens, skip_special_tokens=True)\n    response = clean_llm_output(response)\n\n    # 強いゲート：危なければ norm を採用\n    if not is_safe_edit(norm, response):\n        response = norm\n    else:\n        response = basic_normalize(response)\n        if USE_OA_LEXICON:\n            response = post_process_with_oa_lexicon(src, response, threshold=OA_THRESHOLD)\n\n    cache[norm] = response\n    polished_texts.append(response)\n\nprint(f\"TM hits: exact={n_tm_exact} | loose={n_tm_loose} | near={n_tm_near} | rep_fix={n_rep_fix}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": polished_texts\n})\n\nsubmission[\"translation\"] = submission[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission file saved successfully!\")\nsubmission.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}