{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":15163650,"datasetId":9710096,"databundleVersionId":16055049},{"sourceType":"datasetVersion","sourceId":15313069,"datasetId":9794222,"databundleVersionId":16218378}],"dockerImageVersionId":31287,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install sacrebleu --no-index --find-links=/kaggle/input/datasets/seungjunleeofficial/sacrebleu-offline/sacrebleu_wheels","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nDeep Past Challenge — Improved Inference Pipeline (Steps 1–5)\n=============================================================\n\nPipeline overview:\n  Step 1: Exact string match TM (literal equality after normalization)\n  Step 2: Near-duplicate TM (char TF-IDF >= threshold)\n  Step 3: ByT5 generates 3 candidates → score & pick best\n  Step 4: Basic rule-based cleanup (punctuation, brackets, whitespace)\n  Step 5: OA Lexicon proper noun normalization (learn canonical spellings from train)\n\nUsage:\n  - Edit the PATHS section below to match your Kaggle environment\n  - Run all cells in order\n\"\"\"\n\n# =========================================================\n# INSTALL DEPENDENCIES\n# =========================================================\n# !pip install sacrebleu --no-index --find-links=/kaggle/input/datasets/seungjunleeofficial/sacrebleu-offline/sacrebleu_wheels\n\nimport os\nimport re\nimport unicodedata\nfrom pathlib import Path\nfrom collections import Counter, defaultdict\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport sacrebleu\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import linear_kernel\n\n# =========================================================\n# PATHS — EDIT THESE\n# =========================================================\nMODEL_DIR = Path(\"/kaggle/input/datasets/seungjunleeofficial/stage2-training-best/content/byt5_stage2_checkpoints/checkpoint-1680\")\nCOMP_DIR = Path(\"/kaggle/input/competitions/deep-past-initiative-machine-translation\")\n\nTRAIN_PATH = COMP_DIR / \"train.csv\"\nTEST_PATH = COMP_DIR / \"test.csv\"\nSAMPLE_SUB_PATH = COMP_DIR / \"sample_submission.csv\"\nPUBLISHED_PATH = COMP_DIR / \"published_texts.csv\"\nOA_LEXICON_PATH = COMP_DIR / \"OA_Lexicon_eBL.csv\"\nEBL_DICT_PATH = COMP_DIR / \"eBL_Dictionary.csv\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =========================================================\n# DEVICE\n# =========================================================\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(\"DEVICE:\", DEVICE)\n\n# =========================================================\n# HYPERPARAMETERS\n# =========================================================\nMAX_SOURCE_LEN = 512\nMAX_TARGET_LEN = 192\nEVAL_BS = 16\nGEN_NUM_BEAMS = 6\nGEN_LEN_PENALTY = 0.9\n\n# Step 1: Exact match TM\nUSE_EXACT_MATCH_TM = True\n\n# Step 2: Near-duplicate TM\nUSE_NEAR_DUP_TM = True\nNEAR_DUP_THRESHOLD = 0.997\n\n# Step 5: OA Lexicon proper noun normalization\nUSE_OA_LEXICON_POSTPROCESS = True\nOA_MIN_SURFACE_FREQ = 3          # only use spellings appearing >= this many times in train\nOA_MIN_SURFACE_FREQ_NE = 2       # for explicit NE types (DN/GN/PN), allow rarer spellings\nOA_MIN_FOLD_LEN = 4              # minimum folded key length to consider\n\n# =========================================================\n# TEXT NORMALIZATION UTILS\n# =========================================================\nSPACE_RE = re.compile(r\"\\s+\")\nMULTI_HYPHEN_RE = re.compile(r\"-{2,}\")\nACCENT_REMAP = str.maketrans({\n    \"\\u201c\": '\"', \"\\u201d\": '\"', \"\\u2018\": \"'\", \"\\u2019\": \"'\",\n    \"\\u2014\": \"-\", \"\\u2013\": \"-\", \"\\u2011\": \"-\"\n})\n\ndef normalize_source_text(x):\n    if pd.isna(x):\n        return \"\"\n    x = str(x).translate(ACCENT_REMAP)\n    x = SPACE_RE.sub(\" \", x).strip()\n    x = MULTI_HYPHEN_RE.sub(\"-\", x)\n    return x\n\ndef normalize_target_text(x):\n    if pd.isna(x):\n        return \"\"\n    x = str(x).translate(ACCENT_REMAP)\n    x = SPACE_RE.sub(\" \", x).strip()\n    return x\n\ndef norm_src_for_exact_match(s):\n    \"\"\"Normalize source text for exact-match TM lookup.\"\"\"\n    s = normalize_source_text(s)\n    s = re.sub(r\"\\s+\", \" \", s).strip()\n    return s\n\ndef find_col(df, candidates, required=False):\n    if df is None:\n        if required:\n            raise ValueError(\"DataFrame is None\")\n        return None\n    lower = {c.lower(): c for c in df.columns}\n    for cand in candidates:\n        if cand.lower() in lower:\n            return lower[cand.lower()]\n    for c in df.columns:\n        cl = c.lower()\n        for cand in candidates:\n            if cand.lower() in cl:\n                return c\n    if required:\n        raise ValueError(f\"Could not find any of {candidates} in {list(df.columns)}\")\n    return None\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# =========================================================\n# LOAD DATA\n# =========================================================\nprint(\"\\n\" + \"=\"*60)\nprint(\"LOADING DATA\")\nprint(\"=\"*60)\n\ntrain_df = pd.read_csv(TRAIN_PATH)\ntest_df = pd.read_csv(TEST_PATH)\nsub_df = pd.read_csv(SAMPLE_SUB_PATH)\npublished_df = pd.read_csv(PUBLISHED_PATH) if PUBLISHED_PATH.exists() else None\noa_lex_df = pd.read_csv(OA_LEXICON_PATH) if OA_LEXICON_PATH.exists() else None\nebl_dict_df = pd.read_csv(EBL_DICT_PATH) if EBL_DICT_PATH.exists() else None\n\nID_COL = find_col(train_df, [\"oare_id\", \"id\"], required=True)\nSRC_COL = find_col(train_df, [\"transliteration\", \"source\", \"src\", \"text\"], required=True)\nTGT_COL = find_col(train_df, [\"translation\", \"target\", \"english\"], required=True)\nTEST_ID_COL = find_col(test_df, [ID_COL, \"oare_id\", \"id\"], required=True)\nTEST_SRC_COL = find_col(test_df, [SRC_COL, \"transliteration\", \"source\", \"src\", \"text\"], required=True)\n\ntrain_df[SRC_COL] = train_df[SRC_COL].map(normalize_source_text)\ntrain_df[TGT_COL] = train_df[TGT_COL].map(normalize_target_text)\ntest_df[TEST_SRC_COL] = test_df[TEST_SRC_COL].map(normalize_source_text)\n\nif published_df is not None:\n    pub_src_col = find_col(published_df, [\"transliteration\", \"transliteration_orig\", \"text\"], required=False)\n    if pub_src_col is not None:\n        published_df[pub_src_col] = published_df[pub_src_col].map(normalize_source_text)\n\nprint(f\"  train: {train_df.shape}\")\nprint(f\"  test:  {test_df.shape}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Build model inputs (must match training config: meta_cols = [])\nmeta_cols = []\n\ndef build_model_input(row, src_col, meta_cols):\n    parts = []\n    for c in meta_cols:\n        v = row.get(c, None)\n        if pd.notna(v):\n            s = str(v).strip()\n            if s:\n                parts.append(f\"{c.lower()}: {s}\")\n    prefix = \" | \".join(parts)\n    src = str(row[src_col]).strip()\n    if prefix:\n        return f\"translate Akkadian to English || {prefix} || text: {src}\"\n    return f\"translate Akkadian to English || text: {src}\"\n\ntest_aug = test_df.copy()\ntest_aug[\"model_input\"] = test_aug.apply(\n    lambda r: build_model_input(r, TEST_SRC_COL, meta_cols), axis=1\n)\n\n# =========================================================\n# STEP 1: EXACT STRING MATCH TM\n# =========================================================\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 1: Building Exact-Match Translation Memory\")\nprint(\"=\"*60)\n\nexact_match_map = {}\n\nif USE_EXACT_MATCH_TM:\n    # For each unique source in train, pick the most frequent translation\n    src_tgt_counter = defaultdict(Counter)\n    for _, row in train_df.iterrows():\n        src_key = norm_src_for_exact_match(row[SRC_COL])\n        tgt = str(row[TGT_COL]).strip()\n        if src_key and tgt:\n            src_tgt_counter[src_key][tgt] += 1\n\n    for src_key, counter in src_tgt_counter.items():\n        exact_match_map[src_key] = counter.most_common(1)[0][0]\n\n    print(f\"  Exact-match TM entries: {len(exact_match_map)}\")\n\n# =========================================================\n# STEP 2: NEAR-DUPLICATE TM (TF-IDF based)\n# =========================================================\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 2: Building Retrieval Index + Near-Dup TM\")\nprint(\"=\"*60)\n\nretrieval_corpus = []\nretrieval_meta = []\n\nfor _, row in train_df.iterrows():\n    retrieval_corpus.append(str(row[SRC_COL]))\n    retrieval_meta.append({\n        \"kind\": \"train\",\n        \"src\": str(row[SRC_COL]),\n        \"tgt\": str(row[TGT_COL]),\n        \"id\": row.get(ID_COL, None)\n    })\n\nif published_df is not None:\n    pub_src_col = find_col(published_df, [\"transliteration\", \"transliteration_orig\", \"text\"], required=False)\n    pub_tgt_col = find_col(published_df, [\"translation\", \"english\", \"target\"], required=False)\n    pub_id_col = find_col(published_df, [\"oare_id\", \"oare_text_id\", \"id\", \"text_id\"], required=False)\n    if pub_src_col is not None:\n        for _, row in published_df.iterrows():\n            src = row.get(pub_src_col, None)\n            if pd.notna(src) and str(src).strip():\n                retrieval_corpus.append(str(src))\n                retrieval_meta.append({\n                    \"kind\": \"published\",\n                    \"src\": str(src),\n                    \"tgt\": str(row.get(pub_tgt_col, \"\")) if pub_tgt_col is not None else \"\",\n                    \"id\": row.get(pub_id_col, None) if pub_id_col is not None else None\n                })\n\nprint(f\"  Retrieval corpus size: {len(retrieval_corpus)}\")\n\nvectorizer = TfidfVectorizer(\n    analyzer=\"char_wb\",\n    ngram_range=(3, 6),\n    min_df=2,\n    sublinear_tf=True,\n)\nretrieval_matrix = vectorizer.fit_transform(retrieval_corpus)\n\n\ndef retrieve_neighbors(query, top_k=10):\n    q = vectorizer.transform([normalize_source_text(query)])\n    sims = linear_kernel(q, retrieval_matrix).ravel()\n    idx = np.argsort(-sims)[:top_k]\n    rows = []\n    for i in idx:\n        item = dict(retrieval_meta[i])\n        item[\"sim\"] = float(sims[i])\n        rows.append(item)\n    return pd.DataFrame(rows)\n\n\ndef tm_predict_near_dup(query, threshold=0.997):\n    \"\"\"Near-duplicate TM: returns (translation, similarity, meta) or (None, sim, meta).\"\"\"\n    nbs = retrieve_neighbors(query, top_k=1)\n    if len(nbs) == 0:\n        return None, 0.0, None\n    row = nbs.iloc[0]\n    if row[\"kind\"] == \"train\" and row[\"sim\"] >= threshold and str(row[\"tgt\"]).strip():\n        return str(row[\"tgt\"]).strip(), float(row[\"sim\"]), row.to_dict()\n    return None, float(row[\"sim\"]), row.to_dict()\n\n\n# =========================================================\n# STEP 5 SETUP: OA LEXICON PROPER NOUN NORMALIZATION\n# (built here so it's ready when we need it at inference)\n# =========================================================\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 5 SETUP: Building OA Lexicon Proper Noun Normalizer\")\nprint(\"=\"*60)\n\n# --- 5a: Diacritic/folding helpers ---\nSUB_DIGITS = str.maketrans(\"\\u2080\\u2081\\u2082\\u2083\\u2084\\u2085\\u2086\\u2087\\u2088\\u2089\",\n                           \"0123456789\")\n_DIACRITIC_MAP = str.maketrans({\n    \"\\u0161\": \"s\", \"\\u0160\": \"s\",   # š Š\n    \"\\u1E63\": \"s\", \"\\u1E62\": \"s\",   # ṣ Ṣ\n    \"\\u1E6D\": \"t\", \"\\u1E6C\": \"t\",   # ṭ Ṭ\n    \"\\u1E2B\": \"h\", \"\\u1E2A\": \"h\",   # ḫ Ḫ\n    \"\\u0101\": \"a\", \"\\u0100\": \"a\",   # ā Ā\n    \"\\u0113\": \"e\", \"\\u0112\": \"e\",   # ē Ē\n    \"\\u012B\": \"i\", \"\\u012A\": \"i\",   # ī Ī\n    \"\\u016B\": \"u\", \"\\u016A\": \"u\",   # ū Ū\n    \"\\u02BE\": \"\", \"\\u02BC\": \"\",     # ʾ ʼ\n    \"\\u2018\": \"\", \"\\u2019\": \"\",     # ' '\n})\n\n_DIACRITIC_CHARS = set(\"šŠṣṢṭṬḫḪāēīūĀĒĪŪ\")\n\nEXPLICIT_NE_TYPES = {\"DN\", \"GN\", \"PN\", \"MN\", \"ON\", \"TN\"}\n\nEN_STOPWORDS = {\n    'the','a','an','and','or','of','to','in','on','at','by','for','from','with',\n    'as','but','not','no','nor','is','are','was','were','be','been','being',\n    'i','you','he','she','it','we','they','me','him','her','us','them',\n    'my','your','his','their','our','its','this','that','these','those',\n    'there','here','who','whom','which','what','when','where','why','how',\n}\n\n\ndef norm_key_token(s):\n    \"\"\"Key for matching transliteration tokens to lexicon tokens.\"\"\"\n    s = \"\" if s is None else str(s)\n    s = unicodedata.normalize(\"NFKC\", s).translate(SUB_DIGITS).strip()\n    s = re.sub(r'^[\"\\'\\u201c\\u201d\\u2018\\u2019()\\[\\]{}<>]+', \"\", s)\n    s = re.sub(r'[\"\\'\\u201c\\u201d\\u2018\\u2019()\\[\\]{}<>]+$', \"\", s)\n    s = s.strip(\".,;:!?\")\n    return s.lower()\n\n\ndef strip_disambig(s):\n    \"\"\"Remove trailing numeric homograph markers: Inanna2 -> Inanna.\"\"\"\n    s = \"\" if s is None else str(s)\n    s = unicodedata.normalize(\"NFKC\", s)\n    s = re.sub(r\"(?<=\\D)\\d+$\", \"\", s)\n    return s\n\n\ndef fold_for_match(s):\n    \"\"\"Aggressively fold a string for name matching (remove diacritics etc.).\"\"\"\n    s = \"\" if s is None else str(s)\n    s = unicodedata.normalize(\"NFKC\", s)\n    s = strip_disambig(s)\n    s = s.translate(_DIACRITIC_MAP)\n    s = s.lower()\n    s = s.replace(\"sh\", \"s\").replace(\"kh\", \"h\")\n    s = re.sub(r\"[^a-z]+\", \"\", s)\n    return s\n\n\ndef looks_like_name(lexeme, typ):\n    \"\"\"Heuristic: does this lexicon entry look like a proper name?\"\"\"\n    if not lexeme:\n        return False\n    t = (typ or \"\").strip().upper()\n    if t in EXPLICIT_NE_TYPES:\n        return True\n    if any(ch.isupper() for ch in lexeme):\n        return True\n    if any(ch in _DIACRITIC_CHARS for ch in lexeme):\n        return True\n    return False\n\n\n# --- 5b: Build token -> lexeme index from OA Lexicon + eBL Dictionary ---\ntoken2lexemes = defaultdict(list)\n\nif USE_OA_LEXICON_POSTPROCESS and oa_lex_df is not None:\n    for _, r in oa_lex_df.iterrows():\n        typ = \"\" if pd.isna(r.get(\"type\")) else str(r[\"type\"]).strip()\n        lex = \"\" if pd.isna(r.get(\"lexeme\")) else str(r[\"lexeme\"]).strip()\n        if not lex:\n            continue\n        for col in [\"form\", \"norm\", \"Alt_lex\"]:\n            if col not in oa_lex_df.columns:\n                continue\n            v = r.get(col)\n            if pd.isna(v):\n                continue\n            for tok in str(v).split():\n                k = norm_key_token(tok)\n                if k:\n                    token2lexemes[k].append((lex, typ))\n\n    # De-duplicate\n    for k in list(token2lexemes.keys()):\n        seen = set()\n        uniq = []\n        for lex, typ in token2lexemes[k]:\n            key = (lex, typ)\n            if key not in seen:\n                seen.add(key)\n                uniq.append((lex, typ))\n        token2lexemes[k] = uniq\n\n    print(f\"  OA Lexicon token keys indexed: {len(token2lexemes)}\")\nelse:\n    print(\"  OA Lexicon postprocessing disabled or file not found\")\n\n\n# --- 5c: Learn canonical name spellings from training translations ---\nfold2surface = {}   # fold_key -> most common surface form in train translations\nfold2freq = {}      # fold_key -> count of that surface form\n\nif USE_OA_LEXICON_POSTPROCESS:\n    train_tgt_col = TGT_COL\n    surf_counter = defaultdict(Counter)\n    token_re = re.compile(r\"[A-Za-z\\u0161\\u0160\\u1E63\\u1E62\\u1E6D\\u1E6C\\u1E2B\\u1E2A\\u0101\\u0113\\u012B\\u016B\\u0100\\u0112\\u012A\\u016A\\u2018\\u2019\\-]+\")\n\n    for text in train_df[train_tgt_col].astype(str).tolist():\n        for tok in token_re.findall(text):\n            if len(tok) < 3:\n                continue\n            # Focus on tokens that look like proper nouns\n            if not (tok[0].isupper() or any(ch in _DIACRITIC_CHARS for ch in tok)):\n                continue\n            f = fold_for_match(tok)\n            if len(f) < OA_MIN_FOLD_LEN:\n                continue\n            surf_counter[f][tok] += 1\n\n    for f, counter in surf_counter.items():\n        tok, cnt = counter.most_common(1)[0]\n        fold2surface[f] = tok\n        fold2freq[f] = cnt\n\n    print(f\"  Learned canonical name spellings from train: {len(fold2surface)} entries\")\n\n\n# --- 5d: Functions to apply name normalization ---\ndef extract_name_targets(translit):\n    \"\"\"From source transliteration, find proper names via OA Lexicon\n    and return {fold_key: canonical_surface_form} for names found in train.\"\"\"\n    if not USE_OA_LEXICON_POSTPROCESS:\n        return {}\n\n    translit = \"\" if translit is None else str(translit)\n    targets = {}\n    seen = set()\n\n    for tok in translit.split():\n        k = norm_key_token(tok)\n        for lex, typ in token2lexemes.get(k, []):\n            if lex in seen:\n                continue\n            seen.add(lex)\n\n            if not looks_like_name(lex, typ):\n                continue\n\n            lex_clean = strip_disambig(lex)\n            f = fold_for_match(lex_clean)\n            if len(f) < OA_MIN_FOLD_LEN:\n                continue\n\n            # Determine minimum frequency based on NE type\n            t = (typ or \"\").strip().upper()\n            min_freq = OA_MIN_SURFACE_FREQ_NE if t in EXPLICIT_NE_TYPES else OA_MIN_SURFACE_FREQ\n\n            if f in fold2surface and fold2freq.get(f, 0) >= min_freq:\n                targets[f] = fold2surface[f]\n\n        if len(targets) >= 50:\n            break\n\n    return targets\n\n\ndef normalize_proper_nouns(pred, targets):\n    \"\"\"Replace name variants in prediction with canonical spellings from train.\"\"\"\n    if not pred or not targets:\n        return pred\n\n    parts = str(pred).split()\n    out = []\n\n    for p in parts:\n        m = re.match(r\"^(\\W*)(.*?)(\\W*)$\", p)\n        pre, core, suf = m.group(1), m.group(2), m.group(3)\n\n        if not core:\n            out.append(p)\n            continue\n\n        # Handle possessive endings: Assur's\n        poss = \"\"\n        core_base = core\n        pm = re.match(r\"^(.*?)([\\u2018\\u2019']s)$\", core)\n        if pm:\n            core_base = pm.group(1)\n            poss = pm.group(2)\n\n        f = fold_for_match(core_base)\n        if len(f) < OA_MIN_FOLD_LEN:\n            out.append(p)\n            continue\n\n        is_cap = core_base[:1].isupper()\n\n        # Match: replace with canonical spelling\n        if f in targets:\n            if is_cap:\n                out.append(pre + targets[f] + poss + suf)\n                continue\n            # Also fix lowercased proper names if long enough and not a stopword\n            if len(f) >= OA_MIN_FOLD_LEN and core_base.lower() not in EN_STOPWORDS:\n                out.append(pre + targets[f] + poss + suf)\n                continue\n\n        out.append(p)\n\n    return \" \".join(out)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# =========================================================\n# STEP 3 SETUP: LOAD MODEL\n# =========================================================\nprint(\"\\n\" + \"=\"*60)\nprint(\"STEP 3 SETUP: Loading ByT5 Model\")\nprint(\"=\"*60)\n\ntokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_DIR).to(DEVICE)\nmodel.eval()\nprint(f\"  Loaded model from: {MODEL_DIR}\")\n\n\ndef generate_candidates(texts, batch_size=8, num_beams=6, num_return_sequences=3):\n    preds = []\n    for i in range(0, len(texts), batch_size):\n        batch = texts[i:i+batch_size]\n        toks = tokenizer(\n            batch,\n            return_tensors=\"pt\",\n            padding=True,\n            truncation=True,\n            max_length=MAX_SOURCE_LEN,\n        ).to(DEVICE)\n\n        with torch.no_grad():\n            out = model.generate(\n                **toks,\n                max_length=MAX_TARGET_LEN,\n                num_beams=num_beams,\n                num_return_sequences=num_return_sequences,\n                early_stopping=True,\n                length_penalty=GEN_LEN_PENALTY,\n            )\n\n        dec = tokenizer.batch_decode(out, skip_special_tokens=True)\n        for j in range(0, len(dec), num_return_sequences):\n            preds.append(dec[j:j+num_return_sequences])\n\n    return preds\n\n\ndef choose_candidate(src_text, candidates):\n    \"\"\"Rerank candidates using retrieval similarity (chrF++ overlap with neighbors).\"\"\"\n    nbs = retrieve_neighbors(src_text, top_k=3)\n    scored = []\n\n    for cand in candidates:\n        cand = normalize_target_text(cand)\n        score = 0.0\n\n        for _, row in nbs.iterrows():\n            tgt = str(row.get(\"tgt\", \"\")).strip()\n            if tgt:\n                overlap = sacrebleu.sentence_chrf(cand, [tgt], word_order=2).score / 100.0\n                score += overlap * row[\"sim\"]\n\n        scored.append((score, cand))\n\n    scored.sort(reverse=True)\n    return scored[0][1]\n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# =========================================================\n# STEP 4: RULE-BASED CLEANUP FUNCTION\n# =========================================================\n_DASH_NORM = str.maketrans({\"\\u2013\": \"-\", \"\\u2014\": \"-\", \"\\u2212\": \"-\"})\n_QUOTE_NORM = str.maketrans({\"\\u201c\": '\"', \"\\u201d\": '\"', \"\\u2018\": \"'\", \"\\u2019\": \"'\"})\n\n\ndef rule_based_cleanup(text):\n    \"\"\"Step 4: fix punctuation spacing, brackets, quotes, whitespace.\"\"\"\n    text = str(text)\n\n    # Normalize dashes and quotes to ASCII\n    text = text.translate(_DASH_NORM).translate(_QUOTE_NORM)\n\n    # Collapse multiple spaces/tabs\n    text = re.sub(r\"[ \\t]+\", \" \", text)\n\n    # Remove space before punctuation: \"word ,\" -> \"word,\"\n    text = re.sub(r\"\\s+([,.;:!?])\", r\"\\1\", text)\n\n    # Ensure space after punctuation before letters: \"word,next\" -> \"word, next\"\n    text = re.sub(r\"([,.;:!?])([A-Za-z])\", r\"\\1 \\2\", text)\n\n    # Fix bracket spacing: \"( word\" -> \"(word\", \"word )\" -> \"word)\"\n    text = re.sub(r\"\\(\\s+\", \"(\", text)\n    text = re.sub(r\"\\s+\\)\", \")\", text)\n    text = re.sub(r\"\\[\\s+\", \"[\", text)\n    text = re.sub(r\"\\s+\\]\", \"]\", text)\n\n    # Fix mismatched brackets: add missing closers at end\n    if text.count(\"[\") > text.count(\"]\"):\n        text += \"]\" * (text.count(\"[\") - text.count(\"]\"))\n    if text.count(\"(\") > text.count(\")\"):\n        text += \")\" * (text.count(\"(\") - text.count(\")\"))\n\n    text = text.strip()\n    return text\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# =========================================================\n# MAIN INFERENCE LOOP (Steps 1–5)\n# =========================================================\nprint(\"\\n\" + \"=\"*60)\nprint(\"RUNNING INFERENCE PIPELINE (Steps 1-5)\")\nprint(\"=\"*60)\n\ntest_inputs = test_aug[\"model_input\"].tolist()\ntest_src_raw = test_aug[TEST_SRC_COL].tolist()\n\n# --- Step 3: Generate candidates for ALL test inputs ---\nprint(\"\\n  Step 3: Generating ByT5 candidates...\")\ntest_candidates = generate_candidates(\n    test_inputs,\n    batch_size=EVAL_BS,\n    num_beams=GEN_NUM_BEAMS,\n    num_return_sequences=3,\n)\nprint(f\"  Generated candidates for {len(test_candidates)} inputs\")\n\n# --- Process each test input through the full pipeline ---\nfinal_preds = []\nstats = {\"exact_tm\": 0, \"near_dup_tm\": 0, \"model\": 0}\n\nprint(\"\\n  Running Steps 1-5 per sentence...\")\nfor i, (src_raw, cands) in enumerate(zip(test_src_raw, test_candidates)):\n\n    # ---- STEP 1: Exact string match TM ----\n    if USE_EXACT_MATCH_TM:\n        exact_key = norm_src_for_exact_match(src_raw)\n        if exact_key in exact_match_map:\n            pred = exact_match_map[exact_key]\n            # Still apply Steps 4 + 5 to TM output\n            pred = rule_based_cleanup(pred)\n            if USE_OA_LEXICON_POSTPROCESS:\n                targets = extract_name_targets(src_raw)\n                pred = normalize_proper_nouns(pred, targets)\n            final_preds.append(pred)\n            stats[\"exact_tm\"] += 1\n            continue\n\n    # ---- STEP 2: Near-duplicate TM ----\n    if USE_NEAR_DUP_TM:\n        tm_pred, sim, meta = tm_predict_near_dup(src_raw, threshold=NEAR_DUP_THRESHOLD)\n        if tm_pred is not None:\n            pred = tm_pred\n            # Still apply Steps 4 + 5\n            pred = rule_based_cleanup(pred)\n            if USE_OA_LEXICON_POSTPROCESS:\n                targets = extract_name_targets(src_raw)\n                pred = normalize_proper_nouns(pred, targets)\n            final_preds.append(pred)\n            stats[\"near_dup_tm\"] += 1\n            continue\n\n    # ---- STEP 3: Pick best candidate via reranking ----\n    best = choose_candidate(src_raw, cands)\n\n    # ---- STEP 4: Rule-based cleanup ----\n    best = rule_based_cleanup(best)\n\n    # ---- STEP 5: OA Lexicon proper noun normalization ----\n    if USE_OA_LEXICON_POSTPROCESS:\n        targets = extract_name_targets(src_raw)\n        best = normalize_proper_nouns(best, targets)\n\n    final_preds.append(best)\n    stats[\"model\"] += 1\n\nprint(f\"\\n  Results:\")\nprint(f\"    Exact TM matches:    {stats['exact_tm']}\")\nprint(f\"    Near-dup TM matches: {stats['near_dup_tm']}\")\nprint(f\"    Model predictions:   {stats['model']}\")\nprint(f\"    Total:               {len(final_preds)}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# =========================================================\n# SUBMISSION\n# =========================================================\nprint(\"\\n\" + \"=\"*60)\nprint(\"SAVING SUBMISSION\")\nprint(\"=\"*60)\n\ntarget_sub_col = find_col(sub_df, [\"translation\", \"target\"], required=False)\nif target_sub_col is None:\n    non_id_cols = [c for c in sub_df.columns if c != TEST_ID_COL]\n    assert len(non_id_cols) >= 1, \"Could not infer submission target column\"\n    target_sub_col = non_id_cols[0]\n\nsubmission = sub_df.copy()\nsubmission[target_sub_col] = final_preds\n\nsubmission_path = \"/kaggle/working/submission.csv\"\nsubmission.to_csv(submission_path, index=False)\n\nprint(submission.head())\nprint(f\"\\nSaved: {submission_path}\")\nprint(\"Done!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}