{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport unicodedata\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter, defaultdict\nfrom difflib import SequenceMatcher\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nUSE_PUBLISHED = False     \nBATCH = 256               \nTOPK = 60                 \nRERANK_K = 10             \nW_CHAR = 0.78\nW_WORD = 0.22\nW_SEQ  = 0.06             \nLEN_PENALTY_POWER = 1.6   \nMIN_ACCEPT_SCORE = 0.16   \nMIN_NNZ = 4               \n\n\ntrain = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest  = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\npublished = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/published_texts.csv')\n\nprint(\"train:\", train.shape, \"test:\", test.shape, \"published:\", published.shape)\n\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T01:15:40.994243Z","iopub.execute_input":"2025-12-17T01:15:40.994708Z","iopub.status.idle":"2025-12-17T01:15:41.294124Z","shell.execute_reply.started":"2025-12-17T01:15:40.994674Z","shell.execute_reply":"2025-12-17T01:15:41.293103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# normalisation\nSUB_MAP = str.maketrans({\"₀\":\"0\",\"₁\":\"1\",\"₂\":\"2\",\"₃\":\"3\",\"₄\":\"4\",\"₅\":\"5\",\"₆\":\"6\",\"₇\":\"7\",\"₈\":\"8\",\"₉\":\"9\"})\n\nSPECIAL = {\n    \"š\":\"sh\", \"Š\":\"sh\",\n    \"ṣ\":\"s2\", \"Ṣ\":\"s2\",\n    \"ṭ\":\"t2\", \"Ṭ\":\"t2\",\n    \"ḫ\":\"h2\", \"Ḫ\":\"h2\",\n    \"ā\":\"aa\",\"Ā\":\"aa\",\n    \"ē\":\"ee\",\"Ē\":\"ee\",\n    \"ī\":\"ii\",\"Ī\":\"ii\",\n    \"ū\":\"uu\",\"Ū\":\"uu\",\n}\n\ndef clean_ws(x: str) -> str:\n    if pd.isna(x): return \"\"\n    return re.sub(r\"\\s+\", \" \", str(x)).strip()\n\ndef strip_combining(s: str) -> str:\n    s = unicodedata.normalize(\"NFKD\", s)\n    return \"\".join(ch for ch in s if not unicodedata.combining(ch))\n\ndef norm_src(x: str) -> str:\n    \"\"\"Normalize transliteration for retrieval.\"\"\"\n    if pd.isna(x): return \"\"\n    x = clean_ws(x).lower()\n    x = x.translate(SUB_MAP)\n\n    # determinatives: {d} {m} {ki} ... => DET_d etc\n    x = re.sub(r\"\\{([^}]+)\\}\", r\" DET_\\1 \", x)\n\n    x = re.sub(r\"\\(([a-z]{1,6})\\)\", r\" DET_\\1 \", x)\n\n    # keep distinction for Akkadian letters\n    for k, v in SPECIAL.items():\n        x = x.replace(k, v)\n\n    x = strip_combining(x)\n\n    # remove apostrophe-like signs\n    x = re.sub(r\"[ʾʿˀˁ']\", \"\", x)\n\n    # gap markers\n    x = x.replace(\"…\", \" GAP \").replace(\"...\", \" GAP \")\n    x = re.sub(r\"\\b[xX]{1,3}\\b\", \" GAP \", x)\n\n    # editorial brackets\n    x = x.replace(\"[\", \" \").replace(\"]\", \" \").replace(\"<\", \" \").replace(\">\", \" \")\n    x = x.replace(\"<<\", \" \").replace(\">>\", \" \")\n\n    # dot separators (e.g. KÙ.AN)\n    x = x.replace(\".\", \" \")\n\n    # plus sometimes appears in sign writing\n    x = x.replace(\"+\", \"\")\n\n    # keep ascii \n    x = re.sub(r\"[^a-z0-9\\-_ ]+\", \" \", x)\n    x = re.sub(r\"\\s+\", \" \", x).strip()\n    return x\n\ndef norm_tgt(x: str) -> str:\n    \"\"\"Light cleanup for English; keep case/punct mostly.\"\"\"\n    if pd.isna(x): return \"\"\n    x = clean_ws(x)\n    x = x.replace(\"“\", '\"').replace(\"”\", '\"').replace(\"’\", \"'\")\n    x = re.sub(r\"\\s+\", \" \", x).strip()\n    return x\n\n\nMETA_PREFIX = re.compile(r\"^\\s*(Seal of|Sealed by|Envelope|Copy of|Impression of)\\b\", re.I)\n\ndef strip_leading_metadata(en: str) -> str:\n    \"\"\"If translation starts with metadata sentence (Seal of ...), drop it if there is more after.\"\"\"\n    en = norm_tgt(en)\n    if not en:\n        return en\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [p.strip() for p in parts if p.strip()]\n    if len(parts) >= 2 and META_PREFIX.search(parts[0]):\n        return \" \".join(parts[1:]).strip()\n    return en\n\ndef make_one_sentence(en: str) -> str:\n    \"\"\"Join multi-sentence text into ONE sentence using '; '.\"\"\"\n    en = norm_tgt(en)\n    if not en:\n        return \"...\"\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [p.strip() for p in parts if p.strip()]\n    # remove sentence-ending punct to avoid multiple sentences\n    parts = [re.sub(r\"[.!?]+\\s*$\", \"\", p).strip() for p in parts]\n    parts = [p for p in parts if p]\n    if not parts:\n        return \"...\"\n    out = \"; \".join(parts).strip()\n    if not out.endswith(\".\"):\n        out += \".\"\n    return out\n\ndef finalize_translation(en: str) -> str:\n    en = strip_leading_metadata(en)\n    return make_one_sentence(en)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T01:15:41.295888Z","iopub.execute_input":"2025-12-17T01:15:41.296235Z","iopub.status.idle":"2025-12-17T01:15:41.313347Z","shell.execute_reply.started":"2025-12-17T01:15:41.296206Z","shell.execute_reply":"2025-12-17T01:15:41.312385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build translation memory \nmem = train[[\"transliteration\", \"translation\"]].copy()\n\nif USE_PUBLISHED and (\"transliteration\" in published.columns) and (\"translation\" in published.columns):\n    mem = pd.concat([mem, published[[\"transliteration\", \"translation\"]]], ignore_index=True)\n\nmem[\"src_norm\"] = mem[\"transliteration\"].map(norm_src)\nmem[\"tgt\"] = mem[\"translation\"].map(norm_tgt)\nmem = mem[(mem[\"src_norm\"] != \"\") & (mem[\"tgt\"] != \"\")].copy()\n\ndef mode_str(vals):\n    c = Counter(vals)\n    return c.most_common(1)[0][0]\n\nmem = mem.groupby(\"src_norm\", as_index=False).agg({\"tgt\": mode_str})\nmem_src = mem[\"src_norm\"].tolist()\nmem_tgt = mem[\"tgt\"].tolist()\nexact_map = dict(zip(mem_src, mem_tgt))\n\nprint(\"Translation memory rows:\", len(mem))\n\nmem_src_len = np.array([max(1, len(s.split())) for s in mem_src], dtype=np.int32)\n\n\nchar_vec = TfidfVectorizer(analyzer=\"char_wb\", ngram_range=(3, 7), min_df=1, max_df=0.98)\nword_vec = TfidfVectorizer(\n    analyzer=\"word\",\n    ngram_range=(1, 2),\n    token_pattern=r\"(?u)\\b[a-z0-9\\-_]+\\b\",\n    min_df=1,\n    max_df=0.98,\n)\n\nXc_mem = char_vec.fit_transform(mem_src)\nXw_mem = word_vec.fit_transform(mem_src)\n\nprint(\"TFIDF vocab sizes — char:\", len(char_vec.vocabulary_), \"word:\", len(word_vec.vocabulary_))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T01:15:41.314345Z","iopub.execute_input":"2025-12-17T01:15:41.314656Z","iopub.status.idle":"2025-12-17T01:15:43.593718Z","shell.execute_reply.started":"2025-12-17T01:15:41.314624Z","shell.execute_reply":"2025-12-17T01:15:43.592801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Backoff dictionary \nsrc_train_norm = train[\"transliteration\"].map(norm_src)\ntgt_train = train[\"translation\"].map(norm_tgt)\n\ntgt_surface = defaultdict(Counter)\nsrc_cnt = Counter()\ntgt_cnt = Counter()\npair_cnt = Counter()\n\nfor s, t in zip(src_train_norm, tgt_train):\n    if not s or not t:\n        continue\n    s_tokens = set(s.split())\n    t_tokens_raw = t.split()\n    t_tokens = set([w.lower() for w in t_tokens_raw])\n\n    for w in t_tokens_raw:\n        tgt_surface[w.lower()][w] += 1\n\n    for sw in s_tokens: src_cnt[sw] += 1\n    for tw in t_tokens: tgt_cnt[tw] += 1\n    for sw in s_tokens:\n        for tw in t_tokens:\n            pair_cnt[(sw, tw)] += 1\n\nbest_word = {}\nbest_score = {}\nfor (sw, tw), c in pair_cnt.items():\n    score = c / np.sqrt(src_cnt[sw] * tgt_cnt[tw])\n    if (sw not in best_score) or (score > best_score[sw]):\n        best_score[sw] = score\n        best_word[sw] = tw\n\nbest_surface = {lw: surf.most_common(1)[0][0] for lw, surf in tgt_surface.items()}\n\ndef dict_backoff_translate(src_norm: str, min_assoc: float = 0.10) -> str:\n    if not src_norm:\n        return \"...\"\n    out = []\n    for sw in src_norm.split():\n        if sw.startswith(\"det_\") or sw == \"gap\":\n            continue\n        tw = best_word.get(sw)\n        if tw is not None and best_score.get(sw, 0.0) >= min_assoc:\n            out.append(best_surface.get(tw, tw))\n        else:\n            out.append(sw)\n    return finalize_translation(\" \".join(out))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T01:15:43.595423Z","iopub.execute_input":"2025-12-17T01:15:43.595703Z","iopub.status.idle":"2025-12-17T01:15:52.581705Z","shell.execute_reply.started":"2025-12-17T01:15:43.595676Z","shell.execute_reply":"2025-12-17T01:15:52.580641Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Retrieval scoring utilities\ndef length_penalty(q_len: int, cand_len: np.ndarray) -> np.ndarray:\n    q_len = max(1, int(q_len))\n    r = cand_len.astype(np.float32) / np.float32(q_len)\n    # symmetric penalty in log space\n    return np.exp(-np.abs(np.log(r)) * np.float32(LEN_PENALTY_POWER)).astype(np.float32)\n\ntest_src_norm = test[\"transliteration\"].map(norm_src).tolist()\npreds = [\"\"] * len(test_src_norm)\n\nneed_rows = []\nfor i, q in enumerate(test_src_norm):\n    if q in exact_map:\n        preds[i] = finalize_translation(exact_map[q])\n    else:\n        need_rows.append(i)\n\nprint(\"Exact matched:\", len(test_src_norm) - len(need_rows), \"Need retrieval:\", len(need_rows))\n\nfor start in range(0, len(need_rows), BATCH):\n    idxs = need_rows[start:start+BATCH]\n    qs = [test_src_norm[i] for i in idxs]\n\n    qC = char_vec.transform(qs)\n    qW = word_vec.transform(qs)\n\n    nnz = qC.getnnz(axis=1) + qW.getnnz(axis=1)\n\n    # compute similarities\n    sc = (qC @ Xc_mem.T).toarray().astype(np.float32)\n    sw = (qW @ Xw_mem.T).toarray().astype(np.float32)\n\n    base = W_CHAR * sc + W_WORD * sw  # shape: (batch, mem)\n\n    for r in range(base.shape[0]):\n        global_i = idxs[r]\n        q = qs[r]\n\n        if nnz[r] < MIN_NNZ or not q:\n            preds[global_i] = dict_backoff_translate(q)\n            continue\n\n        # candidate set\n        row = base[r]\n        if TOPK < len(row):\n            cand = np.argpartition(-row, TOPK)[:TOPK]\n        else:\n            cand = np.arange(len(row))\n\n        q_len = len(q.split())\n        lp = length_penalty(q_len, mem_src_len[cand])\n        score0 = row[cand] * lp\n\n        # take RERANK_K best by score0 then apply SequenceMatcher as tie-break\n        k = min(RERANK_K, len(cand))\n        topk_local = np.argpartition(-score0, k-1)[:k]\n        topcand = cand[topk_local]\n\n        best_i = None\n        best_s = -1.0\n\n        for cidx in topcand:\n            # base score with length penalty\n            s0 = float(row[cidx]) * float(length_penalty(q_len, np.array([mem_src_len[cidx]]))[0])\n            # tie-breaker\n            ss = SequenceMatcher(None, q, mem_src[cidx]).ratio()\n            s = s0 + W_SEQ * ss\n            if s > best_s:\n                best_s = s\n                best_i = int(cidx)\n\n        if best_i is None or best_s < MIN_ACCEPT_SCORE:\n            preds[global_i] = dict_backoff_translate(q)\n        else:\n            preds[global_i] = finalize_translation(mem_tgt[best_i])\n\nprint(\"len(test):\", len(test), \"len(preds):\", len(preds))\nprint(\"Unique preds:\", len(set(preds)))\nprint(\"Most common preds:\", Counter(preds).most_common(3))\n\n\nsub = pd.DataFrame({\"id\": test[\"id\"], \"translation\": preds})\nsub.to_csv(\"submission.csv\", index=False)\nprint(\"Wrote submission.csv\")\nsub.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T01:15:52.582948Z","iopub.execute_input":"2025-12-17T01:15:52.583382Z","iopub.status.idle":"2025-12-17T01:15:52.665847Z","shell.execute_reply.started":"2025-12-17T01:15:52.583354Z","shell.execute_reply":"2025-12-17T01:15:52.664762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preview = pd.DataFrame({\n    \"id\": test[\"id\"],\n    \"original_transliteration\": test[\"transliteration\"],\n    \"predicted_translation\": preds\n})\n\nfor _, r in preview.head(10).iterrows():\n    print(\"=\"*90)\n    print(\"id:\", r[\"id\"])\n    print(\"ORIGINAL:\", r[\"original_transliteration\"])\n    print(\"PREDICT :\", r[\"predicted_translation\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T01:15:52.666988Z","iopub.execute_input":"2025-12-17T01:15:52.667389Z","iopub.status.idle":"2025-12-17T01:15:52.676399Z","shell.execute_reply.started":"2025-12-17T01:15:52.667353Z","shell.execute_reply":"2025-12-17T01:15:52.675232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}