{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14192666,"sourceType":"datasetVersion","datasetId":9049931}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re, unicodedata\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter, defaultdict\nfrom difflib import SequenceMatcher\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import linear_kernel\n\n# --- load data (adjust paths if needed) ---\ntrain = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\ntest  = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n\n# --- normalization utilities (kept simple) ---\nSUB_MAP = str.maketrans({\"₀\":\"0\",\"₁\":\"1\",\"₂\":\"2\",\"₃\":\"3\",\"₄\":\"4\",\"₅\":\"5\",\"₆\":\"6\",\"₇\":\"7\",\"₈\":\"8\",\"₉\":\"9\"})\nSPECIAL = {\"š\":\"sh\",\"Š\":\"sh\",\"ṣ\":\"s2\",\"Ṣ\":\"s2\",\"ṭ\":\"t2\",\"Ṭ\":\"t2\",\"ḫ\":\"h2\",\"Ḫ\":\"h2\",\"ā\":\"aa\",\"Ā\":\"aa\",\"ē\":\"ee\",\"Ē\":\"ee\",\"ī\":\"ii\",\"Ī\":\"ii\",\"ū\":\"uu\",\"Ū\":\"uu\"}\n\ndef clean_ws(x):\n    if pd.isna(x): return \"\"\n    return re.sub(r\"\\s+\", \" \", str(x)).strip()\n\ndef strip_combining(s):\n    s = unicodedata.normalize(\"NFKD\", s)\n    return \"\".join(ch for ch in s if not unicodedata.combining(ch))\n\ndef norm_src(x):\n    if pd.isna(x): return \"\"\n    x = clean_ws(x).lower()\n    x = x.translate(SUB_MAP)\n    x = re.sub(r\"\\{([^}]+)\\}\", r\" DET_\\1 \", x)\n    x = re.sub(r\"\\(([a-z]{1,6})\\)\", r\" DET_\\1 \", x)\n    for k,v in SPECIAL.items(): x = x.replace(k, v)\n    x = strip_combining(x)\n    x = re.sub(r\"[ʾʿˀˁ']\", \"\", x)\n    x = x.replace(\"…\", \" GAP \").replace(\"...\", \" GAP \")\n    x = re.sub(r\"\\b[xX]{1,3}\\b\", \" GAP \", x)\n    x = x.replace(\"[\",\" \").replace(\"]\",\" \").replace(\"<\",\" \").replace(\">\",\" \")\n    x = x.replace(\".\", \" \")\n    x = x.replace(\"+\", \"\")\n    x = re.sub(r\"[^a-z0-9\\-_ ]+\", \" \", x)\n    return re.sub(r\"\\s+\", \" \", x).strip()\n\ndef norm_tgt(x):\n    if pd.isna(x): return \"\"\n    x = clean_ws(x)\n    x = x.replace(\"“\", '\"').replace(\"”\", '\"').replace(\"’\", \"'\")\n    return re.sub(r\"\\s+\", \" \", x).strip()\n\ndef make_one_sentence(en):\n    en = norm_tgt(en)\n    if not en: return \"...\"\n    parts = re.split(r\"(?<=[.!?])\\s+\", en)\n    parts = [re.sub(r\"[.!?]+\\s*$\",\"\",p.strip()) for p in parts if p.strip()]\n    if not parts: return \"...\"\n    out = \"; \".join(parts).strip()\n    if not out.endswith(\".\"): out += \".\"\n    return out\n\ndef finalize_translation(en):\n    return make_one_sentence(en)\n\n# --- build translation memory (TM) ---\nmem = train[[\"transliteration\",\"translation\"]].copy()\nmem[\"src_norm\"] = mem[\"transliteration\"].map(norm_src)\nmem[\"tgt_norm\"] = mem[\"translation\"].map(finalize_translation)\nmem = mem[(mem[\"src_norm\"]!=\"\") & (mem[\"tgt_norm\"]!=\"\")].copy()\n\n# collapse duplicates by most common target\ndef mode_str(vals):\n    c = Counter(vals)\n    return c.most_common(1)[0][0]\n\nmem = mem.groupby(\"src_norm\", as_index=False).agg({\"tgt_norm\": mode_str})\nmem_src = mem[\"src_norm\"].tolist()\nmem_tgt = mem[\"tgt_norm\"].tolist()\nexact_map = dict(zip(mem_src, mem_tgt))\nprint(\"TM rows:\", len(mem))\n\n# --- TF-IDF indices (smaller, less noisy) ---\nchar_vec = TfidfVectorizer(analyzer=\"char_wb\", ngram_range=(3,6), min_df=2, max_df=0.98)\nword_vec = TfidfVectorizer(analyzer=\"word\", ngram_range=(1,2), token_pattern=r\"(?u)\\b[a-z0-9\\-_]+\\b\", min_df=2, max_df=0.98)\n\nXc_mem = char_vec.fit_transform(mem_src)\nXw_mem = word_vec.fit_transform(mem_src)\nprint(\"Char vocab:\", len(char_vec.vocabulary_), \"Word vocab:\", len(word_vec.vocabulary_))\n\n# --- build simple cooccurrence backoff (single-token associations) ---\nsrc_train_norm = train[\"transliteration\"].map(norm_src)\ntgt_train = train[\"translation\"].map(norm_tgt)\n\ntgt_surface = defaultdict(Counter)\nsrc_cnt = Counter()\ntgt_cnt = Counter()\npair_cnt = Counter()\n\nfor s, t in zip(src_train_norm, tgt_train):\n    if not s or not t: continue\n    s_tokens = set(s.split())\n    t_tokens = set(w.lower() for w in t.split())\n    for w in t.split():\n        tgt_surface[w.lower()][w] += 1\n    for sw in s_tokens: src_cnt[sw] += 1\n    for tw in t_tokens: tgt_cnt[tw] += 1\n    for sw in s_tokens:\n        for tw in t_tokens:\n            pair_cnt[(sw, tw)] += 1\n\nbest_word = {}\nbest_score = {}\nfor (sw, tw), c in pair_cnt.items():\n    score = c / np.sqrt(src_cnt[sw] * tgt_cnt[tw])\n    if (sw not in best_score) or (score > best_score[sw]):\n        best_score[sw] = score\n        best_word[sw] = tw\nbest_surface = {lw: surf.most_common(1)[0][0] for lw, surf in tgt_surface.items()}\n\ndef dict_backoff_translate(src_norm, min_assoc=0.10):\n    if not src_norm: return \"...\"\n    # exact TM hit\n    if src_norm in exact_map:\n        return exact_map[src_norm]\n    out = []\n    for sw in src_norm.split():\n        if sw.startswith(\"det_\") or sw == \"gap\": continue\n        tw = best_word.get(sw)\n        if tw is not None and best_score.get(sw,0.0) >= min_assoc:\n            out.append(best_surface.get(tw, tw))\n        else:\n            out.append(sw)\n    return finalize_translation(\" \".join(out))\n\n# --- retrieval + rerank + final decision ---\nW_CHAR = 0.78\nW_WORD = 0.22\nW_SEQ = 0.06   # small bonus\nLEN_PENALTY_POWER = 1.6\nMIN_ACCEPT_SCORE = 0.16\nTOPK = 60\nRERANK_K = 10\n\ndef seq_ratio(a,b):\n    return SequenceMatcher(None, a, b).ratio()\n\ndef length_penalty(q, c):\n    lq = max(1, len(q.split()))\n    lc = max(1, len(c.split()))\n    ratio = min(lq, lc) / max(lq, lc)\n    return ratio ** LEN_PENALTY_POWER\n\ndef translate_query(q):\n    qn = norm_src(q)\n    if not qn: return \"...\"\n    # exact TM\n    if qn in exact_map:\n        return exact_map[qn]\n    # vectorize\n    qc = char_vec.transform([qn])\n    qw = word_vec.transform([qn])\n    sc = linear_kernel(qc, Xc_mem).ravel()   # cosine-like\n    sw = linear_kernel(qw, Xw_mem).ravel()\n    # normalize\n    sc = (sc - sc.min()) / (sc.max() - sc.min() + 1e-12)\n    sw = (sw - sw.min()) / (sw.max() - sw.min() + 1e-12)\n    combined = W_CHAR * sc + W_WORD * sw\n    # topk candidates\n    top_idx = np.argpartition(-combined, min(TOPK, len(combined)-1))[:min(TOPK, len(combined))]\n    # rerank small set with sequence ratio\n    rerank_idx = top_idx[np.argsort(-combined[top_idx])][:min(RERANK_K, len(top_idx))]\n    best_i, best_score = None, -1.0\n    for i in rerank_idx:\n        s_seq = seq_ratio(qn, mem_src[i])\n        score = combined[i] + W_SEQ * s_seq\n        score *= length_penalty(qn, mem_src[i])\n        if score > best_score:\n            best_score = score\n            best_i = i\n    # accept if above threshold, else backoff\n    if best_score >= MIN_ACCEPT_SCORE and best_i is not None:\n        return mem_tgt[best_i]\n    return dict_backoff_translate(qn, min_assoc=0.10)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-17T10:51:31.452306Z","iopub.execute_input":"2025-12-17T10:51:31.452998Z","iopub.status.idle":"2025-12-17T10:51:42.091598Z","shell.execute_reply.started":"2025-12-17T10:51:31.452957Z","shell.execute_reply":"2025-12-17T10:51:42.090619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Display the Output","metadata":{}},{"cell_type":"code","source":"\nif 'preds' not in globals() or len(preds) != len(test):\n    preds = [translate_query(row.get(\"transliteration\",\"\")) for _, row in test.iterrows()]\n\n# Build the tabular output with column name id0\nout_df = pd.DataFrame({\n    \"id\": test[\"id\"].astype(str).tolist(),\n    \"translation\": preds\n})\n\n# Display the table (first 20 rows) and save to CSV\ndisplay(out_df.head(20))\nout_df.to_csv(\"submission.csv\", index=True)\n\nprint(\"Saved submission.csv\", len(out_df), \"rows.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T10:52:47.034897Z","iopub.execute_input":"2025-12-17T10:52:47.035259Z","iopub.status.idle":"2025-12-17T10:52:47.048549Z","shell.execute_reply.started":"2025-12-17T10:52:47.035229Z","shell.execute_reply":"2025-12-17T10:52:47.047820Z"}},"outputs":[],"execution_count":null}]}