{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport unicodedata\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter, defaultdict\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom difflib import SequenceMatcher\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-19T08:13:04.858148Z","iopub.execute_input":"2025-12-19T08:13:04.859083Z","iopub.status.idle":"2025-12-19T08:13:04.863977Z","shell.execute_reply.started":"2025-12-19T08:13:04.859053Z","shell.execute_reply":"2025-12-19T08:13:04.863062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === CONFIG ===\nTRAIN_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/train.csv\"\nTEST_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nTOPK = 100\nRERANK_K = 20","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T08:13:04.865072Z","iopub.execute_input":"2025-12-19T08:13:04.865313Z","iopub.status.idle":"2025-12-19T08:13:04.878587Z","shell.execute_reply.started":"2025-12-19T08:13:04.865294Z","shell.execute_reply":"2025-12-19T08:13:04.877983Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === ADVANCED PREPROCESSING ===\ndef advanced_norm_src(x: str) -> str:\n    if pd.isna(x): return \"\"\n    x = str(x).lower().strip()\n    \n    # Fold subscripts: u2,u₃ → u\n    x = re.sub(r'([a-z₀-₉])[₀-₉0-9]*', r'\\1', x)\n    \n    # Determinatives\n    x = re.sub(r\"\\{([^}]+)\\}|\\(([a-z]+)\\)\", r\"DET_\\1\", x)\n    \n    # Editorial marks\n    x = re.sub(r\"[ʾʿˀˁ'\\[\\]<>]\", \" \", x)\n    x = re.sub(r\"\\.{2,}|\\…|[xX]{1,3}\", \" GAP \", x)\n    \n    # Clean\n    x = unicodedata.normalize(\"NFKD\", x)\n    x = \"\".join(c for c in x if not unicodedata.combining(c))\n    x = re.sub(r\"[^a-z0-9\\s\\-_]+\", \" \", x)\n    x = re.sub(r\"\\s+\", \" \", x).strip()\n    return x\n\nSTOP_WORDS_EN = {\"the\", \"a\", \"an\", \"and\", \"or\", \"but\", \"if\", \"then\", \"of\", \"at\", \"by\", \"from\", \"with\", \"in\", \"on\"}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T08:13:04.880165Z","iopub.execute_input":"2025-12-19T08:13:04.880563Z","iopub.status.idle":"2025-12-19T08:13:04.891202Z","shell.execute_reply.started":"2025-12-19T08:13:04.880540Z","shell.execute_reply":"2025-12-19T08:13:04.890659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === BUILD SUPERIOR TRANSLATION MEMORY ===\nprint(\"📚 Building Translation Memory Bank...\")\ntrain = pd.read_csv(TRAIN_PATH).iloc[800:]  # Skip first 800 for better quality\ntrain[\"src_norm\"] = train[\"transliteration\"].map(advanced_norm_src)\ntrain[\"tgt_norm\"] = train[\"translation\"].str.lower().str.strip()\n\n# Bidirectional dictionary (Dice coefficient)\nsrc_cnt, tgt_cnt, pair_cnt = Counter(), Counter(), Counter()\n\nfor _, row in train.iterrows():\n    s_tokens = set(row[\"src_norm\"].split())\n    t_tokens = set(w for w in row[\"tgt_norm\"].split() if w not in STOP_WORDS_EN)\n    \n    for sw in s_tokens: src_cnt[sw] += 1\n    for tw in t_tokens: tgt_cnt[tw] += 1\n    for sw in s_tokens:\n        for tw in t_tokens:\n            pair_cnt[(sw, tw)] += 1\n\n# Build best word dictionary\nprob_dict = {}\nfor (sw, tw), c in pair_cnt.items():\n    score = 2 * c / (src_cnt[sw] + tgt_cnt[tw] + 1e-6)\n    if sw not in prob_dict or score > prob_dict[sw][1]:\n        prob_dict[sw] = (tw, score)\n\nprint(f\"✅ Memory: {len(train)} samples, {len(prob_dict)} dictionary entries\")\n\n# Multi-level TF-IDF (char + word)\nprint(\"🔍 Preparing TF-IDF vectors...\")\nchar_vec = TfidfVectorizer(analyzer=\"char_wb\", ngram_range=(2,6), max_features=8000)\nword_vec = TfidfVectorizer(analyzer=\"word\", ngram_range=(1,3), max_features=4000)\n\nmem_srcs = train[\"src_norm\"].tolist()\nXc_mem = char_vec.fit_transform(mem_srcs)\nXw_mem = word_vec.fit_transform(mem_srcs)\nmem_src_len = np.array([len(s.split()) for s in mem_srcs])\nmem_translations = train[\"translation\"].tolist()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T08:13:04.892622Z","iopub.execute_input":"2025-12-19T08:13:04.892893Z","iopub.status.idle":"2025-12-19T08:13:07.523999Z","shell.execute_reply.started":"2025-12-19T08:13:04.892863Z","shell.execute_reply":"2025-12-19T08:13:07.523399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === CORE RETRIEVAL ENGINE ===\ndef length_penalty(q_len, cand_lens):\n    r = cand_lens / max(1, q_len)\n    return np.exp(-np.abs(np.log(r + 1e-8)) * 1.8)\n\ndef translate_query(q, w_char=0.7, w_word=0.2, w_seq=0.1):\n    q_norm = advanced_norm_src(q)\n    if len(q_norm) < 3: \n        return \"broken text\"\n    \n    # TF-IDF similarity\n    qc = char_vec.transform([q_norm])\n    qw = word_vec.transform([q_norm])\n    sc_char = qc @ Xc_mem.T\n    sc_word = qw @ Xw_mem.T\n    scores = w_char * sc_char.toarray()[0] + w_word * sc_word.toarray()[0]\n    \n    # TopK candidates + length penalty\n    cand_idx = np.argpartition(-scores, TOPK)[:TOPK]\n    lps = length_penalty(len(q_norm.split()), mem_src_len[cand_idx])\n    final_scores = scores[cand_idx] * lps\n    \n    top20 = cand_idx[np.argsort(-final_scores)[:RERANK_K]]\n    best_score, best_idx = -1, -1\n    \n    for idx in top20:\n        seq_score = SequenceMatcher(None, q_norm, mem_srcs[idx]).ratio()\n        total_score = final_scores[np.where(cand_idx == idx)[0][0]] + w_seq * seq_score\n        \n        if total_score > best_score:\n            best_score = total_score\n            best_idx = idx\n    \n    # Accept if score > threshold, else word-by-word\n    if best_score > 0.18:\n        return mem_translations[best_idx]\n    else:\n        words = q_norm.split()\n        translated = []\n        for w in words:\n            if w in prob_dict and prob_dict[w][1] > 0.12:\n                translated.append(prob_dict[w][0])\n            else:\n                translated.append(w)\n        return \" \".join(translated)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T08:13:07.524923Z","iopub.execute_input":"2025-12-19T08:13:07.525200Z","iopub.status.idle":"2025-12-19T08:13:07.533340Z","shell.execute_reply.started":"2025-12-19T08:13:07.525169Z","shell.execute_reply":"2025-12-19T08:13:07.532454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === GENERATE PREDICTIONS ===\nprint(\"🎯 Generating predictions...\")\ntest_df = pd.read_csv(TEST_PATH)\nfinal_predictions = []\n\nfor i, translit in enumerate(tqdm(test_df['transliteration'], desc=\"Translating\")):\n    pred = translate_query(translit)\n    if not pred.endswith('.'):\n        pred += \".\"\n    final_predictions.append(pred)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T08:13:07.534368Z","iopub.execute_input":"2025-12-19T08:13:07.534691Z","iopub.status.idle":"2025-12-19T08:13:07.614449Z","shell.execute_reply.started":"2025-12-19T08:13:07.534634Z","shell.execute_reply":"2025-12-19T08:13:07.613699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === 5. SUBMISSION ===\nsubmission = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": final_predictions\n})\nsubmission.to_csv(\"submission.csv\", index=False)  # ← EXACT FILENAME\nprint(\"🚀 submission.csv READY - Click 'Submit to Competition'!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T08:13:07.615535Z","iopub.execute_input":"2025-12-19T08:13:07.615899Z","iopub.status.idle":"2025-12-19T08:13:07.621482Z","shell.execute_reply.started":"2025-12-19T08:13:07.615871Z","shell.execute_reply":"2025-12-19T08:13:07.620923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n📊 Sample predictions\")\nn = min(4, len(test_df), len(final_predictions))\n\nfor i in range(n):\n    print(f\"SRC: {test_df['transliteration'].iloc[i]}\")\n    print(f\"ENG: {final_predictions[i]}\")\n    print(\"-\" * 80)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-19T08:13:07.622918Z","iopub.execute_input":"2025-12-19T08:13:07.623116Z","iopub.status.idle":"2025-12-19T08:13:07.637574Z","shell.execute_reply.started":"2025-12-19T08:13:07.623098Z","shell.execute_reply":"2025-12-19T08:13:07.637005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}