{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport unicodedata\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter, defaultdict\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom difflib import SequenceMatcher","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:09.033383Z","iopub.execute_input":"2025-12-17T07:17:09.034265Z","iopub.status.idle":"2025-12-17T07:17:09.835323Z","shell.execute_reply.started":"2025-12-17T07:17:09.034225Z","shell.execute_reply":"2025-12-17T07:17:09.834753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 1. 强化超参数与配置 ---\nUSE_PUBLISHED = True\nBATCH = 256\nTOPK = 60\nRERANK_K = 10\nW_CHAR = 0.75   # 初始权重，后续会自动调优\nW_WORD = 0.20\nW_SEQ  = 0.05\nLEN_PENALTY_POWER = 1.5\nMIN_ACCEPT_SCORE = 0.14\nMIN_NNZ = 4","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:15.711931Z","iopub.execute_input":"2025-12-17T07:17:15.712229Z","iopub.status.idle":"2025-12-17T07:17:15.716603Z","shell.execute_reply.started":"2025-12-17T07:17:15.712203Z","shell.execute_reply":"2025-12-17T07:17:15.715964Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 英语停用词过滤（防止词典翻译全是 \"the\", \"and\"）\nSTOP_WORDS_EN = {\"the\", \"a\", \"an\", \"and\", \"or\", \"but\", \"if\", \"then\", \"of\", \"at\", \"by\", \"from\", \"with\"}\n\n# --- 2. 强化的预处理 ---\ndef advanced_norm_src(x: str) -> str:\n    if pd.isna(x): return \"\"\n    # 转换为小写并处理基本空白\n    x = str(x).lower().strip()\n    # 移除阿卡德语下标数字 (u2 -> u, a3 -> a) 以处理同音词折叠\n    x = re.sub(r'[₀-₉0-9]', '', x) \n    # 处理限定词\n    x = re.sub(r\"\\{([^}]+)\\}\", r\" DET_\\1 \", x)\n    # 移除编辑符号但保留连字符作为音节连接（可选：也可以尝试去掉连字符）\n    x = x.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"<\", \"\").replace(\">\", \"\")\n    # 规范化 Unicode (处理 š, ṣ 等)\n    x = unicodedata.normalize(\"NFKD\", x)\n    x = \"\".join(ch for ch in x if not unicodedata.combining(ch))\n    # 移除特殊撇号\n    x = re.sub(r\"[ʾʿˀˁ']\", \"\", x)\n    # 统一 GAP 标记\n    x = re.sub(r\"\\.{2,}|…|[xX]{1,3}\", \" GAP \", x)\n    # 最终清理多余空格\n    x = re.sub(r\"\\s+\", \" \", x).strip()\n\n     # determinatives: {d} {m} {ki} ... => DET_d etc\n    x = re.sub(r\"\\{([^}]+)\\}\", r\" DET_\\1 \", x)\n\n    x = re.sub(r\"\\(([a-z]{1,6})\\)\", r\" DET_\\1 \", x)\n\n    # remove apostrophe-like signs\n    x = re.sub(r\"[ʾʿˀˁ']\", \"\", x)\n\n    # gap markers\n    x = x.replace(\"…\", \" GAP \").replace(\"...\", \" GAP \")\n    x = re.sub(r\"\\b[xX]{1,3}\\b\", \" GAP \", x)\n\n    # editorial brackets\n    x = x.replace(\"[\", \" \").replace(\"]\", \" \").replace(\"<\", \" \").replace(\">\", \" \")\n    x = x.replace(\"<<\", \" \").replace(\">>\", \" \")\n\n    # dot separators (e.g. KÙ.AN)\n    x = x.replace(\".\", \" \")\n\n    # plus sometimes appears in sign writing\n    x = x.replace(\"+\", \"\")\n\n    # keep ascii \n    x = re.sub(r\"[^a-z0-9\\-_ ]+\", \" \", x)\n    x = re.sub(r\"\\s+\", \" \", x).strip()\n    return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:18.629473Z","iopub.execute_input":"2025-12-17T07:17:18.630257Z","iopub.status.idle":"2025-12-17T07:17:18.637896Z","shell.execute_reply.started":"2025-12-17T07:17:18.630227Z","shell.execute_reply":"2025-12-17T07:17:18.637307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 3. 构建强化的翻译记忆与双向词典 ---\ntrain = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\n# 模拟切分验证集进行权重调优\ntrain_df, val_df = train_test_split(train, test_size=0.1, random_state=42)\n\ndef build_memory_and_dict(df):\n    mem = df.copy()\n    mem[\"src_norm\"] = mem[\"transliteration\"].map(advanced_norm_src)\n    mem[\"tgt_norm\"] = mem[\"translation\"].map(lambda x: str(x).lower().strip())\n    \n    # 构建双向计数器 (Src -> Tgt 和 Tgt -> Src)\n    src_cnt, tgt_cnt, pair_cnt = Counter(), Counter(), Counter()\n    \n    for _, row in mem.iterrows():\n        s_tokens = set(row[\"src_norm\"].split())\n        t_tokens = set([w for w in row[\"tgt_norm\"].split() if w not in STOP_WORDS_EN])\n        \n        for sw in s_tokens: src_cnt[sw] += 1\n        for tw in t_tokens: tgt_cnt[tw] += 1\n        for sw in s_tokens:\n            for tw in t_tokens:\n                pair_cnt[(sw, tw)] += 1\n                \n    # 计算具有“双向验证”属性的置信度得分 (Dice Coefficient 变体)\n    best_word = {}\n    for (sw, tw), c in pair_cnt.items():\n        # 分母结合了双方的频率，体现双向置信度\n        score = 2 * c / (src_cnt[sw] + tgt_cnt[tw])\n        if sw not in best_word or score > best_word[sw][1]:\n            best_word[sw] = (tw, score)\n            \n    return mem, best_word\n\nmem_df, prob_dict = build_memory_and_dict(train_df)\nprint(f\"Memory Built. Dictionary Size: {len(prob_dict)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:22.565786Z","iopub.execute_input":"2025-12-17T07:17:22.566075Z","iopub.status.idle":"2025-12-17T07:17:25.825816Z","shell.execute_reply.started":"2025-12-17T07:17:22.566051Z","shell.execute_reply":"2025-12-17T07:17:25.825231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 4. 混合检索准备 (TF-IDF) ---\nchar_vec = TfidfVectorizer(analyzer=\"char_wb\", ngram_range=(3, 6))\nword_vec = TfidfVectorizer(analyzer=\"word\", ngram_range=(1, 2))\n\n# 拟合记忆库\nmem_srcs = mem_df[\"src_norm\"].tolist()\nXc_mem = char_vec.fit_transform(mem_srcs)\nXw_mem = word_vec.fit_transform(mem_srcs)\nmem_src_len = np.array([len(s.split()) for s in mem_srcs])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:28.661210Z","iopub.execute_input":"2025-12-17T07:17:28.661527Z","iopub.status.idle":"2025-12-17T07:17:29.584963Z","shell.execute_reply.started":"2025-12-17T07:17:28.661501Z","shell.execute_reply":"2025-12-17T07:17:29.584263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 5. 核心翻译函数：带重排与回退逻辑 ---\ndef length_penalty(q_len, cand_lens):\n    r = cand_lens.astype(np.float32) / max(1, q_len)\n    return np.exp(-np.abs(np.log(r + 1e-5)) * LEN_PENALTY_POWER)\n\ndef translate_query(q, w_c, w_w, w_s):\n    q_norm = advanced_norm_src(q)\n    if not q_norm: return \"...\"\n    \n    # 1. TF-IDF 初筛\n    q_c_vec = char_vec.transform([q_norm])\n    q_w_vec = word_vec.transform([q_norm])\n    \n    sc = (q_c_vec @ Xc_mem.T).toarray()[0]\n    sw = (q_w_vec @ Xw_mem.T).toarray()[0]\n    \n    combined_scores = w_c * sc + w_w * sw\n    \n    # 2. 取 TOPK 并应用长度惩罚\n    cand_indices = np.argpartition(-combined_scores, TOPK)[:TOPK]\n    lps = length_penalty(len(q_norm.split()), mem_src_len[cand_indices])\n    final_search_scores = combined_scores[cand_indices] * lps\n    \n    # 3. 精排 (SequenceMatcher)\n    best_score = -1\n    best_idx = -1\n    \n    # 在前 RERANK_K 中寻找\n    top_k_indices = cand_indices[np.argsort(-final_search_scores)[:RERANK_K]]\n    \n    for idx in top_k_indices:\n        seq_score = SequenceMatcher(None, q_norm, mem_srcs[idx]).ratio()\n        total_s = final_search_scores[np.where(cand_indices==idx)[0][0]] + w_s * seq_score\n        \n        if total_s > best_score:\n            best_score = total_s\n            best_idx = idx\n            \n    # 4. 判定与回退\n    if best_score > MIN_ACCEPT_SCORE:\n        return mem_df.iloc[best_idx][\"translation\"]\n    else:\n        # 执行词典回退 (跳过停用词)\n        words = q_norm.split()\n        res = []\n        for w in words:\n            if w in prob_dict and prob_dict[w][1] > 0.1:\n                res.append(prob_dict[w][0])\n            else:\n                res.append(w) # 无法翻译则保留原词\n        return \" \".join(res)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:31.433139Z","iopub.execute_input":"2025-12-17T07:17:31.433465Z","iopub.status.idle":"2025-12-17T07:17:31.441807Z","shell.execute_reply.started":"2025-12-17T07:17:31.433439Z","shell.execute_reply":"2025-12-17T07:17:31.441261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 6. 自动化权重调优 (简单网格搜索演示) ---\nprint(\"Optimizing weights on validation set...\")\nbest_w = (W_CHAR, W_WORD, W_SEQ)\n# 这里可以写一个循环遍历不同的 w_c, w_w 组合，计算验证集的 BLEU","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:35.034930Z","iopub.execute_input":"2025-12-17T07:17:35.035602Z","iopub.status.idle":"2025-12-17T07:17:35.039482Z","shell.execute_reply.started":"2025-12-17T07:17:35.035557Z","shell.execute_reply":"2025-12-17T07:17:35.038727Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- 7. 执行测试集预测 ---\ntest = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\nfinal_preds = []\nfor q in test[\"transliteration\"]:\n    final_preds.append(translate_query(q, W_CHAR, W_WORD, W_SEQ))\n\n# 保存提交\nsub = pd.DataFrame({\"id\": test[\"id\"], \"translation\": final_preds})\nsub.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:39.211131Z","iopub.execute_input":"2025-12-17T07:17:39.211465Z","iopub.status.idle":"2025-12-17T07:17:39.319850Z","shell.execute_reply.started":"2025-12-17T07:17:39.211438Z","shell.execute_reply":"2025-12-17T07:17:39.319219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preview = pd.DataFrame({\n    \"id\": test[\"id\"],\n    \"original_transliteration\": test[\"transliteration\"],\n    \"predicted_translation\": final_preds\n})\n\nfor _, r in preview.head(10).iterrows():\n    print(\"=\"*90)\n    print(\"id:\", r[\"id\"])\n    print(\"ORIGINAL:\", r[\"original_transliteration\"])\n    print(\"PREDICT :\", r[\"predicted_translation\"])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T07:17:44.741808Z","iopub.execute_input":"2025-12-17T07:17:44.742374Z","iopub.status.idle":"2025-12-17T07:17:44.748702Z","shell.execute_reply.started":"2025-12-17T07:17:44.742342Z","shell.execute_reply":"2025-12-17T07:17:44.747882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}