{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":121150,"databundleVersionId":14976537,"sourceType":"competition"},{"sourceId":14214302,"sourceType":"datasetVersion","datasetId":9066943},{"sourceId":287036789,"sourceType":"kernelVersion"},{"sourceId":287153207,"sourceType":"kernelVersion"}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Past Initiative – Machine Translation (Inference Notebook)\n\nThis notebook is a **starter / baseline** for this Kaggle competition.\n\nTraining Code is [here](https://www.kaggle.com/code/takamichitoda/dpc-starter-train).","metadata":{}},{"cell_type":"markdown","source":"# A Rule-Based Baseline Solution (TR-TRY Notebook)\n\nThis notebook builds a **\"translation memory bank\"** and **\"bidirectional confidence dictionary\"** using the training set, and generates translation results by matching the most similar training samples for test set texts through multi-dimensional retrieval.\n\nTR-TRY Notebook is [here](https://www.kaggle.com/code/jackcerion/tr-try). **(copy from @jackcerion)**","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-18T16:54:28.747274Z","iopub.execute_input":"2025-12-18T16:54:28.748076Z","iopub.status.idle":"2025-12-18T16:54:28.752269Z","shell.execute_reply.started":"2025-12-18T16:54:28.748042Z","shell.execute_reply":"2025-12-18T16:54:28.751446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#MODEL_PATH = \"/kaggle/input/dpc-starter-train/byt5-akkadian-model/\"\n#MODEL_PATH = \"/kaggle/input/epoch30-seed42/byt5-akkadian-model\"\nMODEL_PATH=\"/kaggle/input/k/qifeihhh666/dpc-starter-train/byt5-akkadian-model/\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T16:54:29.097159Z","iopub.execute_input":"2025-12-18T16:54:29.097442Z","iopub.status.idle":"2025-12-18T16:54:29.101062Z","shell.execute_reply.started":"2025-12-18T16:54:29.097415Z","shell.execute_reply":"2025-12-18T16:54:29.100280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_DATA_PATH = \"/kaggle/input/deep-past-initiative-machine-translation/test.csv\"\nBATCH_SIZE = 16\nMAX_LENGTH = 512\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# --- Model Loading ---\nprint(f\"Loading model from {MODEL_PATH}...\")\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\nmodel = AutoModelForSeq2SeqLM.from_pretrained(MODEL_PATH).to(DEVICE)\nmodel.eval()\n\n# --- Data Preparation ---\ntest_df = pd.read_csv(TEST_DATA_PATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T16:54:29.102210Z","iopub.execute_input":"2025-12-18T16:54:29.102469Z","iopub.status.idle":"2025-12-18T16:54:38.366011Z","shell.execute_reply.started":"2025-12-18T16:54:29.102433Z","shell.execute_reply":"2025-12-18T16:54:38.365195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PREFIX = \"translate Akkadian to English: \"\n\nclass InferenceDataset(Dataset):\n    def __init__(self, df, tokenizer):\n        self.texts = df['transliteration'].astype(str).tolist()\n        self.texts = [PREFIX + i for i in self.texts]\n        self.tokenizer = tokenizer\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def __getitem__(self, idx):\n        text = self.texts[idx]\n        inputs = self.tokenizer(\n            text, \n            max_length=MAX_LENGTH, \n            padding=\"max_length\", \n            truncation=True, \n            return_tensors=\"pt\"\n        )\n        return {\n            \"input_ids\": inputs[\"input_ids\"].squeeze(0),\n            \"attention_mask\": inputs[\"attention_mask\"].squeeze(0)\n        }\n\ntest_dataset = InferenceDataset(test_df, tokenizer)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n# --- Inference Loop ---\nprint(\"Starting Inference...\")\nall_predictions = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T16:54:38.367176Z","iopub.execute_input":"2025-12-18T16:54:38.367564Z","iopub.status.idle":"2025-12-18T16:54:38.374516Z","shell.execute_reply.started":"2025-12-18T16:54:38.367541Z","shell.execute_reply":"2025-12-18T16:54:38.373787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with torch.no_grad():\n    for batch in tqdm(test_loader):\n        input_ids = batch[\"input_ids\"].to(DEVICE)\n        attention_mask = batch[\"attention_mask\"].to(DEVICE)\n  \n        outputs = model.generate(\n            input_ids=input_ids,\n            attention_mask=attention_mask,\n            max_length=MAX_LENGTH,\n            num_beams=4,\n            early_stopping=True\n        )\n        \n        decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n        all_predictions.extend([d.strip() for d in decoded])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T16:54:38.376279Z","iopub.execute_input":"2025-12-18T16:54:38.376822Z","iopub.status.idle":"2025-12-18T16:54:40.936608Z","shell.execute_reply.started":"2025-12-18T16:54:38.376800Z","shell.execute_reply":"2025-12-18T16:54:40.935863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Submission ---\nsubmission_Deep = pd.DataFrame({\n    \"id\": test_df[\"id\"],\n    \"translation\": all_predictions\n})\n\nsubmission_Deep[\"translation\"] = submission_Deep[\"translation\"].apply(lambda x: x if len(x) > 0 else \"broken text\")\n\nsubmission_Deep.to_csv(\"submission.csv\", index=False)\nsubmission_Deep.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T17:10:54.781719Z","iopub.execute_input":"2025-12-18T17:10:54.782314Z","iopub.status.idle":"2025-12-18T17:10:54.792130Z","shell.execute_reply.started":"2025-12-18T17:10:54.782281Z","shell.execute_reply":"2025-12-18T17:10:54.791362Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# TR-TRY","metadata":{}},{"cell_type":"code","source":"# import re\n# import unicodedata\n# import numpy as np\n# import pandas as pd\n# from collections import Counter, defaultdict\n# from sklearn.feature_extraction.text import TfidfVectorizer\n# from sklearn.model_selection import train_test_split\n# from difflib import SequenceMatcher\n\n\n# # --- 1. 强化超参数与配置 ---\n# USE_PUBLISHED = True\n# BATCH = 256\n# TOPK = 60\n# RERANK_K = 10\n# W_CHAR = 0.75   # 初始权重，后续会自动调优\n# W_WORD = 0.20\n# W_SEQ  = 0.05\n# LEN_PENALTY_POWER = 1.5\n# MIN_ACCEPT_SCORE = 0.14\n# MIN_NNZ = 4\n\n\n# # 英语停用词过滤（防止词典翻译全是 \"the\", \"and\"）\n# STOP_WORDS_EN = {\"the\", \"a\", \"an\", \"and\", \"or\", \"but\", \"if\", \"then\", \"of\", \"at\", \"by\", \"from\", \"with\"}\n\n# # --- 2. 强化的预处理 ---\n# def advanced_norm_src(x: str) -> str:\n#     if pd.isna(x): return \"\"\n#     # 转换为小写并处理基本空白\n#     x = str(x).lower().strip()\n#     # 移除阿卡德语下标数字 (u2 -> u, a3 -> a) 以处理同音词折叠\n#     x = re.sub(r'[₀-₉0-9]', '', x) \n#     # 处理限定词\n#     x = re.sub(r\"\\{([^}]+)\\}\", r\" DET_\\1 \", x)\n#     # 移除编辑符号但保留连字符作为音节连接（可选：也可以尝试去掉连字符）\n#     x = x.replace(\"[\", \"\").replace(\"]\", \"\").replace(\"<\", \"\").replace(\">\", \"\")\n#     # 规范化 Unicode (处理 š, ṣ 等)\n#     x = unicodedata.normalize(\"NFKD\", x)\n#     x = \"\".join(ch for ch in x if not unicodedata.combining(ch))\n#     # 移除特殊撇号\n#     x = re.sub(r\"[ʾʿˀˁ']\", \"\", x)\n#     # 统一 GAP 标记\n#     x = re.sub(r\"\\.{2,}|…|[xX]{1,3}\", \" GAP \", x)\n#     # 最终清理多余空格\n#     x = re.sub(r\"\\s+\", \" \", x).strip()\n\n#      # determinatives: {d} {m} {ki} ... => DET_d etc\n#     x = re.sub(r\"\\{([^}]+)\\}\", r\" DET_\\1 \", x)\n\n#     x = re.sub(r\"\\(([a-z]{1,6})\\)\", r\" DET_\\1 \", x)\n\n#     # remove apostrophe-like signs\n#     x = re.sub(r\"[ʾʿˀˁ']\", \"\", x)\n\n#     # gap markers\n#     x = x.replace(\"…\", \" GAP \").replace(\"...\", \" GAP \")\n#     x = re.sub(r\"\\b[xX]{1,3}\\b\", \" GAP \", x)\n\n#     # editorial brackets\n#     x = x.replace(\"[\", \" \").replace(\"]\", \" \").replace(\"<\", \" \").replace(\">\", \" \")\n#     x = x.replace(\"<<\", \" \").replace(\">>\", \" \")\n\n#     # dot separators (e.g. KÙ.AN)\n#     x = x.replace(\".\", \" \")\n\n#     # plus sometimes appears in sign writing\n#     x = x.replace(\"+\", \"\")\n\n#     # keep ascii \n#     x = re.sub(r\"[^a-z0-9\\-_ ]+\", \" \", x)\n#     x = re.sub(r\"\\s+\", \" \", x).strip()\n#     return x\n\n# # --- 3. 构建强化的翻译记忆与双向词典 ---\n# train = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/train.csv')\n# train=train.iloc[1000:]  \n# # 模拟切分验证集进行权重调优\n# train_df, val_df = train_test_split(train, test_size=0.1, random_state=42)\n\n# def build_memory_and_dict(df):\n#     mem = df.copy()\n#     mem[\"src_norm\"] = mem[\"transliteration\"].map(advanced_norm_src)\n#     mem[\"tgt_norm\"] = mem[\"translation\"].map(lambda x: str(x).lower().strip())\n    \n#     # 构建双向计数器 (Src -> Tgt 和 Tgt -> Src)\n#     src_cnt, tgt_cnt, pair_cnt = Counter(), Counter(), Counter()\n    \n#     for _, row in mem.iterrows():\n#         s_tokens = set(row[\"src_norm\"].split())\n#         t_tokens = set([w for w in row[\"tgt_norm\"].split() if w not in STOP_WORDS_EN])\n        \n#         for sw in s_tokens: src_cnt[sw] += 1\n#         for tw in t_tokens: tgt_cnt[tw] += 1\n#         for sw in s_tokens:\n#             for tw in t_tokens:\n#                 pair_cnt[(sw, tw)] += 1\n                \n#     # 计算具有“双向验证”属性的置信度得分 (Dice Coefficient 变体)\n#     best_word = {}\n#     for (sw, tw), c in pair_cnt.items():\n#         # 分母结合了双方的频率，体现双向置信度\n#         score = 2 * c / (src_cnt[sw] + tgt_cnt[tw])\n#         if sw not in best_word or score > best_word[sw][1]:\n#             best_word[sw] = (tw, score)\n            \n#     return mem, best_word\n\n# mem_df, prob_dict = build_memory_and_dict(train_df)\n# print(f\"Memory Built. Dictionary Size: {len(prob_dict)}\")\n\n\n# # --- 4. 混合检索准备 (TF-IDF) ---\n# char_vec = TfidfVectorizer(analyzer=\"char_wb\", ngram_range=(3, 6))\n# word_vec = TfidfVectorizer(analyzer=\"word\", ngram_range=(1, 2))\n\n# # 拟合记忆库\n# mem_srcs = mem_df[\"src_norm\"].tolist()\n# Xc_mem = char_vec.fit_transform(mem_srcs)\n# Xw_mem = word_vec.fit_transform(mem_srcs)\n# mem_src_len = np.array([len(s.split()) for s in mem_srcs])\n\n# # --- 5. 核心翻译函数：带重排与回退逻辑 ---\n# def length_penalty(q_len, cand_lens):\n#     r = cand_lens.astype(np.float32) / max(1, q_len)\n#     return np.exp(-np.abs(np.log(r + 1e-5)) * LEN_PENALTY_POWER)\n\n# def translate_query(q, w_c, w_w, w_s):\n#     q_norm = advanced_norm_src(q)\n#     if not q_norm: return \"...\"\n    \n#     # 1. TF-IDF 初筛\n#     q_c_vec = char_vec.transform([q_norm])\n#     q_w_vec = word_vec.transform([q_norm])\n    \n#     sc = (q_c_vec @ Xc_mem.T).toarray()[0]\n#     sw = (q_w_vec @ Xw_mem.T).toarray()[0]\n    \n#     combined_scores = w_c * sc + w_w * sw\n    \n#     # 2. 取 TOPK 并应用长度惩罚\n#     cand_indices = np.argpartition(-combined_scores, TOPK)[:TOPK]\n#     lps = length_penalty(len(q_norm.split()), mem_src_len[cand_indices])\n#     final_search_scores = combined_scores[cand_indices] * lps\n    \n#     # 3. 精排 (SequenceMatcher)\n#     best_score = -1\n#     best_idx = -1\n    \n#     # 在前 RERANK_K 中寻找\n#     top_k_indices = cand_indices[np.argsort(-final_search_scores)[:RERANK_K]]\n    \n#     for idx in top_k_indices:\n#         seq_score = SequenceMatcher(None, q_norm, mem_srcs[idx]).ratio()\n#         total_s = final_search_scores[np.where(cand_indices==idx)[0][0]] + w_s * seq_score\n        \n#         if total_s > best_score:\n#             best_score = total_s\n#             best_idx = idx\n            \n#     # 4. 判定与回退\n#     if best_score > MIN_ACCEPT_SCORE:\n#         return mem_df.iloc[best_idx][\"translation\"]\n#     else:\n#         # 执行词典回退 (跳过停用词)\n#         words = q_norm.split()\n#         res = []\n#         for w in words:\n#             if w in prob_dict and prob_dict[w][1] > 0.1:\n#                 res.append(prob_dict[w][0])\n#             else:\n#                 res.append(w) # 无法翻译则保留原词\n#         return \" \".join(res)\n\n# # --- 6. 自动化权重调优 (简单网格搜索演示) ---\n# print(\"Optimizing weights on validation set...\")\n# best_w = (W_CHAR, W_WORD, W_SEQ)\n# # 这里可以写一个循环遍历不同的 w_c, w_w 组合，计算验证集的 BLEU\n\n# # --- 7. 执行测试集预测 ---\n# test = pd.read_csv('/kaggle/input/deep-past-initiative-machine-translation/test.csv')\n# #test=test.head(1000)\n# final_preds = []\n# for q in test[\"transliteration\"]:\n#     final_preds.append(translate_query(q, W_CHAR, W_WORD, W_SEQ))\n\n# # 保存提交\n# submission_tr = pd.DataFrame({\"id\": test[\"id\"], \"translation\": final_preds})\n# submission_tr.to_csv(\"submission_tr.csv\", index=False)\n# submission_tr.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T16:54:40.949988Z","iopub.execute_input":"2025-12-18T16:54:40.950212Z","iopub.status.idle":"2025-12-18T16:54:42.279915Z","shell.execute_reply.started":"2025-12-18T16:54:40.950190Z","shell.execute_reply":"2025-12-18T16:54:42.279213Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# SentenceAlign: append longer segment to shorter string","metadata":{}},{"cell_type":"code","source":"# end_result=[]\n# for k in range(len(submission_Deep)):\n#     text_Deep=submission_Deep.iloc[k][\"translation\"]\n#     text_tr=submission_tr.iloc[k][\"translation\"]\n\n#     if k < 4:\n#         print(text_Deep)\n        \n#     text_source=test.iloc[k][\"transliteration\"]\n#     if len(text_source)>=256:\n#         if len(text_tr)>len(text_Deep):\n#             blank_matches=re.finditer(r' ', text_tr)\n#             blank_positions = [m.start() for m in blank_matches]\n#             if blank_positions:  \n#                 text_Deep=text_Deep[:350]\n#                 text_Deep_len=len(text_Deep)\n#                 closest_blank = min(blank_positions, key=lambda x: abs(text_Deep_len - x))\n#                 text_Deep+=text_tr[closest_blank:]\n    \n#     if not text_Deep.endswith(\".\"):\n#         text_Deep += \".\"\n\n#     if k < 4:\n#         print()\n#         print(text_Deep)\n#         print(\"*\"*100)\n        \n#     end_result.append(text_Deep)\n    \n# submission_Deep[\"translation\"]=end_result\n# submission_Deep.to_csv(\"submission.csv\",index=False)\n# submission_Deep.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-18T16:55:33.412737Z","iopub.execute_input":"2025-12-18T16:55:33.413293Z","iopub.status.idle":"2025-12-18T16:55:33.426034Z","shell.execute_reply.started":"2025-12-18T16:55:33.413265Z","shell.execute_reply":"2025-12-18T16:55:33.425452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}