{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"}],"dockerImageVersionId":31234,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport re\nimport os\nfrom tqdm.auto import tqdm\n\n# ------------------------------------------------------------\n# 1. 配置與路徑\n# ------------------------------------------------------------\nPATH_BASE = \"/kaggle/input/deep-past-initiative-machine-translation/\"\nLEXICON_PATH = os.path.join(PATH_BASE, \"OA_Lexicon_eBL.csv\")\nPUBS_PATH = os.path.join(PATH_BASE, \"publications.csv\")\nPUB_TEXTS_PATH = os.path.join(PATH_BASE, \"published_texts.csv\")\nTRAIN_PATH = os.path.join(PATH_BASE, \"train.csv\")\n\n# ------------------------------------------------------------\n# 2. 官方格式清理邏輯 (Official Format Alignment)\n# ------------------------------------------------------------\ndef official_format_cleaner(text):\n    if not isinstance(text, str): return \"\"\n    # 移除現代標註: !, ?, /, ˹, ˺, [, ]\n    text = re.sub(r'[!?/˹˺\\[\\]]', '', text)\n    # 限定詞格式校正 (d) -> {d}\n    text = re.sub(r'\\((\\w+)\\)', r'{\\1}', text)\n    # 標籤對齊\n    text = text.replace(\"<gap>\", \"x\").replace(\"<big_gap>\", \"...\")\n    return re.sub(r'\\s+', ' ', text).strip()\n\n# ------------------------------------------------------------\n# 3. 詞典引擎載入 (Lexicon Normalization)\n# ------------------------------------------------------------\nlex_df = pd.read_csv(LEXICON_PATH)\n# 優先處理長單詞避免子字串誤殺\nlex_df['form_len'] = lex_df['form'].str.len()\nlex_df = lex_df.sort_values('form_len', ascending=False)\nnorm_map = dict(zip(lex_df['form'].astype(str), lex_df['norm'].astype(str)))\n\ndef lexicon_normalize(text):\n    if not isinstance(text, str): return \"\"\n    for form, norm in norm_map.items():\n        if form in text:\n            pattern = r'\\b' + re.escape(form) + r'\\b'\n            text = re.sub(pattern, norm, text)\n    return text\n\nprint(\"✅ Lexicon & Official Cleaner Ready.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. 準備錨點與資料\npub_meta = pd.read_csv(PUB_TEXTS_PATH)\nanchors = set(pub_meta['oare_id'].dropna().astype(str).tolist() + \n              pub_meta['label'].dropna().astype(str).tolist())\n# 過濾掉太短的錨點，防止過多誤報\nvalid_anchors = [a for a in anchors if len(a) > 6]\n\n# 2. 正則分塊編譯 (避免單個正則表達式過長導致溢出)\nCHUNK_SIZE = 400\nanchor_chunks = [valid_anchors[i:i + CHUNK_SIZE] for i in range(0, len(valid_anchors), CHUNK_SIZE)]\nchunk_patterns = [re.compile(\"|\".join([re.escape(a) for a in chunk])) for chunk in anchor_chunks]\n\n# 3. 採礦執行\npubs_df = pd.read_csv(PUBS_PATH)\nakk_pages = pubs_df[pubs_df['has_akkadian'] == True]\nakk_pattern = re.compile(r'\\b([a-zšṣṭḫ0-9₀-₉]+(?:-[a-zšṣṭḫ0-9₀-₉]+)+)\\b', re.I)\n\nextra_data = []\nprint(f\"🚀 Starting High-Speed Mining (30k pages x {len(chunk_patterns)} patterns)...\")\n\nfor _, row in tqdm(akk_pages.iterrows(), total=len(akk_pages)):\n    text = str(row['page_text'])\n    \n    # 快速初步過濾\n    hit_anchor = None\n    for cp in chunk_patterns:\n        match = cp.search(text)\n        if match:\n            hit_anchor = match.group()\n            break\n            \n    if hit_anchor:\n        lines = text.split('\\n')\n        cur_akk, cur_eng = [], []\n        for line in lines:\n            line = line.strip()\n            if not line: continue\n            # 判斷轉寫行\n            if len(akk_pattern.findall(line)) >= 2:\n                cur_akk.append(line)\n            elif cur_akk and len(line.split()) > 4:\n                cur_eng.append(line)\n        \n        if cur_akk and cur_eng:\n            extra_data.append({\n                \"src\": \" \".join(cur_akk),\n                \"tgt\": \" \".join(cur_eng)\n            })\n\nextra_df = pd.DataFrame(extra_data).drop_duplicates(subset=['src'])\nprint(f\"💎 Mining Success! Found {len(extra_df)} high-quality pairs.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. 載入與處理原始訓練集\ntrain_df = pd.read_csv(TRAIN_PATH)\n\n# 2. 定義統一處理流\ndef final_pipeline(text):\n    text = lexicon_normalize(text) # 詞典正規化\n    text = official_format_cleaner(text) # 官方格式清理\n    return text\n\n# 3. 合併與生成最終 CSV\nprint(\"🛠️ Applying final alignment to all data...\")\ntrain_rows = []\nfor _, row in train_df.iterrows():\n    train_rows.append({\n        \"input\": final_pipeline(row['transliteration']),\n        \"target\": row['translation']\n    })\n\nextra_rows = []\nif not extra_df.empty:\n    for _, row in extra_df.iterrows():\n        extra_rows.append({\n            \"input\": final_pipeline(row['src']),\n            \"target\": row['tgt']\n        })\n\nfinal_augmented_df = pd.DataFrame(train_rows + extra_rows)\nfinal_augmented_df = final_augmented_df.drop_duplicates(subset=['input'])\n\n# 4. 存檔\nfinal_augmented_df.to_csv(\"titan_final_augmented.csv\", index=False)\nprint(f\"🌟 Final Dataset Size: {len(final_augmented_df)} rows.\")\nfinal_augmented_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}