{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":121150,"databundleVersionId":15061024,"sourceType":"competition"},{"sourceId":288835156,"sourceType":"kernelVersion"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport re\nimport unicodedata\n\ndef prepare_super_dataset():\n    # 1. 載入原始訓練集與您的純化辭典\n    train_df = pd.read_csv(\"/kaggle/input/deep-past-initiative-machine-translation/train.csv\")\n    purified_lexicon = pd.read_csv(\"/kaggle/input/notebookfb3a681aee/titan_v1873_final_purified.csv\") # 您之前的 1.5 萬筆數據\n\n    # 2. 執行 Baseline 的句子對齊邏輯 (吸取其精華)\n    aligned_data = []\n    for _, row in train_df.iterrows():\n        src, tgt = str(row['transliteration']), str(row['translation'])\n        tgt_sents = [t.strip() for t in re.split(r'(?<=[.!?])\\s+', tgt) if t.strip()]\n        src_lines = [s.strip() for s in src.split('\\n') if s.strip()]\n        \n        if len(tgt_sents) > 1 and len(tgt_sents) == len(src_lines):\n            for s, t in zip(src_lines, tgt_sents):\n                aligned_data.append({'src': s, 'tgt': t})\n        else:\n            aligned_data.append({'src': src, 'tgt': tgt})\n    \n    df_aligned = pd.DataFrame(aligned_data)\n\n    # 3. 合併辭典與句子\n    df_lexicon = purified_lexicon[['src', 'tgt']]\n    df_final = pd.concat([df_aligned, df_lexicon]).drop_duplicates().sample(frac=1).reset_index(drop=True)\n    \n    # 4. NFC 正規化\n    df_final['src'] = df_final['src'].apply(lambda x: unicodedata.normalize('NFC', str(x)))\n    df_final['tgt'] = df_final['tgt'].apply(lambda x: unicodedata.normalize('NFC', str(x)))\n    \n    df_final.to_csv(\"titan_v1879_super_aligned.csv\", index=False)\n    print(f\"✅ Super Dataset Ready: {len(df_final)} samples.\")\n\nprepare_super_dataset()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}