{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%writefile cleaner.py\nimport pandas as pd\nimport re\nimport os\nfrom fractions import Fraction\nimport ast\nimport json\nimport uuid\nimport unicodedata\nfrom config import DATA_PATH, EDA_PATH\n\ndef check_malformed_literal(df, col_name):\n    \"\"\"\n    检查指定列中哪些行无法被 ast.literal_eval 解析。\n    \"\"\"\n    print(f\"检查列是否符合列表格式: {col_name} ...\")\n    error_count = 0\n    \n    for index, value in df[col_name].items():\n        # 只处理字符串类型，因为 ast.literal_eval 只接受字符串\n        if isinstance(value, str):\n            try:\n                ast.literal_eval(value)\n            except (ValueError, SyntaxError) as e:\n                error_count += 1\n                print(\"-\" * 30)\n                print(f\"Index: {index}\")\n                print(f\"Error Type: {type(e).__name__}\")\n                print(f\"Content: {repr(value)}\") # 使用 repr 打印出原始字符串（包括空格和换行）\n\ndef process_fractions_dynamic(text_to_process):\n    if not isinstance(text_to_process, str):\n        return str(text_to_process)\n    # Step0: 预处理\n    # --- 0.1 清理数值边界的特殊标记符号 ---\n    # (123) [123] ˹123˺ ⌈123⌋ 「123」 ⸢123⸣ → 123\n    text_to_process = re.sub(r'\\((\\+?\\d+)\\)', r'\\1', text_to_process)\n    text_to_process = re.sub(r'\\[(\\+?\\d+)\\]', r'\\1', text_to_process)\n    text_to_process = re.sub(r'˹(\\+?\\d+)˺', r'\\1', text_to_process)\n    text_to_process = re.sub(r'⌈(\\+?\\d+)⌋', r'\\1', text_to_process)\n    text_to_process = re.sub(r'「(\\+?\\d+)」', r'\\1', text_to_process)\n    text_to_process = re.sub(r'⸢(\\+?\\d+)⸣', r'\\1', text_to_process)\n\n    # --- 0.2 转换 X+X 格式的数值：循环处理直到没有更多加法运算需要处理 ---\n    def calculate_addition(match):\n        try:\n            val1 = float(match.group(1))\n            val2 = float(match.group(2))\n            result = val1 + val2\n            # 格式化逻辑: 如果是整数则转int str，否则保留有效小数\n            return \"{:.5f}\".format(result).rstrip('0').rstrip('.') if result % 1 != 0 else str(int(result))\n        except ValueError:\n            return match.group(0)\n    addition_pattern = r'(\\d+(?:\\.\\d+)?)\\s*\\+\\s*(\\d+(?:\\.\\d+)?)'\n    while True:\n        new_text = re.sub(addition_pattern, calculate_addition, text_to_process)\n        if new_text == text_to_process:\n            break\n        text_to_process = new_text\n\n    # --- 0.3 月份替换 ---\n    # Month I - month XII -> month 1 - month 12\n    roman_to_arabic = {\n        \"I\": \"1\", \"II\": \"2\", \"III\": \"3\", \"IV\": \"4\", \"V\": \"5\", \"VI\": \"6\",\n        \"VII\": \"7\", \"VIII\": \"8\", \"IX\": \"9\", \"X\": \"10\", \"XI\": \"11\", \"XII\": \"12\"\n    }\n    roman_pattern = r'\\b(Month|month)\\s+(VIII|XII|VII|III|XI|IX|VI|IV|II|X|V|I)\\b'\n    def replace_roman(match):\n        return f\"{match.group(1)} {roman_to_arabic[match.group(2)]}\"\n    text_to_process = re.sub(roman_pattern, replace_roman, text_to_process)\n\n    # Step1: 转换 X / X 格式的分数\n    # --- 1.1 分数转换：规则内的分数转换到 Unicode 分数符号 ---\n    fraction_map = {\n        r\"1\\s*/\\s*6\": \"⅙\",\n        r\"1\\s*/\\s*4\": \"¼\",\n        r\"1\\s*/\\s*3\": \"⅓\",\n        r\"1\\s*/\\s*2\": \"½\",\n        r\"2\\s*/\\s*3\": \"⅔\",\n        r\"3\\s*/\\s*4\": \"¾\",\n        r\"5\\s*/\\s*6\": \"⅚\",\n        # r\"1\\s*/\\s*8\": \"⅛\", r\"3\\s*/\\s*8\": \"⅜\", r\"5\\s*/\\s*8\": \"⅝\", r\"7\\s*/\\s*8\": \"⅞\",\n    }\n    for pattern, char in fraction_map.items():\n        text_to_process = re.sub(pattern, char, text_to_process)\n\n    # --- 1.2 通用数值分数转换：剩余的分数形式转换为小数格式 ---\n    def calc_fraction(match):\n        groups = match.groups()\n        if groups[0]: # 如果匹配到了整数部分 (例如 \"5 11/12\")\n            integer_part = int(groups[0])\n            num = int(groups[1])\n            den = int(groups[2])\n            val = integer_part + (num / den)\n        else: # 只匹配到了分数部分 (例如 \"11/12\")\n            num = int(groups[3])\n            den = int(groups[4])\n            val = num / den\n        # 格式化为5位小数 (例如 5.916666... -> 5.91667)\n        return \"{:.5f}\".format(val).rstrip('0').rstrip('.') if val % 1 != 0 else str(int(val))\n    # 模式1 (混合分数): (\\d+)\\s+(\\d+)/(\\d+) -> 匹配 \"5 11/12\"  |  模式2 (纯分数): (\\d+)/(\\d+) -> 匹配 \"11/12\"\n    fraction_pattern = r'(\\d+)\\s+(\\d+)/(\\d+)|(\\d+)/(\\d+)'   \n    text_to_process = re.sub(fraction_pattern, calc_fraction, text_to_process)\n    \n    # Step2: 转换小数格式的分数到 Unicode 分数符号\n    # --- 2.1 处理长浮点数损坏数值 ---\n    # 正则逻辑：匹配 数字 + 小数点 + 至少5位数字\n    # \\d+\\.\\d{5,} 匹配如 1.3333300000000001\n    # lambda 表达式将其截断至小数点后 5 位\n    text_to_process = re.sub(r'(\\d+\\.\\d{5})\\d+', r'\\1', text_to_process)\n\n    # --- 2.2 统一成 UNICODE 分数符号 ---\n    fraction_lookup = {\n            (1, 6): \"⅙\", # .166\n            (1, 4): \"¼\", # .25\n            (1, 3): \"⅓\", # .333\n            (1, 2): \"½\", # .5\n            (2, 3): \"⅔\", # .666\n            (3, 4): \"¾\", # .75\n            (5, 6): \"⅚\", # .833\n        #    (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n        #    (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n    }\n    def replacer(match):\n        full_str = match.group(0)\n        try:\n            val = float(full_str)\n            integer_part = int(val)\n            decimal_part = val - integer_part\n            # Return integer if decimal is negligible\n            if decimal_part < 0.0001: \n                return str(integer_part)\n            # Limit denominator handles 0.3333 -> 1/3\n            frac = Fraction(decimal_part).limit_denominator(12) \n\n            unicode_frac = fraction_lookup.get((frac.numerator, frac.denominator))\n            if unicode_frac:\n                if integer_part == 0:\n                    return unicode_frac\n                else:\n                    # Current logic: Space between Integer and Fraction\n                    return f\"{integer_part} {unicode_frac}\"\n\n            return full_str \n        except ValueError:\n            return full_str\n    processed_text = re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text_to_process)\n    \n    # Step3: Robustness check\n    # --- 3.1 保证带分数中间有空格 ---\n    processed_text = re.sub(r'(\\d)([¼½¾⅓⅔⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞])', r'\\1 \\2', processed_text)\n    \n    return processed_text\n\ndef preprocess_transliteration(text):\n    if pd.isna(text):\n        return \"\"\n    # --- 0. oare 数据特殊符号移除 ---\n    pattern = r\"\\s*Seal Impression [A-Z]\\s*\"\n    text = re.sub(pattern, \"\", text)\n    pattern = r\"Seal Impression\"\n    text = re.sub(pattern, \"\", text)\n    pattern = r\"broken\"\n    text = re.sub(pattern, \"xxx\", text)\n    pattern = r\"------------\"\n    text = re.sub(pattern, \"\", text)\n    pattern = r\"---- Single Ruling ----\"\n    text = re.sub(pattern, \"\", text)\n    pattern = r\"---- Double Ruling ----\"\n    text = re.sub(pattern, \"\", text)\n    \n    # --- 1. 字符替换与括号处理 ---\n    char_map = str.maketrans({\n        \"X\": \"x\",\n        \"×\": \"x\",\n        \"ā\": \"a\",\n        \"Ş\": \"Ṣ\",\n        \"ş\": \"ṣ\",\n        \"Ș\": \"Ṣ\",\n        \"ș\": \"ṣ\",\n        \"Ț\": \"Ṭ\",\n        \"ț\": \"ṭ\",\n        \"İ\": \"I\",\n        \"Ī\": \"I\",\n        \"ī\": \"i\",\n        \"Î\": \"I\",\n        \"î\": \"i\",\n        \"ı\": \"i\",\n        \"h\": \"ḫ\",\n        \"H\": \"Ḫ\",\n        \"ḥ\": \"ḫ\",\n        \"Ḥ\": \"Ḫ\",\n        \"=\": \"-\",\n        \"(\": \"{\",  # for deterministic\n        \")\": \"}\",\n        \"—\": \"-\",\n        \"–\": \"-\",\n    })\n    text = text.translate(char_map)\n    text = text.replace(\"ᵈ\", r\"{d}\").replace(\"ᵏⁱ\", r\"{ki}\")   # 手动操作\n    text = text.replace(r\"{{\", \"{\").replace(r\"}}\", \"}\")\n    # text = text.replace(\"lá+lá\", \"lá-lá\").replace(\"la+la\", \"la-la\")\n    text = text.replace(\"??\", \"?\").replace(\"!!\", \"!\").replace(\"//\", \"/\")\n    text = text.replace(\"-/\", \"-\")\n    text = text.replace(\"{?}\", \"\").replace(\"{!}\", \"\")\n    text = text.replace(\"(?)\", \"\").replace(\"(!)\", \"\")\n    # Remove bad characters\n    upper_chars = [\"⁰\",\"¹\",\"²\",\"³\",\"⁴\",\"⁵\",\"⁶\",\"⁷\",\"⁸\",\"⁹\",\"⁻\",\"°\", \"˚\"]\n    for ch in upper_chars:\n        text = text.replace(ch, \"\")\n    bad_chars = \"\"\"\"ʺʾ’´ʼˊ˘ˋˇˈʿʹ'`^˹˺⸢⸣⌜⌝「」⌈⌋⌊⌉˻˼⸤⸥≪≫《》⟪⟫«»〈〉⟨⟩˃‹›⁽⁾ˁˀ\"\"\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n    bad_chars = \"\"\"!?⸮ʔ⸮,/;˷˴|:⁺*\"\"\"  # +\n    for ch in bad_chars:\n        text = text.replace(ch, \" \")\n\n    # --- 2.1 处理数值符号 ---\n    text = process_fractions_dynamic(text)\n    \n    # --- 2. 角标统一处理 ---\n    # 4 a-bi4-a 0.5 2+10 ù a-šùr-DU10: 4 a-bi₄-a 0.5 2+10 ù a-šùr-DU₁₀\n    # 定义允许的字母范围（包含普通字母和特殊字符）\n    letters_str = \"a-zA-ZšŠṣṢṭṬḫḪ\"\n    # 定义数字到角标的映射表\n    subscript_trans = str.maketrans(\"0123456789\", \"₀₁₂₃₄₅₆₇₈₉\")\n    # 正则逻辑：匹配紧跟在字母后面的数字\n    def replace_group_with_subscript(match):\n        digits = match.group(0)\n        return digits.translate(subscript_trans)\n    text = re.sub(f\"(?<=[{letters_str}])\\d+\", replace_group_with_subscript, text)\n\n    # --- 2. 建议替换的标注 ---\n    # ['(fem.)', '(sing.)', '(pl.)', '(plural)', '(f.)', '(plur.)'] -> ''\n    pattern = r'\\(\\s*(?:fem\\.|sing\\.|pl\\.|plural|f\\.|plur\\.)(?:\\s+(?:fem\\.|sing\\.|pl\\.|plural|f\\.|plur\\.))*\\s*\\)'\n    text = re.sub(pattern, '', text)\n    #  ['(K)', '(Rs)', '(lR)'] -> ''\n    pattern = r'\\(\\s*(?:K|Rs|lR)(?:\\s+(?:K|Rs|lR))*\\s*\\)'\n    text = re.sub(pattern, '', text)\n    # ['le.e.', 'lo.e.', 'l.o.e.', 'obv.', 'rev.', 'r.e.', 'u.e.'] -> ''\n    pattern = r'\\b(?:le\\.e\\.|lo\\.e\\.|l\\.o\\.e\\.|obv\\.|rev\\.|r\\.e\\.|u\\.e\\.)\\s*'\n    text = re.sub(pattern, '', text)\n\n    # --- 3. Gap替换 ---\n    # ['PN'/'Pn'] -> '<gap>'\n    pattern = r'\\bPN\\b|\\bPn\\b'\n    text = re.sub(pattern, '<gap>', text)\n    # break/broken -> ''\n    text = re.sub(r'\\bbreak\\b|\\bbroken\\b', '', text)\n    # [] -> [xxx]\n    text = re.sub(r'\\[[\\sx?]*\\]', '[xxx]', text)\n    bad_chars = \"[]\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n    # Multi-dot patterns → <gap>\n    text = re.sub(r'(?:\\.\\s+){2,}\\.', lambda m: m.group(0).replace(' ', ''), text)\n    text = re.sub(r'\\.{3,}(\\s+\\.{3,})*', '<gap>', text)\n    text = re.sub(r'……|…', '<gap>', text)\n    # x patterns → <gap>\n    text = re.sub(r'xx+', '<gap>', text)\n    # 处理孤立的 x (前后是空格或边界)\n    text = re.sub(r'\\bx\\b', '<gap>', text)\n    # 直接替换 {large break}\n    text = text.replace(\"{large break}\", \"<gap>\")\n    # 正则替换 {N broken lines} (例如 {3 broken lines}, {4 broken lines})\n    text = re.sub(r'\\{\\d+\\s+broken\\s+lines\\}', '<gap>', text)\n    # Temporarily protect tokens from cleanup\n    text = text.replace(\"<gap>\", \"\\x00GAP\\x00\")\n    # Remove bad characters\n    bad_chars = \"<>\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n    # Restore tokens\n    text = text.replace(\"\\x00GAP\\x00\", \" <gap> \")\n    \n    # Normalize whitespace and trim\n    text = re.sub(r\"\\s+\", \" \", text).strip().strip(\"-\")\n    text = text.replace(\" -\", \"-\").replace(\"- \", \"-\")\n    text = text.replace(\" .\", \".\").replace(\". \", \".\")\n    text = text.replace(\"> .\", \">.\").replace(\"> :\", \">:\").replace(\"> ’\", \">’\")\n\n    # merge <gap>\n    pattern = r'<gap>([ \\-\\t]*<gap>)+'\n    text = re.sub(pattern, '<gap>', text)\n    text = text.replace(\"{ <gap> }\", \"<gap>\")\n    text = text.replace(\"[ <gap> ]\", \"<gap>\")\n    \n    return text\n\ndef preprocess_train_translation(text):\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    \n    # --- 1. 字符替换与括号处理 ---\n    char_map = str.maketrans({\n        \"X\": \"x\",\n        \"ʾ\": \"'\",\n        \"Ş\": \"Ṣ\",\n        \"ş\": \"ṣ\",\n        \"ș\": \"ṣ\",\n        \"Ț\": \"Ṭ\",\n        \"ț\": \"ṭ\",\n        \"ḫ\": \"h\",\n        \"Ḫ\": \"H\",\n        \"ḥ\": \"ḫ\",\n        \"Ḥ\": \"Ḫ\",\n    })\n    text = text.translate(char_map)\n    text = text.replace(\"{?}\", \"\").replace(\"{!}\", \"\")\n    text = text.replace(\"(?)\", \"\").replace(\"(!)\", \"\")\n    # Remove bad characters\n    upper_chars = [\"⁰\",\"¹\",\"²\",\"³\",\"⁴\",\"⁵\",\"⁶\",\"⁷\",\"⁸\",\"⁹\",\"⁻\",\"°\",\"˚\"]\n    for ch in upper_chars:\n        text = text.replace(ch, \"\")\n    lower_chars = [\"₀\",\"₁\",\"₂\",\"₃\",\"₄\",\"₅\",\"₆\",\"₇\",\"₈\",\"₉\"]\n    for ch in lower_chars:\n        text = text.replace(ch, \"\")\n    bad_chars = \"\"\"\"ʺˊ˘ˋˇˈʿ`^˹˺⸢⸣⌜⌝「」⌈⌋⌊⌉˻˼⸤⸥≪≫《》⟪⟫«»〈〉⟨⟩˃‹›⁽⁾ˁˀ\"\"\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n    bad_chars = \"\"\"ʹ'ʾ´ʼ\"\"\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"’\")\n\n    # --- 2.1 处理数值符号 ---\n    text = process_fractions_dynamic(text)\n\n    # --- 2.2 建议替换的标注 ---\n    # ['(fem.)', '(sing.)', '(pl.)', '(plural)', '(f.)', '(plur.)'] -> ''\n    pattern = r'\\(\\s*(?:fem\\.|sing\\.|pl\\.|plural|f\\.|plur\\.)(?:\\s+(?:fem\\.|sing\\.|pl\\.|plural|f\\.|plur\\.))*\\s*\\)'\n    text = re.sub(pattern, '', text)\n    #  ['(K)', '(Rs)', '(lR)'] -> ''\n    pattern = r'\\(\\s*(?:K|Rs|lR)(?:\\s+(?:K|Rs|lR))*\\s*\\)'\n    text = re.sub(pattern, '', text)\n    # ['le.e.', 'lo.e.', 'l.o.e.', 'obv.', 'rev.', 'r.e.', 'u.e.'] -> ''\n    pattern = r'\\b(?:le\\.e\\.|lo\\.e\\.|l\\.o\\.e\\.|obv\\.|rev\\.|r\\.e\\.|u\\.e\\.)\\s*'\n    text = re.sub(pattern, '', text)\n\n    # --- 3. Gap替换 ---\n    # ['PN'] -> '<gap>'\n    pattern = r'\\bPN\\b|\\bPn\\b'\n    text = re.sub(pattern, '<gap>', text)\n    # [] -> [xxx]\n    text = re.sub(r'\\[[\\sx?]*\\]', '[xxx]', text)\n    # Multi-dot patterns → <gap>\n    text = re.sub(r'(?:\\.\\s+){2,}\\.', lambda m: m.group(0).replace(' ', ''), text)\n    text = re.sub(r'\\.{3,}(\\s+\\.{3,})*', '<gap>', text)\n    text = re.sub(r'……|…', '<gap>', text)\n    # x patterns → <gap>\n    text = re.sub(r'xx+', '<gap>', text, flags=re.I)\n    # 处理孤立的 x (前后是空格或边界)\n    text = re.sub(r'\\bx\\b', '<gap>', text, flags=re.I)\n    # 直接替换 {large break}\n    text = text.replace(\"{large break}\", \"<gap>\")\n    # 正则替换 {N broken lines} (例如 {3 broken lines}, {4 broken lines})\n    text = re.sub(r'\\{\\d+\\s+broken\\s+lines\\}', '<gap>', text)\n    # Temporarily protect tokens from cleanup\n    text = text.replace(\"<gap>\", \"\\x00GAP\\x00\")\n    # Remove bad characters\n    bad_chars = '˹˺⸢⸣「」⌈⌋⌊|'  # []\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n    bad_chars = \"/*+\"\n    for ch in bad_chars:\n        text = text.replace(ch, \" \")\n    # Restore tokens\n    text = text.replace(\"\\x00GAP\\x00\", \" <gap> \")\n\n    # Normalize whitespace and trim\n    text = re.sub(r\"\\s+\", \" \", text).strip().strip(\"-\")\n    text = text.replace(\" -\", \"-\").replace(\"- \", \"-\")\n    text = text.replace(\"> .\", \">.\").replace(\"> :\", \">:\")\n\n    # merge <gap>\n    pattern = r'<gap>([ \\-\\t]*<gap>)+'\n    text = re.sub(pattern, '<gap>', text)\n    text = text.replace(\"{ <gap> }\", \"<gap>\")\n    text = text.replace(\"[ <gap> ]\", \"<gap>\")\n    \n    return text\n\nif __name__ == \"__main__\":\n    data_dir = DATA_PATH\n    df_path = os.path.join(data_dir, \"synth_df0308_en.csv\")\n    df_target_lang_code = 'en'\n    TRANSLITERATION_COL = 'transliteration'\n    TRANSLATION_COL = f'{df_target_lang_code}_translation'\n    output_path = os.path.join(data_dir, \"pipeline/train_synth_akka2en-sentlevel0308.csv\")\n\n    df = pd.read_csv(df_path, dtype={'chapter_id': str})\n    print(f\"Read df: {df.shape} <- {df_path}\")\n    df = df.dropna(subset=[TRANSLATION_COL])\n    print(f\"After dropna: {df.shape}\")\n    \n    cols_to_fix = ['line_range', TRANSLITERATION_COL, TRANSLATION_COL]\n    for col in cols_to_fix:\n        check_malformed_literal(df, col)\n        # 使用 ast.literal_eval 安全地解析字符串列表\n        df[col] = df[col].apply(lambda x: ast.literal_eval(x) if isinstance(x, str) else x)\n\n    # Check for length mismatch before explode\n    for idx, row in df.iterrows():\n        l1 = len(row['line_range']) if isinstance(row['line_range'], list) else 0\n        l2 = len(row[TRANSLITERATION_COL]) if isinstance(row[TRANSLITERATION_COL], list) else 0\n        l3 = len(row[TRANSLATION_COL]) if isinstance(row[TRANSLATION_COL], list) else 0\n        if not (l1 == l2 == l3):\n            print(f\"Mismatch found - oare_id: {row.get('oare_id', 'Unknown')}, lengths: line_range={l1}, transliteration={l2}, translation={l3}\")\n\n    df = df.explode(cols_to_fix)\n    print(f\"After explode: {df.shape}\")\n\n    # df[TRANSLITERATION_COL] = df[TRANSLITERATION_COL].apply(process_fractions_dynamic)\n    # df[TRANSLATION_COL] = df[TRANSLATION_COL].apply(process_fractions_dynamic)\n    df[TRANSLITERATION_COL] = df[TRANSLITERATION_COL].apply(preprocess_transliteration)\n    df[TRANSLATION_COL] = df[TRANSLATION_COL].apply(preprocess_train_translation)\n    df[f'{TRANSLITERATION_COL}_char_length'] = df.apply(lambda row: len(row[TRANSLITERATION_COL]), axis=1)\n    df[f'{TRANSLATION_COL}_char_length'] = df.apply(lambda row: len(row[TRANSLATION_COL]), axis=1)\n\n    # df = df.drop_duplicates(subset=[TRANSLITERATION_COL, TRANSLATION_COL])\n    df['sentence_id'] = df.groupby(['publication_name', 'chapter_id']).cumcount() + 1\n    print(f\"Save to csv: {df.shape} -> {output_path}\")\n    df.to_csv(output_path, index=False)\n\n    # df = pd.read_csv(output_path)\n    # print(f\"Read df: {df.shape} <- {output_path}\")\n    # # 统计字符，写成json文件\n    # for col in [TRANSLATION_COL, TRANSLITERATION_COL]:\n    #     unique_chars_set = set()\n    #     for text in df[col].fillna(\"\").astype(str):\n    #         unique_chars_set.update(list(text))\n    #     unique_chars_list = sorted(list(unique_chars_set))\n    #     char_eda_path = os.path.join(EDA_PATH, f\"chars_in_train_ocr_akka2en-sentlevel0318_{col}.json\")\n    #     with open(char_eda_path, \"w\", encoding=\"utf-8\") as f:\n    #         json.dump(unique_chars_list, f, ensure_ascii=False, indent=4)\n    #     print(f\"Save char eda: {char_eda_path}\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}