{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391},{"sourceType":"datasetVersion","sourceId":15327792,"datasetId":9803655,"databundleVersionId":16234975},{"sourceType":"datasetVersion","sourceId":15326426,"datasetId":9802618,"databundleVersionId":16233466},{"sourceType":"datasetVersion","sourceId":15327805,"datasetId":9803668,"databundleVersionId":16234989},{"sourceType":"datasetVersion","sourceId":15327030,"datasetId":9803092,"databundleVersionId":16234122},{"sourceType":"datasetVersion","sourceId":15326883,"datasetId":9802974,"databundleVersionId":16233961},{"sourceType":"datasetVersion","sourceId":15322203,"datasetId":9800006,"databundleVersionId":16228574},{"sourceType":"datasetVersion","sourceId":15326395,"datasetId":9802605,"databundleVersionId":16233434},{"sourceType":"datasetVersion","sourceId":15324021,"datasetId":9801099,"databundleVersionId":16230650},{"sourceType":"datasetVersion","sourceId":15327917,"datasetId":9803719,"databundleVersionId":16235114},{"sourceType":"datasetVersion","sourceId":15327835,"datasetId":9803688,"databundleVersionId":16235025},{"sourceType":"datasetVersion","sourceId":15327007,"datasetId":9803076,"databundleVersionId":16234095},{"sourceType":"datasetVersion","sourceId":15327817,"datasetId":9803677,"databundleVersionId":16235002},{"sourceType":"modelInstanceVersion","sourceId":798279,"databundleVersionId":16225889,"modelInstanceId":608012,"modelId":619926},{"sourceType":"modelInstanceVersion","sourceId":799701,"databundleVersionId":16238605,"modelInstanceId":608941,"modelId":620795},{"sourceType":"modelInstanceVersion","sourceId":799898,"databundleVersionId":16240393,"modelInstanceId":609092,"modelId":620943},{"sourceType":"modelInstanceVersion","sourceId":799903,"databundleVersionId":16240424,"modelInstanceId":609095,"modelId":620945},{"sourceType":"modelInstanceVersion","sourceId":799942,"databundleVersionId":16240771,"modelInstanceId":609127,"modelId":620975},{"sourceType":"modelInstanceVersion","sourceId":799904,"databundleVersionId":16240436,"modelInstanceId":609096,"modelId":620946},{"sourceType":"modelInstanceVersion","sourceId":799944,"databundleVersionId":16240783,"modelInstanceId":609129,"modelId":620977},{"sourceType":"modelInstanceVersion","sourceId":799920,"databundleVersionId":16240565,"modelInstanceId":609109,"modelId":620958},{"sourceType":"modelInstanceVersion","sourceId":799941,"databundleVersionId":16240757,"modelInstanceId":609126,"modelId":620974},{"sourceType":"modelInstanceVersion","sourceId":799928,"databundleVersionId":16240647,"modelInstanceId":609116,"modelId":620966},{"sourceType":"modelInstanceVersion","sourceId":799934,"databundleVersionId":16240712,"modelInstanceId":609121,"modelId":620970},{"sourceType":"modelInstanceVersion","sourceId":799946,"databundleVersionId":16240824,"modelInstanceId":609131,"modelId":620979}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":148.047288,"end_time":"2026-03-19T11:45:07.808063","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-03-19T11:42:39.760775","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0039c6cacd80415db6dc2592fc285ef1":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"025dc84a2bf14e6dbe6f5587fc5b40fa":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_d7216e4e518c43e2bdad423b7a23464f","placeholder":"​","style":"IPY_MODEL_88ed2cfe9a264830a8717ba4639f1103","tabbable":null,"tooltip":null,"value":"Generating translations: 100%"}},"119ba4b80dee4c5a824665f0affba043":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_025dc84a2bf14e6dbe6f5587fc5b40fa","IPY_MODEL_92a3196be3e44969a1a6f43847a016e7","IPY_MODEL_27b74d93bdbc424e99d6db28c3160609"],"layout":"IPY_MODEL_16d2371943ba461fa4598b7d2c91cf40","tabbable":null,"tooltip":null}},"16d2371943ba461fa4598b7d2c91cf40":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1eb7ef2cad76422caf72d331ad6299fd":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_e8acde88cbff4ee6808e41f460f3774c","IPY_MODEL_c61219568c124054b31a7f2cf129a4ba","IPY_MODEL_a57df47e428343f38eca6e01cde35ddf"],"layout":"IPY_MODEL_c78341b3b0cb4ff9896c5ede030afd40","tabbable":null,"tooltip":null}},"240037c8295e4aaebe8a9ad853131e5f":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"27b74d93bdbc424e99d6db28c3160609":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_ecaa9d9f3d8e4e1fa9eb38776f345ed5","placeholder":"​","style":"IPY_MODEL_240037c8295e4aaebe8a9ad853131e5f","tabbable":null,"tooltip":null,"value":" 1/1 [00:53&lt;00:00, 53.00s/it]"}},"3121bfdea6194f8d9fe3ac81a16f1c24":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"36367e99d1f6448ebd7eb54a349faa93":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"446959eac3ed494594b6b0265761d861":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"4635236c6b4e4258bb4900aaffcbc9d0":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"88ed2cfe9a264830a8717ba4639f1103":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"92a3196be3e44969a1a6f43847a016e7":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_a714adeda5ab4c9cba1dc8bc501a1824","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_a506755f7d1f44dcb8c09989e07375c6","tabbable":null,"tooltip":null,"value":1}},"a506755f7d1f44dcb8c09989e07375c6":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"a57df47e428343f38eca6e01cde35ddf":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_3121bfdea6194f8d9fe3ac81a16f1c24","placeholder":"​","style":"IPY_MODEL_4635236c6b4e4258bb4900aaffcbc9d0","tabbable":null,"tooltip":null,"value":" 3/3 [00:40&lt;00:00, 13.13s/it]"}},"a714adeda5ab4c9cba1dc8bc501a1824":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c61219568c124054b31a7f2cf129a4ba":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_cc302b5a4b4b48138ed3b3ed3f782bd0","max":3,"min":0,"orientation":"horizontal","style":"IPY_MODEL_446959eac3ed494594b6b0265761d861","tabbable":null,"tooltip":null,"value":3}},"c78341b3b0cb4ff9896c5ede030afd40":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cc302b5a4b4b48138ed3b3ed3f782bd0":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d7216e4e518c43e2bdad423b7a23464f":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e8acde88cbff4ee6808e41f460f3774c":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_36367e99d1f6448ebd7eb54a349faa93","placeholder":"​","style":"IPY_MODEL_0039c6cacd80415db6dc2592fc285ef1","tabbable":null,"tooltip":null,"value":"Loading checkpoint shards: 100%"}},"ecaa9d9f3d8e4e1fa9eb38776f345ed5":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nimport transformers\nfrom transformers import AutoTokenizer, AutoModelForSeq2SeqLM\nimport os\nimport re\nimport gc\nimport sys\nimport shutil\nimport subprocess\nimport unicodedata\nimport pkg_resources\nimport pandas as pd\nimport numpy as np\nimport random\nfrom tqdm.auto import tqdm\nfrom fractions import Fraction\nfrom threading import Thread\nfrom joblib import Parallel, delayed\nimport sacrebleu\nimport ctranslate2\n\ngc.collect()\npd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)\npd.set_option('display.max_colwidth', None)\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"max_split_size_mb:128,garbage_collection_threshold:0.6\"\nos.environ['TOKENIZERS_PARALLELISM'] = 'false'\nos.environ['CUDA_VISIBLE_DEVICES'] = '0,1'\n\n# 1) Python version & Device\nprint(\"Python:\", sys.version.replace(\"\\n\", \" \"))\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"CUDA Available:\", torch.cuda.is_available())\nprint(\"CPU Available:\", os.cpu_count())\nprint(\"CTranslate2 支持的计算类型:\", ctranslate2.get_supported_compute_types(\"cuda\"))\nprint(\"CT2 Version:\", ctranslate2.__version__)\n\n# 2) All installed packages\nprint(\"pandas:\", pd.__version__)\nprint(\"numpy:\", np.__version__)\nprint(\"torch:\", torch.__version__)\nprint(\"transformers:\", transformers.__version__)\n# packages = sorted(pkg_resources.working_set, key=lambda x: x.key)\n# for pkg in packages:\n#     print(f\"{pkg.key}=={pkg.version}\")\n\n# 3) Config\nDATA_PATH = '/kaggle/input/deep-past-initiative-machine-translation'\nOUTPUT_PATH = '/kaggle/working'\nCT2_BASE_PATH = os.path.join(OUTPUT_PATH, 'ct2_models')\nos.makedirs(CT2_BASE_PATH, exist_ok=True)\nDEBUG = False\n\n# DATA\nTRAIN_DF_PATH = os.path.join(DATA_PATH, 'train.csv')\nTEST_DF_PATH = os.path.join(DATA_PATH, 'test.csv')\n\n# Inference\nMODEL_SPECS = [\n    {\n        \"name\": \"Model A\",\n        \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0323-data1-all1404/GuoqinGu/byt5-xl-0323-data1-all-checkpoint-1404\",\n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-a-byt5-xl-0323-data1/transformers/all-1404/1/pre_convert_model\"\n    },\n    {    \n        \"name\": \"Model B\", \n        \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0323-data1-llmlabel-all2424/GuoqinGu/byt5-xl-0323-data1_llmlabel-all-checkpoint-2424\",\n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-b-byt5-xl-0323-data1-llmlabel/transformers/all-2424/1/pre_convert_model\"\n    },\n    {\n        \"name\": \"Model C\",\n        \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0321-data2-all1452/GuoqinGu/byt5-xl-0321-data2-all-checkpoint-1452\", \n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-c-byt5-xl-0321-data2/transformers/all-1452/1/pre_convert_model\"\n    },\n    {\n        \"name\": \"Model D\",\n        \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0321-data2-llmlabel-all2472/GuoqinGu/byt5-xl-0321-data2_llmlabel-all-checkpoint-2472\",\n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-d-byt5-xl-0321-data2-llmlabel/transformers/all-2472/1/pre_convert_model\"\n    },\n    {\n        \"name\": \"Model E\",\n        \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0321-data3-llmlabel-all2622/GuoqinGu/byt5-xl-0321-data3_llmlabel-all-checkpoint-2622\",\n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-e-byt5-xl-0321-data3-llmlabel/transformers/all-2622/1/pre_convert_model\"\n    },\n    {    \n        \"name\": \"Model F\", \n        \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0321-data3-all1602/GuoqinGu/byt5-xl-0321-data3-all-checkpoint-1602\",\n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-f-byt5-xl-0321-data3/transformers/all-1602/1/pre_convert_model\"\n    },\n\n    # {\n    #     \"name\": \"Model G\",\n    #     \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0323-data2-llmlabel-synth12-all3159/GuoqinGu/byt5-xl-0323-data2_llmlabel_synth12-all-checkpoint-3159\",\n    #     \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-g-byt5-xl-0323-data2-llmlabel-synth12/transformers/all-3159/1/pre_convert_model\"\n    # },\n    # {    \n    #     \"name\": \"Model H\", \n    #     \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0323-data3-llmlabel-synth12-all3309/GuoqinGu/byt5-xl-0323-data3_llmlabel_synth12-all-checkpoint-3309\",\n    #     \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-h-byt5-xl-0323-data3-llmlabel-synth12/transformers/all-3309/1/pre_convert_model\"\n    # },\n\n    {\n        \"name\": \"Model I\",\n        \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0322-data2-split-checkpoint-1308\",\n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-i-byt5-xl-0322-data2/transformers/split-1308/1/pre_convert_model\"\n    },\n    {    \n        \"name\": \"Model J\", \n        \"ckpt_path\": \"/kaggle/input/datasets/guoqingu/byt5-xl-0322-data3-split-checkpoint-1443\",\n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-j-byt5-xl-0322-data3/transformers/split-1443/1/pre_convert_model\"\n    },\n\n    {    \n        \"name\": \"Model K\", \n        \"ckpt_path\": \"/kaggle/input/models/hongori/byt-5-weighted-bs48-epoch4/transformers/default/1/checkpoint-2512\",\n        \"ct2_path\": \"/kaggle/input/models/guoqingu/ct2-model-k-byt5-xl-weighted/transformers/bs48-ep4/1/pre_convert_model\"\n    },\n]\n\nMAX_LENGTH = 1024\nQUANTIZATION = \"int8_float32\"\n# ===== 单模型推理参数 =====\nBATCH_SIZE = 10\nNUM_BEAMS = 8\nNUM_BEAM_CANDS = 4\nLENGTH_PENALTY = 1.05\nEARLY_STOPPING = True\nREPETITION_PENALTY = 1.0\n\n# ===== 多温度 sampling =====\nSAMPLE_TEMPERATURES = [0.60, 0.80, 1.05]\nNUM_SAMPLE_PER_TEMP = 2\nMBR_TOP_P = 0.92\n\nMAX_NEW_TOKENS = 512\nMBR_POOL_CAP = 200\n\n# ===== chrF++混合 BLEU、Jaccard 和长度奖励 =====\nMBR_W_CHRF = 0.55\nMBR_W_BLEU = 0.25\nMBR_W_JACCARD = 0.20\nMBR_W_LENGTH = 0.10\n\nFALLBACK_TEXT = \"The tablet is too damaged to translate.\"\n","metadata":{"papermill":{"duration":14.350414,"end_time":"2026-03-19T11:42:56.502035","exception":false,"start_time":"2026-03-19T11:42:42.151621","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 模型在推理阶段按统一列表顺序加载。\n# 这里不预先常驻加载模型，避免后续扩展到更多模型时主流程分散。\nmodel = None\ntokenizer = None\n","metadata":{"papermill":{"duration":75.244414,"end_time":"2026-03-19T11:44:11.748847","exception":false,"start_time":"2026-03-19T11:42:56.504433","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T10:53:42.349650Z","iopub.execute_input":"2026-03-23T10:53:42.350138Z","iopub.status.idle":"2026-03-23T10:53:42.353698Z","shell.execute_reply.started":"2026-03-23T10:53:42.350111Z","shell.execute_reply":"2026-03-23T10:53:42.353008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_fractions_dynamic(text_to_process):\n    if not isinstance(text_to_process, str):\n        return str(text_to_process)\n    # Step0: 预处理\n    # --- 0.1 清理数值边界的特殊标记符号 ---\n    # (123) [123] ˹123˺ ⌈123⌋ 「123」 → 123\n    text_to_process = re.sub(r'\\((\\d+)\\)', r'\\1', text_to_process)\n    text_to_process = re.sub(r'\\[(\\d+)\\]', r'\\1', text_to_process)\n    text_to_process = re.sub(r'˹(\\d+)˺', r'\\1', text_to_process)\n    text_to_process = re.sub(r'⌈(\\d+)⌋', r'\\1', text_to_process)\n    text_to_process = re.sub(r'「(\\d+)」', r'\\1', text_to_process)\n\n    # --- 0.2 转换 X+X 格式的数值：循环处理直到没有更多加法运算需要处理 ---\n    def calculate_addition(match):\n        try:\n            val1 = float(match.group(1))\n            val2 = float(match.group(2))\n            result = val1 + val2\n            # 格式化逻辑: 如果是整数则转int str，否则保留有效小数\n            return \"{:.5f}\".format(result).rstrip('0').rstrip('.') if result % 1 != 0 else str(int(result))\n        except ValueError:\n            return match.group(0)\n    addition_pattern = r'(\\d+(?:\\.\\d+)?)\\s*\\+\\s*(\\d+(?:\\.\\d+)?)'\n    while True:\n        new_text = re.sub(addition_pattern, calculate_addition, text_to_process)\n        if new_text == text_to_process:\n            break\n        text_to_process = new_text\n\n    # --- 0.3 月份替换 ---\n    # Month I - month XII -> month 1 - month 12\n    roman_to_arabic = {\n        \"I\": \"1\", \"II\": \"2\", \"III\": \"3\", \"IV\": \"4\", \"V\": \"5\", \"VI\": \"6\",\n        \"VII\": \"7\", \"VIII\": \"8\", \"IX\": \"9\", \"X\": \"10\", \"XI\": \"11\", \"XII\": \"12\"\n    }\n    roman_pattern = r'\\b(Month|month)\\s+(VIII|XII|VII|III|XI|IX|VI|IV|II|X|V|I)\\b'\n    def replace_roman(match):\n        return f\"{match.group(1)} {roman_to_arabic[match.group(2)]}\"\n    text_to_process = re.sub(roman_pattern, replace_roman, text_to_process)\n\n    # Step1: 转换 X / X 格式的分数\n    # --- 1.1 分数转换：规则内的分数转换到 Unicode 分数符号 ---\n    fraction_map = {\n        r\"1\\s*/\\s*6\": \"⅙\",\n        r\"1\\s*/\\s*4\": \"¼\",\n        r\"1\\s*/\\s*3\": \"⅓\",\n        r\"1\\s*/\\s*2\": \"½\",\n        r\"2\\s*/\\s*3\": \"⅔\",\n        r\"3\\s*/\\s*4\": \"¾\",\n        r\"5\\s*/\\s*6\": \"⅚\",\n        # r\"1\\s*/\\s*8\": \"⅛\", r\"3\\s*/\\s*8\": \"⅜\", r\"5\\s*/\\s*8\": \"⅝\", r\"7\\s*/\\s*8\": \"⅞\",\n    }\n    for pattern, char in fraction_map.items():\n        text_to_process = re.sub(pattern, char, text_to_process)\n\n    # --- 1.2 通用数值分数转换：剩余的分数形式转换为小数格式 ---\n    def calc_fraction(match):\n        groups = match.groups()\n        if groups[0]: # 如果匹配到了整数部分 (例如 \"5 11/12\")\n            integer_part = int(groups[0])\n            num = int(groups[1])\n            den = int(groups[2])\n            val = integer_part + (num / den)\n        else: # 只匹配到了分数部分 (例如 \"11/12\")\n            num = int(groups[3])\n            den = int(groups[4])\n            val = num / den\n        # 格式化为5位小数 (例如 5.916666... -> 5.91667)\n        return \"{:.5f}\".format(val).rstrip('0').rstrip('.') if val % 1 != 0 else str(int(val))\n    # 模式1 (混合分数): (\\d+)\\s+(\\d+)/(\\d+) -> 匹配 \"5 11/12\"  |  模式2 (纯分数): (\\d+)/(\\d+) -> 匹配 \"11/12\"\n    fraction_pattern = r'(\\d+)\\s+(\\d+)/(\\d+)|(\\d+)/(\\d+)'   \n    text_to_process = re.sub(fraction_pattern, calc_fraction, text_to_process)\n    \n    # Step2: 转换小数格式的分数到 Unicode 分数符号\n    # --- 2.1 处理长浮点数损坏数值 ---\n    # 正则逻辑：匹配 数字 + 小数点 + 至少5位数字\n    # \\d+\\.\\d{5,} 匹配如 1.3333300000000001\n    # lambda 表达式将其截断至小数点后 5 位\n    text_to_process = re.sub(r'(\\d+\\.\\d{5})\\d+', r'\\1', text_to_process)\n\n    # --- 2.2 统一成 UNICODE 分数符号 ---\n    fraction_lookup = {\n            (1, 6): \"⅙\", # .166\n            (1, 4): \"¼\", # .25\n            (1, 3): \"⅓\", # .333\n            (1, 2): \"½\", # .5\n            (2, 3): \"⅔\", # .666\n            (3, 4): \"¾\", # .75\n            (5, 6): \"⅚\", # .833\n        #    (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n        #    (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\"\n    }\n    def replacer(match):\n        full_str = match.group(0)\n        try:\n            val = float(full_str)\n            integer_part = int(val)\n            decimal_part = val - integer_part\n            # Return integer if decimal is negligible\n            if decimal_part < 0.0001: \n                return str(integer_part)\n            # Limit denominator handles 0.3333 -> 1/3\n            frac = Fraction(decimal_part).limit_denominator(12) \n\n            unicode_frac = fraction_lookup.get((frac.numerator, frac.denominator))\n            if unicode_frac:\n                if integer_part == 0:\n                    return unicode_frac\n                else:\n                    # Current logic: Space between Integer and Fraction\n                    return f\"{integer_part} {unicode_frac}\"\n\n            return full_str \n        except ValueError:\n            return full_str\n    processed_text = re.sub(r'\\b\\d+\\.\\d+\\b', replacer, text_to_process)\n    \n    # Step3: Robustness check\n    # --- 3.1 保证带分数中间有空格 ---\n    processed_text = re.sub(r'(\\d)([¼½¾⅓⅔⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞])', r'\\1 \\2', processed_text)\n    \n    return processed_text\n\ndef preprocess_transliteration(text):\n    if pd.isna(text):\n        return \"\"\n    text = str(text)\n    \n    # --- 1. 字符替换与括号处理 ---\n    char_map = str.maketrans({\n        \"X\": \"x\",\n        \"ā\": \"a\",\n        \"Ş\": \"Ṣ\",\n        \"ş\": \"ṣ\",\n        \"Ș\": \"Ṣ\",\n        \"ș\": \"ṣ\",\n        \"Ț\": \"Ṭ\",\n        \"ț\": \"ṭ\",\n        \"İ\": \"I\",\n        \"Ī\": \"I\",\n        \"ī\": \"i\",\n        \"Î\": \"I\",\n        \"î\": \"i\",\n        \"ı\": \"i\",\n        \"h\": \"ḫ\",\n        \"H\": \"Ḫ\",\n        \"ḥ\": \"ḫ\",\n        \"Ḥ\": \"Ḫ\",\n        \"=\": \"-\",\n        \"(\": \"{\",  # for deterministic\n        \")\": \"}\",\n        \"—\": \"-\",\n        \"–\": \"-\",\n    })\n    text = text.translate(char_map)\n    text = text.replace(\"ᵈ\", r\"{d}\").replace(\"ᵏⁱ\", r\"{ki}\")   # 手动操作\n    text = text.replace(r\"{{\", \"{\").replace(r\"}}\", \"}\")\n    text = text.replace(\"lá+lá\", \"lá-lá\").replace(\"la+la\", \"la-la\")\n    text = text.replace(\"{?}\", \"\").replace(\"{!}\", \"\").replace(\"-/\", \"-\")\n    # Remove bad characters\n    upper_chars = [\"⁰\",\"¹\",\"²\",\"³\",\"⁴\",\"⁵\",\"⁶\",\"⁷\",\"⁸\",\"⁹\",\"⁻\",\"°\", \"˚\"]\n    for ch in upper_chars:\n        text = text.replace(ch, \"\")\n    bad_chars = \"\"\"\"ʺʾ’´ʼˊ˘ˋˇˈʿʹ'`^˹˺⸢⸣⌜⌝「」⌈⌋⌊⌉˻˼⸤⸥≪≫《》«»⟨⟩˃‹›⁽⁾ˁˀ\"\"\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n    bad_chars = \"\"\"!?⸮ʔ⸮,/;˷˴|:+⁺*\"\"\"\n    for ch in bad_chars:\n        text = text.replace(ch, \" \")\n    \n    # --- 2. 角标统一处理 ---\n    # 4 a-bi4-a 0.5 2+10 ù a-šùr-DU10: 4 a-bi₄-a 0.5 2+10 ù a-šùr-DU₁₀\n    # 定义允许的字母范围（包含普通字母和特殊字符）\n    letters_str = \"a-zA-ZšŠṣṢṭṬḫḪ\"\n    # 定义数字到角标的映射表\n    subscript_trans = str.maketrans(\"0123456789\", \"₀₁₂₃₄₅₆₇₈₉\")\n    # 正则逻辑：匹配紧跟在字母后面的数字\n    def replace_group_with_subscript(match):\n        digits = match.group(0)\n        return digits.translate(subscript_trans)\n    text = re.sub(f\"(?<=[{letters_str}])\\d+\", replace_group_with_subscript, text)\n\n    # --- 3. Gap替换 ---\n    # ['PN'/'Pn'] -> '<gap>'\n    pattern = r'\\bPN\\b|\\bPn\\b'\n    text = re.sub(pattern, '<gap>', text)\n    # break/broken -> ''\n    text = re.sub(r'\\bbreak\\b|\\bbroken\\b', '', text)\n    # [] -> [xxx]\n    text = re.sub(r'\\[[\\sx?]*\\]', '[xxx]', text)\n    # Multi-dot patterns → <gap>\n    text = re.sub(r'(?:\\.\\s+){2,}\\.', lambda m: m.group(0).replace(' ', ''), text)\n    text = re.sub(r'\\.{3,}(\\s+\\.{3,})*', '<gap>', text)\n    text = re.sub(r'……|…', '<gap>', text)\n    # x patterns → <gap>\n    text = re.sub(r'xx+', '<gap>', text)\n    # 处理孤立的 x (前后是空格或边界)\n    text = re.sub(r'\\bx\\b', '<gap>', text)\n    # 直接替换 {large break}\n    text = text.replace(\"{large break}\", \"<gap>\")\n    # 正则替换 {N broken lines} (例如 {3 broken lines}, {4 broken lines})\n    text = re.sub(r'\\{\\d+\\s+broken\\s+lines\\}', '<gap>', text)\n    # Temporarily protect tokens from cleanup\n    text = text.replace(\"<gap>\", \"\\x00GAP\\x00\")\n    # Remove bad characters\n    bad_chars = \"<>[]\"\n    for ch in bad_chars:\n        text = text.replace(ch, \"\")\n    # Restore tokens\n    text = text.replace(\"\\x00GAP\\x00\", \" <gap> \")\n    text = text.replace(\"{ <gap> }\", \"<gap>\")\n    text = text.replace(\"[ <gap> ]\", \"<gap>\")\n    \n    # Normalize whitespace and trim\n    text = re.sub(r\"\\s+\", \" \", text).strip().strip(\"-\")\n    text = text.replace(\" -\", \"-\").replace(\"- \", \"-\")\n    text = text.replace(\"> .\", \">.\").replace(\"> :\", \">:\").replace(\"> ’\", \">’\")\n\n    # merge <gap>\n    pattern = r'<gap>([ \\-\\t]*<gap>)+'\n    text = re.sub(pattern, '<gap>', text)\n    \n    return text","metadata":{"papermill":{"duration":0.026875,"end_time":"2026-03-19T11:44:11.778007","exception":false,"start_time":"2026-03-19T11:44:11.751132","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T10:53:42.354826Z","iopub.execute_input":"2026-03-23T10:53:42.355128Z","iopub.status.idle":"2026-03-23T10:53:42.377170Z","shell.execute_reply.started":"2026-03-23T10:53:42.355094Z","shell.execute_reply":"2026-03-23T10:53:42.376405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def postprocess_test_translation(text):\n    # Return empty string for invalid outputs\n    if not isinstance(text, str) or not text.strip():\n        return \"\"\n\n    # Normalize specific diacritics\n    processed_text = text.replace(\"ḫ\", \"h\").replace(\"Ḫ\", \"H\")\n\n    processed_text = unicodedata.normalize(\"NFKC\", text)\n    processed_text = text.lower()\n\n    # Vowels\n    processed_text = re.sub(r'[āâ]', 'a', text)\n    processed_text = re.sub(r'[ēê]', 'e', text)\n    processed_text = re.sub(r'[īî]', 'i', text)\n    processed_text = re.sub(r'[ōô]', 'o', text)\n    processed_text = re.sub(r'[ūû]', 'u', text)\n\n    # Consonants\n    processed_text = re.sub(r'[ṣ]', 's', text)\n    processed_text = re.sub(r'[ṭ]', 't', text)\n    processed_text = re.sub(r'[ḍ]', 'd', text)\n    processed_text = re.sub(r'[ḫ]', 'h', text)\n    processed_text = re.sub(r'[š]', 'sh', text)\n\n    # Determinatives / glosses\n    processed_text = re.sub(r'\\{[^}]+\\}', '', text)\n    processed_text = re.sub(r'\\([^)]*\\)', '', text)\n\n    text = re.sub(r'\\s+', ' ', text).strip()\n\n    # Convert subscript digits to normal digits\n    sub_map = str.maketrans(\"₀₁₂₃₄₅₆₇₈₉\", \"0123456789\")\n    processed_text = processed_text.translate(sub_map)\n\n    # Normalize gap markers\n    processed_text = re.sub(r\"(\\[x\\]|\\(x\\)|\\bx\\b)\", \"<gap>\", processed_text, flags=re.I)\n    processed_text = re.sub(r\"(\\.{3,}|…|\\[\\.+\\])\", \"<gap>\", processed_text)\n\n    # Merge adjacent gaps\n    processed_text = re.sub(r\"<gap>\\s*<gap>\", \" <gap> \", processed_text)\n    processed_text = re.sub(r\"<big_gap>\\s*<big_gap>\", \" <gap> \", processed_text)\n\n    # Remove some parenthetical morphology notes\n    processed_text = re.sub(\n        r\"\\((fem|plur|pl|sing|singular|plural|\\?|!)\\.?\\s*\\w*\\)\",\n        \"\",\n        processed_text,\n        flags=re.I,\n    )\n\n    # Temporarily protect tokens from cleanup\n    processed_text = processed_text.replace(\"<gap>\", \"\\x00GAP\\x00\")\n\n    # Remove bad characters\n    bad_chars = '!?()\"—–<>⌈⌋⌊[]+ʾ/;'\n    processed_text = processed_text.translate(str.maketrans(\"\", \"\", bad_chars))\n\n    # Restore tokens\n    processed_text = processed_text.replace(\"\\x00GAP\\x00\", \" <gap> \")\n\n    # Handle common fractions from decimals\n    frac_map = {\n        r\"\\.5\\b\": \" ½\",\n        r\"\\.25\\b\": \" ¼\",\n        r\"\\.75\\b\": \" ¾\",\n        r\"\\.33+\\d*\\b\": \" ⅓\",\n        r\"\\.66+\\d*\\b\": \" ⅔\",\n    }\n    for pattern, replacement in frac_map.items():\n        processed_text = re.sub(r\"(\\d+)\" + pattern, r\"\\1\" + replacement, processed_text)\n        processed_text = re.sub(r\"\\b0\" + pattern, replacement.strip(), processed_text)\n\n    # Remove repeated single words\n    processed_text = re.sub(r\"\\b(\\w+)(?:\\s+\\1\\b)+\", r\"\\1\", processed_text)\n\n    # Remove repeated n-grams\n    def remove_repeated_ngrams(text, max_n=7):\n        # 我们将 \\s+ 替换为了 \\W+ (匹配任何非字母数字的字符，如空格和标点)\n        # 这样就可以跨越 \". \" 这样的标点+空格组合来寻找重复了\n        for n in range(max_n, 1, -1):\n            # 匹配逻辑：(单词 + 标点/空格)重复n-1次 + 单词\n            pattern = r\"\\b((?:\\w+\\W+){\" + str(n - 1) + r\"}\\w+)(?:\\W+\\1\\b){2,}\"\n            \n            # 为了处理大小写可能不一致的情况，可以加上 re.IGNORECASE (可选)\n            text = re.sub(pattern, r\"\\1\", text)  # , flags=re.IGNORECASE\n            \n        return text\n    processed_text = remove_repeated_ngrams(processed_text, max_n=9)\n\n    # Normalize whitespace and trim\n    processed_text = re.sub(r\"\\s+\", \" \", processed_text).strip().strip(\"-\")\n\n    return processed_text","metadata":{"papermill":{"duration":0.013195,"end_time":"2026-03-19T11:44:11.793375","exception":false,"start_time":"2026-03-19T11:44:11.78018","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T10:53:42.378164Z","iopub.execute_input":"2026-03-23T10:53:42.378448Z","iopub.status.idle":"2026-03-23T10:53:42.389903Z","shell.execute_reply.started":"2026-03-23T10:53:42.378426Z","shell.execute_reply":"2026-03-23T10:53:42.389116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_token_limits(transliteration_len):\n    # 根据你的拟合结果填入系数\n    m, c = 1.1427, -1.1173\n    up, down = 200, 200\n    \n    max_tokens = int(m * transliteration_len + (c + up))\n    min_tokens = max(0, int(m * transliteration_len + (c - down)))\n    \n    return min_tokens, max_tokens\n\nclass AkkadDataset(Dataset):\n    def __init__(self, df):\n        self.ids = df['id'].tolist()\n        self.texts = [\"Translate Akkadian to English: \" + str(t) for t in df['transliteration']]\n        self.min_tokens = df['min_t'].tolist()\n        self.max_tokens = df['max_t'].tolist()\n\n    def __len__(self): return len(self.ids)\n\n    def __getitem__(self, i):\n        return {\n            \"id\": self.ids[i],\n            \"text\": self.texts[i],\n            \"min_t\": self.min_tokens[i],\n            \"max_t\": self.max_tokens[i]\n        }\n\ndef collate_fn(batch):\n    ids = [x[\"id\"] for x in batch]\n    texts = [x[\"text\"] for x in batch]\n    batch_min = int(np.min([x[\"min_t\"] for x in batch]))\n    batch_max = int(np.max([x[\"max_t\"] for x in batch]))\n    \n    inputs = tokenizer(texts, max_length=MAX_LENGTH, padding=True, truncation=True, return_tensors=\"pt\")\n    \n    return ids, inputs, batch_min, batch_max","metadata":{"papermill":{"duration":0.011027,"end_time":"2026-03-19T11:44:11.806666","exception":false,"start_time":"2026-03-19T11:44:11.795639","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T10:53:42.391480Z","iopub.execute_input":"2026-03-23T10:53:42.391888Z","iopub.status.idle":"2026-03-23T10:53:42.403710Z","shell.execute_reply.started":"2026-03-23T10:53:42.391855Z","shell.execute_reply":"2026-03-23T10:53:42.403029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 版本1：双模型加权 MBR\n# 保持现有候选生成方式和加权 MBR\n\nclass MBRSelector:\n    # pairwise score = chrF++ + BLEU + Jaccard 的加权组合\n    # final score    = pairwise score + length bonus\n    def __init__(\n        self,\n        pool_cap=32,\n        w_chrf=0.55,\n        w_bleu=0.25,\n        w_jaccard=0.20,\n        w_length=0.10,\n    ):\n        self._chrf_metric = sacrebleu.metrics.CHRF(word_order=2)\n        self._bleu_metric = sacrebleu.metrics.BLEU(effective_order=True)\n        self.pool_cap = pool_cap\n        self.w_chrf = w_chrf\n        self.w_bleu = w_bleu\n        self.w_jaccard = w_jaccard\n        self.w_length = w_length\n        self._pw_total = max(w_chrf + w_bleu + w_jaccard, 1e-9)\n\n    def _chrfpp(self, a, b):\n        if not a or not b:\n            return 0.0\n        return float(self._chrf_metric.sentence_score(a, [b]).score)\n\n    def _bleu(self, a, b):\n        if not a or not b:\n            return 0.0\n        try:\n            return float(self._bleu_metric.sentence_score(a, [b]).score)\n        except Exception:\n            return 0.0\n\n    @staticmethod\n    def _jaccard(a, b):\n        ta, tb = set(a.lower().split()), set(b.lower().split())\n        if not ta and not tb:\n            return 100.0\n        if not ta or not tb:\n            return 0.0\n        return 100.0 * len(ta & tb) / len(ta | tb)\n\n    def _pairwise_score(self, a, b):\n        score = (\n            self.w_chrf * self._chrfpp(a, b)\n            + self.w_bleu * self._bleu(a, b)\n            + self.w_jaccard * self._jaccard(a, b)\n        )\n        return score / self._pw_total\n\n    @staticmethod\n    def _length_bonus(lengths, idx):\n        if len(lengths) == 0:\n            return 100.0\n        median = float(np.median(lengths))\n        sigma = max(median * 0.4, 5.0)\n        z = (lengths[idx] - median) / sigma\n        return 100.0 * float(np.exp(-0.5 * z * z))\n\n    @staticmethod\n    def _dedup(xs):\n        seen = set()\n        out = []\n        for x in xs:\n            x = str(x).strip()\n            if x and x not in seen:\n                out.append(x)\n                seen.add(x)\n        return out\n\n    def pick(self, candidates):\n        cands = self._dedup(candidates)\n        if self.pool_cap:\n            cands = cands[:self.pool_cap]\n\n        n = len(cands)\n        if n == 0:\n            return \"\"\n        if n == 1:\n            return cands[0]\n\n        lengths = [len(c.split()) for c in cands]\n        scores = []\n\n        for i in range(n):\n            pairwise = sum(\n                self._pairwise_score(cands[i], cands[j])\n                for j in range(n)\n                if j != i\n            ) / max(1, n - 1)\n\n            length_bonus = self._length_bonus(lengths, i)\n            total = pairwise + self.w_length * length_bonus\n            scores.append(total)\n\n        return cands[int(np.argmax(scores))]\n","metadata":{"papermill":{"duration":0.01563,"end_time":"2026-03-19T11:44:11.824565","exception":false,"start_time":"2026-03-19T11:44:11.808935","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T10:53:42.404678Z","iopub.execute_input":"2026-03-23T10:53:42.404958Z","iopub.status.idle":"2026-03-23T10:53:42.417350Z","shell.execute_reply.started":"2026-03-23T10:53:42.404926Z","shell.execute_reply":"2026-03-23T10:53:42.416681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def ensure_ct2_model(ckpt_path, ct2_path):\n    \"\"\"确保模型已经被编译为 CT2 格式，如果没有则进行转换\"\"\"\n    if not os.path.exists(ct2_path):\n        print(f\"\\n⚙️ 正在转换模型到 CTranslate2 ({QUANTIZATION})...\\n  源: {ckpt_path}\\n  目标: {ct2_path}\")\n        result = subprocess.run(\n            [\n                \"ct2-transformers-converter\",\n                \"--model\", ckpt_path,\n                \"--output_dir\", ct2_path,\n                \"--quantization\", QUANTIZATION,\n                \"--force\",\n            ],\n            capture_output=True, text=True\n        )\n        if result.returncode != 0:\n            print(\"STDERR:\", result.stderr)\n            raise RuntimeError(\"模型转换失败，请检查上方输出\")\n        print(f\"✅ {ct2_path} 转换完成\")\n    else:\n        print(f\"✅ {ct2_path} 已存在，跳过转换\")\n\ndef load_ct2_model_on_device(model_spec, device_index):\n    \"\"\"在指定 GPU 索引上加载 CTranslate2 Translator\"\"\"\n    return {\n        \"name\": model_spec[\"name\"],\n        \"translator\": ctranslate2.Translator(\n            model_spec[\"ct2_path\"],\n            device=\"cuda\",\n            device_index=[device_index], # 分配到具体的 GPU 卡\n            compute_type=QUANTIZATION,\n            inter_threads=1,\n            flash_attention=False\n        ),\n        \"tokenizer\": AutoTokenizer.from_pretrained(model_spec[\"ckpt_path\"]),\n        \"device_index\": device_index\n    }\n\ndef release_model_bundle(model_bundle):\n    if model_bundle is None: return\n    if \"translator\" in model_bundle:\n        # 卸载 CTranslate2 模型以释放显存\n        del model_bundle[\"translator\"]\n    if \"tokenizer\" in model_bundle: \n        del model_bundle[\"tokenizer\"]\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T10:53:42.418223Z","iopub.execute_input":"2026-03-23T10:53:42.418528Z","iopub.status.idle":"2026-03-23T10:53:42.428075Z","shell.execute_reply.started":"2026-03-23T10:53:42.418500Z","shell.execute_reply":"2026-03-23T10:53:42.427406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_ct2_collate_fn(current_tokenizer):\n    def _collate(batch):\n        ids = [x[\"id\"] for x in batch]\n        texts = [x[\"text\"] for x in batch]\n        batch_min = int(np.min([x[\"min_t\"] for x in batch]))\n        batch_max = int(np.max([x[\"max_t\"] for x in batch]))\n        \n        # 对于 CT2，我们需要传入 Token String 的列表，而不是 Tensor ID\n        inputs = current_tokenizer(texts, truncation=True, max_length=MAX_LENGTH)\n        source_tokens = [current_tokenizer.convert_ids_to_tokens(seq) for seq in inputs.input_ids]\n        \n        return ids, source_tokens, batch_min, batch_max\n    return _collate\n\ndef _decode_ct2_tokens(tokens, tokenizer):\n    return tokenizer.decode(\n        tokenizer.convert_tokens_to_ids(tokens),\n        skip_special_tokens=True,\n    )\n\ndef generate_candidate_pools_ct2(source_tokens, translator, tokenizer):\n    batch_size = len(source_tokens)\n\n    # ---- Beam Search 候选 ----\n    beam_results = translator.translate_batch(\n        source_tokens,\n        beam_size=NUM_BEAMS,\n        num_hypotheses=NUM_BEAM_CANDS,\n        length_penalty=LENGTH_PENALTY,\n        repetition_penalty=REPETITION_PENALTY,\n        max_decoding_length=MAX_NEW_TOKENS,\n        max_input_length=MAX_LENGTH,\n    )\n    beam_texts = [\n        [_decode_ct2_tokens(hyp, tokenizer) for hyp in r.hypotheses]\n        for r in beam_results\n    ]\n\n    # ---- 多温度 Nucleus Sampling 候选 ----\n    all_sample_texts = [[] for _ in range(batch_size)]\n    for temperature in SAMPLE_TEMPERATURES:\n        for _ in range(NUM_SAMPLE_PER_TEMP):\n            sample_results = translator.translate_batch(\n                source_tokens,\n                beam_size=1,\n                num_hypotheses=1,\n                sampling_topk=0,            \n                sampling_topp=MBR_TOP_P,    \n                sampling_temperature=temperature,\n                repetition_penalty=REPETITION_PENALTY,\n                max_decoding_length=MAX_NEW_TOKENS,\n                max_input_length=MAX_LENGTH,\n            )\n            for i, result in enumerate(sample_results):\n                all_sample_texts[i].append(_decode_ct2_tokens(result.hypotheses[0], tokenizer))\n\n    # ---- 合并候选池 ----\n    pools = [beam_texts[i] + all_sample_texts[i] for i in range(batch_size)]\n    return pools\n\ndef run_ct2_inference_worker(loader, model_bundle, results_dict):\n    device_index = model_bundle[\"device_index\"]\n    translator = model_bundle[\"translator\"]\n    tokenizer = model_bundle[\"tokenizer\"]\n    pools_by_id = {}\n    \n    # CT2 是在 C++ 层执行推理，不需要 torch.inference_mode()\n    for ids, source_tokens, min_t, max_t in tqdm(loader, desc=f\"{model_bundle['name']} on cuda:{device_index}\"):\n        batch_pools = generate_candidate_pools_ct2(source_tokens, translator, tokenizer)\n        for sample_id, pool in zip(ids, batch_pools):\n            pools_by_id[sample_id] = pool\n            \n    results_dict[f\"cuda:{device_index}\"] = pools_by_id","metadata":{"papermill":{"duration":0.010539,"end_time":"2026-03-19T11:44:11.837264","exception":false,"start_time":"2026-03-19T11:44:11.826725","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T10:53:42.428827Z","iopub.execute_input":"2026-03-23T10:53:42.429097Z","iopub.status.idle":"2026-03-23T10:53:42.442262Z","shell.execute_reply.started":"2026-03-23T10:53:42.429062Z","shell.execute_reply":"2026-03-23T10:53:42.441619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================= 数据准备阶段 =================\nif DEBUG:\n    test_df = pd.read_csv(TRAIN_DF_PATH)\n    test_df = test_df.rename(columns={'oare_id': 'id', 'translation': 'translation_groundtruth'})\nelse:\n    test_df = pd.read_csv(TEST_DF_PATH)\n\ntest_df[\"transliteration\"] = test_df[\"transliteration\"].apply(process_fractions_dynamic)\ntest_df[\"transliteration\"] = test_df[\"transliteration\"].apply(preprocess_transliteration)\ntest_df['transliteration_char_length'] = test_df.apply(lambda row: len(row['transliteration']), axis=1)\ntest_df = test_df.sort_values(by='transliteration_char_length', ascending=False).reset_index(drop=True)\ntest_df[['min_t', 'max_t']] = test_df.apply(\n    lambda row: get_token_limits(row['transliteration_char_length']),\n    axis=1, result_type='expand'\n)\nsample_ids = test_df['id'].tolist()\n\n# 奇偶交叉 (Interleave)，平衡双卡的 Padding 负载\ndf_0 = test_df.iloc[0::2].reset_index(drop=True)\ndf_1 = test_df.iloc[1::2].reset_index(drop=True)\n\ndataset_0 = AkkadDataset(df_0)\ndataset_1 = AkkadDataset(df_1)\n\nmbr_selector = MBRSelector(\n    pool_cap=MBR_POOL_CAP,\n    w_chrf=MBR_W_CHRF,\n    w_bleu=MBR_W_BLEU,\n    w_jaccard=MBR_W_JACCARD,\n    w_length=MBR_W_LENGTH,\n)\n\nall_model_pools = {spec[\"name\"]: {} for spec in MODEL_SPECS}\n\n# ================= 模型推理阶段 =================\nfor model_spec in MODEL_SPECS:\n    print(f\"\\n{'='*50}\\n处理 {model_spec['name']}\")\n    \n    # 1. 确保模型已被转换为 CT2 格式\n    ensure_ct2_model(model_spec[\"ckpt_path\"], model_spec[\"ct2_path\"])\n    \n    # 2. 在 0卡 和 1卡 分别加载 CT2 Translator\n    bundle_0 = load_ct2_model_on_device(model_spec, 0) # device_index = 0\n    bundle_1 = load_ct2_model_on_device(model_spec, 1) # device_index = 1\n\n    # 3. 构建返回 token 列表的 DataLoader\n    loader_0 = DataLoader(dataset_0, batch_size=BATCH_SIZE, shuffle=False, collate_fn=build_ct2_collate_fn(bundle_0[\"tokenizer\"]))\n    loader_1 = DataLoader(dataset_1, batch_size=BATCH_SIZE, shuffle=False, collate_fn=build_ct2_collate_fn(bundle_1[\"tokenizer\"]))\n\n    results_dict = {}\n\n    # 4. 开启双线程并行推理\n    t0 = Thread(target=run_ct2_inference_worker, args=(loader_0, bundle_0, results_dict))\n    t1 = Thread(target=run_ct2_inference_worker, args=(loader_1, bundle_1, results_dict))\n\n    t0.start()\n    t1.start()\n    t0.join()\n    t1.join()\n\n    # 5. 聚合双卡结果\n    all_model_pools[model_spec[\"name\"]].update(results_dict.get(\"cuda:0\", {}))\n    all_model_pools[model_spec[\"name\"]].update(results_dict.get(\"cuda:1\", {}))\n\n    # 6. 清理内存 / 显存\n    release_model_bundle(bundle_0)\n    release_model_bundle(bundle_1)\n    \n    # 【可选】如果你发现 kaggle/working 空间 (20GB) 不够用，可以开启以下代码，推理完一个就删掉一个转换后的文件\n    # shutil.rmtree(model_spec[\"ct2_path\"], ignore_errors=True)\n    # print(f\"🧹 已清理 {model_spec['ct2_path']} 以释放磁盘空间\")\n\n# ================= MBR 聚合与提交阶段 =================\nprint(\"\\nStarting MBR selection...\")\n# 单条样本的处理逻辑\ndef process_single_sample(sample_id):\n    raw_pool = []\n    # 从外层作用域读取 all_model_pools 和 MODEL_SPECS\n    for model_spec in MODEL_SPECS:\n        raw_pool.extend(all_model_pools[model_spec[\"name\"]].get(sample_id, []))\n\n    all_cleaned = []        # 保存所有经过基础清理的有效句子 (用于兜底)\n    filtered_cleaned = []   # 保存经过长度过滤后的优质句子\n\n    for raw in raw_pool:\n        cleaned = postprocess_test_translation(raw)\n        if not cleaned:\n            continue\n            \n        all_cleaned.append(cleaned)\n\n        raw_word_count = len(raw.split())\n        cleaned_word_count = len(cleaned.split())\n\n        # 评估缩水情况\n        # 条件A: 清理后的长度小于原来的 50%\n        is_length_halved = cleaned_word_count < 0.5 * raw_word_count\n        # 条件B: 长度变化幅度大于 10 个单词\n        is_drop_large = (raw_word_count - cleaned_word_count) > 10\n\n        # 如果同时满足 A 和 B，说明可能被过度截断，我们将其剔除；否则加入过滤后的池子\n        if is_length_halved and is_drop_large:\n            # 被判定为过度清理，剔除 (这里什么都不做，即不加入 filtered_cleaned)\n            pass\n        else:\n            filtered_cleaned.append(cleaned)\n\n    # 兜底策略 (Fallback)\n    # 如果过滤后剩下的句子数量 <= 20 个，为了保证 MBR 有足够的选择空间，我们放弃这次严格的长度过滤，直接使用所有非空的 cleaned 句子。\n    final_pool = filtered_cleaned if len(filtered_cleaned) > 20 else all_cleaned\n\n    # 调用全局的 mbr_selector\n    chosen = mbr_selector.pick(final_pool)\n\n    # 终极兜底 (如果 MBR 依然什么都没选出来)\n    if not chosen:\n        fallback = postprocess_test_translation(raw_pool[0]) if raw_pool else \"\"\n        chosen = fallback if fallback else FALLBACK_TEXT\n\n    return (sample_id, chosen)\n\n# 使用 joblib 开启 4 核并行计算\nresults = Parallel(n_jobs=4)(\n    delayed(process_single_sample)(sample_id)\n    for sample_id in tqdm(sample_ids, desc=\"MBR Pooling\")\n)\nsubmission = pd.DataFrame(results, columns=['id', 'translation'])\nsubmission = submission.sort_values(by='id', ascending=True).reset_index(drop=True)\nsubmission.to_csv(os.path.join(OUTPUT_PATH, 'submission.csv'), index=False, encoding='utf-8')\n\nprint(submission.shape)\nprint(submission.head())\nprint(\"✅ Saved submission.csv\")","metadata":{"papermill":{"duration":53.058355,"end_time":"2026-03-19T11:45:04.89781","exception":false,"start_time":"2026-03-19T11:44:11.839455","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2026-03-23T10:55:19.887980Z","iopub.execute_input":"2026-03-23T10:55:19.888571Z","execution_failed":"2026-03-23T10:57:15.277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}