{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":121150,"databundleVersionId":15833391,"isSourceIdPinned":false},{"sourceType":"datasetVersion","sourceId":15325130,"datasetId":9801746,"databundleVersionId":16231969},{"sourceType":"datasetVersion","sourceId":15325893,"datasetId":9802266,"databundleVersionId":16232870},{"sourceType":"datasetVersion","sourceId":15319101,"datasetId":9798014,"databundleVersionId":16225046},{"sourceType":"datasetVersion","sourceId":15320399,"datasetId":9798851,"databundleVersionId":16226545},{"sourceType":"datasetVersion","sourceId":14899351,"datasetId":9366631,"databundleVersionId":15763950},{"sourceType":"datasetVersion","sourceId":15350462,"datasetId":9818700,"databundleVersionId":16260160},{"sourceType":"datasetVersion","sourceId":15331103,"datasetId":9805903,"databundleVersionId":16238647},{"sourceType":"modelInstanceVersion","sourceId":755680,"databundleVersionId":15750522,"modelInstanceId":577094,"modelId":589419,"isSourceIdPinned":false}],"dockerImageVersionId":31260,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":309.846981,"end_time":"2026-03-23T08:41:41.259925","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-03-23T08:36:31.412944","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"04633e68b40340a7ba2d0067cb851f83":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"06f29bfb8f6a48809b3e0ad88e456e41":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_926ca45e2ea9465e9bd581a46425acbd","placeholder":"​","style":"IPY_MODEL_5aa8619bebac4f7aa132db2946d2b63d","tabbable":null,"tooltip":null,"value":"MBR selecting: 100%"}},"0752a1dc1db84c659d19e3ca583eb92d":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"089680f2a3f64473a05145b9cb120cfe":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0f7fa209bc904fe899b672d1f9797b78":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"126c77179c5a4a4687c0357ca657c725":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"149e54e5c11249ccb3f0b6d652a0aa6c":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_2076fbd7f3d74e44b547cb9cc4fa3cdf","placeholder":"​","style":"IPY_MODEL_ec9f3861208248bb9fc1d5ee1b349e9b","tabbable":null,"tooltip":null,"value":" 4/4 [00:00&lt;00:00,  6.56it/s]"}},"2076fbd7f3d74e44b547cb9cc4fa3cdf":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"230463a80849410e9ee1dd343b4ef16c":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_06f29bfb8f6a48809b3e0ad88e456e41","IPY_MODEL_b1e60742f38448cc9a929b2a29a3691c","IPY_MODEL_149e54e5c11249ccb3f0b6d652a0aa6c"],"layout":"IPY_MODEL_04633e68b40340a7ba2d0067cb851f83","tabbable":null,"tooltip":null}},"2acc707c62b240d6b972d4be55d9ec36":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2fca79486aab4f54b011d0700672db25":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"319721b43e1c4293a2455ef5527e23bb":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_a2d0c9a4b4074f1984a35069a795a24e","IPY_MODEL_e64ec84471c8460c8dcfc57fc66e7f17","IPY_MODEL_ceadf441072b4e349b4fb2a041900e2b"],"layout":"IPY_MODEL_c3654e33865b431ea1271e86b678bd9f","tabbable":null,"tooltip":null}},"36d77ab3afb74805b0209330f6ce1239":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"3bac291fefeb429f9ace2fbe7af3657c":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_089680f2a3f64473a05145b9cb120cfe","max":4,"min":0,"orientation":"horizontal","style":"IPY_MODEL_929b02223ebe4b899174c5c726695d2a","tabbable":null,"tooltip":null,"value":4}},"3d084a78429249958ea70ba097637825":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_8ca51f90a4054e69937f789937ab712c","placeholder":"​","style":"IPY_MODEL_f29bf639e58c47979a43e34f93a30aed","tabbable":null,"tooltip":null,"value":"Generating (final-byt5): 100%"}},"4f7ea862acf84bcbb40c9632607d6cf9":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"59cef0e7bf804e3bbd5881ec3cba8ab2":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"5aa8619bebac4f7aa132db2946d2b63d":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"5d44e0bc2bce49abbc9a52d93525a446":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"61b8e07f230b48eda41a9a021d102ce7":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3d084a78429249958ea70ba097637825","IPY_MODEL_67fbc4376fc8478d8272bd6818c01197","IPY_MODEL_6c064ac8c75a4ad8a42d9172d875d1a9"],"layout":"IPY_MODEL_c12783d9acaa43da8e93ada44d178e08","tabbable":null,"tooltip":null}},"67fbc4376fc8478d8272bd6818c01197":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_a8c5f5447900451daf10be224356cd09","max":4,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f305c34f144b4d74bfde8bb0fa912727","tabbable":null,"tooltip":null,"value":4}},"6c064ac8c75a4ad8a42d9172d875d1a9":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_126c77179c5a4a4687c0357ca657c725","placeholder":"​","style":"IPY_MODEL_0752a1dc1db84c659d19e3ca583eb92d","tabbable":null,"tooltip":null,"value":" 4/4 [00:34&lt;00:00,  9.53s/it]"}},"7080bae9978d440eb09d8896a324f601":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7cb8001f229945388798c71bdc752ecf":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_9989287548df47c6b5178e984c6956af","placeholder":"​","style":"IPY_MODEL_59cef0e7bf804e3bbd5881ec3cba8ab2","tabbable":null,"tooltip":null,"value":"Generating (byt5-mbr-v2): 100%"}},"7e3ae1c0c40d45bc83e2f67bc08cc124":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8163079771d34d4eb35aa5ca0689c6a5":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"849091a05eff4f35bbba3ef02cc0e769":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7cb8001f229945388798c71bdc752ecf","IPY_MODEL_d3331733b3654ff2869602bac758cd18","IPY_MODEL_dd920405b59a49d48d3ac38a5e0fb049"],"layout":"IPY_MODEL_f9ec3f3fc9744888a3c47742b57ec81e","tabbable":null,"tooltip":null}},"8985d89ba3fa411c8ee2e351cb27f033":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_cabb904b56424e5e95ea15abf232eabd","placeholder":"​","style":"IPY_MODEL_5d44e0bc2bce49abbc9a52d93525a446","tabbable":null,"tooltip":null,"value":" 4/4 [00:33&lt;00:00,  9.63s/it]"}},"8b0c913691ef480e99669e172ce70adb":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_bb0572c508104b0c84cf41d523c20050","IPY_MODEL_3bac291fefeb429f9ace2fbe7af3657c","IPY_MODEL_8985d89ba3fa411c8ee2e351cb27f033"],"layout":"IPY_MODEL_7080bae9978d440eb09d8896a324f601","tabbable":null,"tooltip":null}},"8ca51f90a4054e69937f789937ab712c":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"926ca45e2ea9465e9bd581a46425acbd":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"929b02223ebe4b899174c5c726695d2a":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"9989287548df47c6b5178e984c6956af":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9b6cbcd59da843aeb440d8330702fb4a":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a2d0c9a4b4074f1984a35069a795a24e":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_d2c1bac072144ed99910bb8705a76547","placeholder":"​","style":"IPY_MODEL_2fca79486aab4f54b011d0700672db25","tabbable":null,"tooltip":null,"value":"Generating (byt5-large): 100%"}},"a8c5f5447900451daf10be224356cd09":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b1e60742f38448cc9a929b2a29a3691c":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_2acc707c62b240d6b972d4be55d9ec36","max":4,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0f7fa209bc904fe899b672d1f9797b78","tabbable":null,"tooltip":null,"value":4}},"bb0572c508104b0c84cf41d523c20050":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_bb3485bc7651465a924fae978036d32c","placeholder":"​","style":"IPY_MODEL_c47474fdb645427eaa981ba5f1d35ea4","tabbable":null,"tooltip":null,"value":"Generating (byt5-base): 100%"}},"bb3485bc7651465a924fae978036d32c":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c12783d9acaa43da8e93ada44d178e08":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c3654e33865b431ea1271e86b678bd9f":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c47474fdb645427eaa981ba5f1d35ea4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"cabb904b56424e5e95ea15abf232eabd":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cea55e9b80fb4b9cbdbf71dadd9d814a":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ceadf441072b4e349b4fb2a041900e2b":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_9b6cbcd59da843aeb440d8330702fb4a","placeholder":"​","style":"IPY_MODEL_36d77ab3afb74805b0209330f6ce1239","tabbable":null,"tooltip":null,"value":" 4/4 [01:03&lt;00:00, 17.89s/it]"}},"d2c1bac072144ed99910bb8705a76547":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d3331733b3654ff2869602bac758cd18":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_ffd326f534564ee8ad2f697d316bdd6d","max":4,"min":0,"orientation":"horizontal","style":"IPY_MODEL_4f7ea862acf84bcbb40c9632607d6cf9","tabbable":null,"tooltip":null,"value":4}},"dd920405b59a49d48d3ac38a5e0fb049":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_cea55e9b80fb4b9cbdbf71dadd9d814a","placeholder":"​","style":"IPY_MODEL_f26195ba2004413d8178b1f14d85c4a4","tabbable":null,"tooltip":null,"value":" 4/4 [00:33&lt;00:00,  9.54s/it]"}},"e64ec84471c8460c8dcfc57fc66e7f17":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_8163079771d34d4eb35aa5ca0689c6a5","max":4,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7e3ae1c0c40d45bc83e2f67bc08cc124","tabbable":null,"tooltip":null,"value":4}},"ec9f3861208248bb9fc1d5ee1b349e9b":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"f26195ba2004413d8178b1f14d85c4a4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"f29bf639e58c47979a43e34f93a30aed":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"f305c34f144b4d74bfde8bb0fa912727":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"f9ec3f3fc9744888a3c47742b57ec81e":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ffd326f534564ee8ad2f697d316bdd6d":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!/usr/bin/env python3\nimport os\nimport gc\nimport math\nimport re\nimport unicodedata\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom typing import List\nfrom contextlib import nullcontext\n\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader, Dataset, Sampler\nfrom tqdm.auto import tqdm\nfrom IPython.display import display\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n\nos.environ.setdefault('TOKENIZERS_PARALLELISM', 'true')","metadata":{"execution":{"iopub.execute_input":"2026-03-23T08:36:35.017057Z","iopub.status.busy":"2026-03-23T08:36:35.016762Z","iopub.status.idle":"2026-03-23T08:36:57.519819Z","shell.execute_reply":"2026-03-23T08:36:57.518968Z"},"papermill":{"duration":22.512096,"end_time":"2026-03-23T08:36:57.521635","exception":false,"start_time":"2026-03-23T08:36:35.009539","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def _cuda_bf16_supported() -> bool:\n    if not torch.cuda.is_available():\n        return False\n    try:\n        return bool(getattr(torch.cuda, 'is_bf16_supported', lambda: False)())\n    except Exception:\n        return False\n\n\n@dataclass\nclass CFG:\n    # Competition data path (Kaggle may mount it in either location)\n    input_dirs: tuple[str, ...] = (\n        '/kaggle/input/competitions/deep-past-initiative-machine-translation',\n        '/kaggle/input/deep-past-initiative-machine-translation',\n    )\n\n    # Models\n    byt5_large_dir: str = '/kaggle/input/datasets/tatsuokoshida/byt5-large-multitask-v002-6-20epochs'\n    byt5_base_dir: str  = '/kaggle/input/datasets/tatsuokoshida/byt5-base-multitask-v002-6-20epochs'\n    byt5_mbr_v2_dir: str = '/kaggle/input/datasets/tatsuokoshida/byt5-akkadian-mbr-v6-v6-v002-6-15epochs'\n    final_byt5_dir: str  = '/kaggle/input/datasets/tatsuokoshida/byt5-akkadian-optimized-34x-v6-v002-6-15epochs'\n    byt5_small_dir: str  = '/kaggle/input/datasets/tatsuokoshida/fulltrain-byt5-small'\n    legacy_byt5_mbr_v2_dir: str = '/kaggle/input/models/mattiaangeli/byt5-akkadian-mbr-v2/pytorch/default/1'\n    legacy_final_byt5_dir: str = '/kaggle/input/datasets/assiaben/final-byt5/byt5-akkadian-optimized-34x'\n\n    output_dir: str = '/kaggle/working'\n\n    # Tokenization / generation\n    max_input_length: int = 1024\n    max_new_tokens: int = 1024\n\n    # Decoding\n    num_beams: int = 8\n    num_beam_cands: int = 4\n    num_sample_cands: int = 2\n    sample_top_p: float = 0.92\n    sample_temperature: float = 0.75\n\n    # Inference\n    batch_size: int = 4\n    num_workers: int = 0\n    num_buckets: int = 6\n    early_stopping: bool = True\n    length_penalty: float = 1.3\n    repetition_penalty: float = 1.2\n\n    use_mixed_precision: bool = True\n    use_better_transformer: bool = True\n    use_bucket_batching: bool = True\n    use_adaptive_beams: bool = True\n\n    def __post_init__(self):\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        Path(self.output_dir).mkdir(parents=True, exist_ok=True)\n\n        if not torch.cuda.is_available():\n            self.use_mixed_precision = False\n            self.use_better_transformer = False\n\n        self.use_bf16_amp = bool(\n            self.use_mixed_precision\n            and self.device.type == 'cuda'\n            and _cuda_bf16_supported()\n        )\n\n        assert self.num_beams >= self.num_beam_cands\n\n\ncfg = CFG()\nprint('PyTorch:', torch.__version__)\nprint('Device :', cfg.device)\nif cfg.device.type == 'cuda':\n    print('GPU    :', torch.cuda.get_device_name(0))\n    print('BF16   :', _cuda_bf16_supported())\nprint('BF16 AMP:', cfg.use_bf16_amp)\n","metadata":{"execution":{"iopub.execute_input":"2026-03-23T08:36:57.536381Z","iopub.status.busy":"2026-03-23T08:36:57.535944Z","iopub.status.idle":"2026-03-23T08:36:57.776462Z","shell.execute_reply":"2026-03-23T08:36:57.775575Z"},"papermill":{"duration":0.249526,"end_time":"2026-03-23T08:36:57.77798","exception":false,"start_time":"2026-03-23T08:36:57.528454","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# DIFF (2026-03-21): 前処理/後処理は `[2-8]dpc-starter-train-v1.ipynb` と同一ロジック前提（submit 側は build_inputs を利用）\n# ==========================================\n# 3. Tokenization & preprocessing\n# ==========================================\nfrom typing import List\n\n\nPREFIX_AKK_EN = \"translate Akkadian to English: \"\nPREFIX_EN_AKK = \"translate English to Akkadian: \"\n\n# --- Pre/Post processing (aligned to notebooks/006/lb-35-9-ensembling-post-processing-baseline.ipynb) ---\n_V2 = re.compile(r\"([aAeEiIuU])(?:2|₂)\")\n_V3 = re.compile(r\"([aAeEiIuU])(?:3|₃)\")\n_ACUTE = str.maketrans({\"a\":\"á\",\"e\":\"é\",\"i\":\"í\",\"u\":\"ú\",\"A\":\"Á\",\"E\":\"É\",\"I\":\"Í\",\"U\":\"Ú\"})\n_GRAVE = str.maketrans({\"a\":\"à\",\"e\":\"è\",\"i\":\"ì\",\"u\":\"ù\",\"A\":\"À\",\"E\":\"È\",\"I\":\"Ì\",\"U\":\"Ù\"})\n\ndef _ascii_to_diacritics(s: str) -> str:\n    s = s.replace(\"sz\", \"š\").replace(\"SZ\", \"Š\")\n    s = s.replace(\"s,\", \"ṣ\").replace(\"S,\", \"Ṣ\")\n    s = s.replace(\"t,\", \"ṭ\").replace(\"T,\", \"Ṭ\")\n    s = _V2.sub(lambda m: m.group(1).translate(_ACUTE), s)\n    s = _V3.sub(lambda m: m.group(1).translate(_GRAVE), s)\n    return s\n\n\n# Unicode fraction map (all single-character Unicode vulgar fractions)\n_UNICODE_FRAC_MAP = {\n    (1, 2): \"½\", (1, 3): \"⅓\", (2, 3): \"⅔\",\n    (1, 4): \"¼\", (3, 4): \"¾\",\n    (1, 5): \"⅕\", (2, 5): \"⅖\", (3, 5): \"⅗\", (4, 5): \"⅘\",\n    (1, 6): \"⅙\", (5, 6): \"⅚\",\n    (1, 7): \"⅐\",\n    (1, 8): \"⅛\", (3, 8): \"⅜\", (5, 8): \"⅝\", (7, 8): \"⅞\",\n    (1, 9): \"⅑\",\n    (1, 10): \"⅒\",\n}\ndef _trunc_float(x: float, places: int = 4) -> float:\n    factor = 10 ** places\n    if x >= 0:\n        return math.floor(x * factor) / factor\n    return -math.floor(-x * factor) / factor\n\ndef _fmt_trunc(x: float, places: int = 4) -> str:\n    return f\"{_trunc_float(x, places):.{places}f}\".rstrip(\"0\").rstrip(\".\")\n\n# Map the 4-decimal *truncated* representation to a Unicode fraction.\n# (Host example: 1.333330000... -> 1.3333; 0.1666 -> ⅙)\n_FRAC_DECIMAL_MAP_4 = {}\nfor (n, d), ch in _UNICODE_FRAC_MAP.items():\n    _FRAC_DECIMAL_MAP_4[_fmt_trunc(n / d, 4)] = ch\n\n# Decimals (incl. long floats)\n_DECIMAL_RE = re.compile(r\"(?<![\\w/])(\\d+\\.\\d+)(?![\\w/])\")\n_LONG_DECIMAL_RE = re.compile(r\"(?<![\\w/])(\\d+\\.\\d{5,})(?![\\w/])\")\n\n# General fractions like \"1/2\" or mixed fractions like \"2 1/2\".\n_MIXED_FRAC_RE = re.compile(r\"(?<![\\w/])(\\d+)\\s+(\\d+)\\s*/\\s*(\\d+)(?![\\w/])\")\n_SIMPLE_FRAC_RE = re.compile(r\"(?<![\\w/])(\\d+)\\s*/\\s*(\\d+)(?![\\w/])\")\n\ndef _mixed_frac_repl(m: re.Match) -> str:\n    ip = int(m.group(1))\n    num = int(m.group(2))\n    den = int(m.group(3))\n    ch = _UNICODE_FRAC_MAP.get((num, den))\n    if ch:\n        return f\"{ip} {ch}\" if ip != 0 else ch\n    return m.group(0)\n\ndef _simple_frac_repl(m: re.Match) -> str:\n    num = int(m.group(1))\n    den = int(m.group(2))\n    ch = _UNICODE_FRAC_MAP.get((num, den))\n    if ch:\n        return ch\n    return m.group(0)\n\ndef _canon_decimal_str(s: str) -> str:\n    # Keep x.5 (e.g., 2.5) as-is by request.\n    if re.fullmatch(r\"[1-9]\\d*\\.5\", s):\n        return s\n\n    m = re.fullmatch(r\"(\\d+)\\.(\\d+)\", s)\n    if not m:\n        return s\n\n    ip = int(m.group(1))\n    frac_digits = m.group(2)\n\n    # Truncate fractional part to 4 digits (no rounding), pad with zeros for lookup.\n    frac4 = (frac_digits + \"0000\")[:4]\n    frac_key = (\"0.\" + frac4).rstrip(\"0\").rstrip(\".\")\n    ch = _FRAC_DECIMAL_MAP_4.get(frac_key)\n    if ch and frac_key != \"0\":\n        if ip == 0:\n            return ch\n        return f\"{ip} {ch}\"\n\n    # Long-float shortening: truncate to 4 digits after the decimal.\n    if len(frac_digits) > 4:\n        return f\"{ip}.{frac4}\".rstrip(\"0\").rstrip(\".\")\n\n    return s\n\n\n_TAG_BIGGAP_RE = re.compile(r\"<\\s*big[\\s_\\-]*gap\\s*>\", re.I)\n_TAG_GAP_RE    = re.compile(r\"<\\s*gap\\s*>\", re.I)\n_BARE_BIGGAP   = re.compile(r\"\\bbig[\\s_\\-]*gap\\b\", re.I)\n_ELLIPSIS_RE   = re.compile(r\"(?:\\.{3,}|…+|\\[\\.+\\])\")\n_BRACKET_X_RE  = re.compile(r\"(\\[\\s*x\\s*\\]|\\(\\s*x\\s*\\))\", re.I)\n_XTOKEN_RUN_RE = re.compile(r\"\\bx(?:\\s+x)+\\b\", re.I)\n_XRUN_RE       = re.compile(r\"(?<!\\w)x{2,}(?!\\w)\", re.I)\n_XTOK_RE       = re.compile(r\"(?<!\\w)x(?!\\w)\", re.I)\n_WS_RE         = re.compile(r\"\\s+\")\n\n_GAP_UNIFIED_RE = re.compile(\n    r\"<\\s*big[\\s_\\-]*gap\\s*>\"\n    r\"|<\\s*gap\\s*>\"\n    r\"|\\bbig[\\s_\\-]*gap\\b\"\n    r\"|\\bx(?:\\s+x)+\\b\"\n    r\"|\\.{3,}|…+|\\[\\.+\\]\"\n    r\"|\\[\\s*x\\s*\\]|\\(\\s*x\\s*\\)\"\n    r\"|(?<!\\w)x{2,}(?!\\w)\"\n    r\"|(?<!\\w)x(?!\\w)\"\n    r\"|\\(\\s*large\\s+break\\s*\\)\"\n    r\"|\\(\\s*break\\s*\\)\"\n    r\"|\\(\\s*\\d+\\s+broken\\s+lines?\\s*\\)\",\n    re.I\n)\n\ndef _normalize_gaps_vec(ser: pd.Series) -> pd.Series:\n    return ser.str.replace(_GAP_UNIFIED_RE, \"<gap>\", regex=True)\n\n\n_CHAR_TRANS = str.maketrans({\n    \"ḫ\":\"h\",\"Ḫ\":\"H\",\"ʾ\":\"\",\n    \"₀\":\"0\",\"₁\":\"1\",\"₂\":\"2\",\"₃\":\"3\",\"₄\":\"4\",\n    \"₅\":\"5\",\"₆\":\"6\",\"₇\":\"7\",\"₈\":\"8\",\"₉\":\"9\",\n    \"—\":\"-\",\"–\":\"-\",\n})\n_SUB_X = \"ₓ\"\n\n_UNICODE_UPPER = r\"A-ZŠṬṢḪ\\u00C0-\\u00D6\\u00D8-\\u00DE\\u0160\\u1E00-\\u1EFF\"\n_UNICODE_LOWER = r\"a-zšṭṣḫ\\u00E0-\\u00F6\\u00F8-\\u00FF\\u0161\\u1E01-\\u1EFF\"\n\n_DET_UPPER_RE = re.compile(r\"\\(([\" + _UNICODE_UPPER + r\"0-9]{1,6})\\)\")\n_DET_LOWER_RE = re.compile(r\"\\(([\" + _UNICODE_LOWER + r\"]{1,4})\\)\")\n\n_PN_RE = re.compile(r\"\\bPN\\b\")\n_KUBABBAR_RE = re.compile(r\"KÙ\\.B\\.\")\n\n\nclass OptimizedPreprocessor:\n    def preprocess_batch(self, texts: List[str]) -> List[str]:\n        ser = pd.Series(texts).fillna(\"\").astype(str)\n        ser = ser.apply(_ascii_to_diacritics)\n\n        # Uppercase determinatives are unwrapped, lowercase ones are converted to braces.\n        ser = ser.str.replace(_DET_UPPER_RE, r\"\\1\", regex=True)\n        ser = ser.str.replace(_DET_LOWER_RE, r\"{\\1}\", regex=True)\n\n        ser = _normalize_gaps_vec(ser)\n        ser = ser.str.translate(_CHAR_TRANS)\n        ser = ser.str.replace(_SUB_X, \"\", regex=False)\n        ser = ser.str.replace(_KUBABBAR_RE, \"KÙ.BABBAR\", regex=True)\n        ser = ser.str.replace(_MIXED_FRAC_RE, _mixed_frac_repl, regex=True)\n        ser = ser.str.replace(_SIMPLE_FRAC_RE, _simple_frac_repl, regex=True)\n        ser = ser.str.replace(_DECIMAL_RE, lambda m: _canon_decimal_str(m.group(1)), regex=True)\n        ser = ser.str.replace(_WS_RE, \" \", regex=True).str.strip()\n        return ser.tolist()\n\n_SOFT_GRAM_RE = re.compile(\n    r\"\\(\\s*(?:fem|plur|pl|sing|singular|plural|\\?|\\!)\"\n    r\"(?:\\.\\s*(?:plur|plural|sing|singular))?\"\n    r\"\\.?\\s*[^)]*\\)\", re.I\n)\n_BARE_GRAM_RE = re.compile(r\"(?<!\\w)(?:fem|sing|pl|plural)\\.?(?!\\w)\\s*\", re.I)\n_UNCERTAIN_RE = re.compile(r\"\\(\\?\\)\")\n# Curly quotes → straight quotes（削除せず置換）\n_CURLY_DQ_RE = re.compile(r\"[\\u201c\\u201d]\")\n_CURLY_SQ_RE = re.compile(r\"[\\u2018\\u2019]\")\n\n_MONTH_RE = re.compile(r\"\\bMonth\\s+(XII|XI|X|IX|VIII|VII|VI|V|IV|III|II|I)\\b\", re.I)\n_ROMAN2INT = {\"I\":1,\"II\":2,\"III\":3,\"IV\":4,\"V\":5,\"VI\":6,\"VII\":7,\"VIII\":8,\"IX\":9,\"X\":10,\"XI\":11,\"XII\":12}\n\n_REPEAT_WORD_RE = re.compile(r\"\\b(\\w+)(?:\\s+\\1\\b)+\")\n_REPEAT_PUNCT_RE = re.compile(r\"([.,])\\1+\")\n_PUNCT_SPACE_RE = re.compile(r\"\\s+([.,:])\")\n\n_FORBIDDEN_TRANS = str.maketrans(\"\", \"\", '——<>⌈⌋⌊[]+ʾ;')\n\n_COMMODITY_RE = re.compile(r'(?<=\\s)-(gold|tax|textiles)\\b')\n_COMMODITY_REPL = {\n    \"gold\": \"pašallum gold\",\n    \"tax\": \"šadduātum tax\",\n    \"textiles\": \"kutānum textiles\",\n}\n\ndef _commodity_repl(m: re.Match) -> str:\n    return _COMMODITY_REPL[m.group(1)]\n\n\n_SHEKEL_REPLS = [\n    (re.compile(r'5\\s+11\\s*/\\s*12\\s+shekels?', re.I), '6 shekels less 15 grains'),\n    (re.compile(r'5\\s*/\\s*12\\s+shekels?', re.I), '⅓ shekel 15 grains'),\n    (re.compile(r'7\\s*/\\s*12\\s+shekels?', re.I), '½ shekel 15 grains'),\n    (re.compile(r'1\\s*/\\s*12\\s*(?:\\(shekel\\)|\\bshekel)?', re.I), '15 grains'),\n]\n\n# スラッシュでの選択肢表現（\"you / she\" 等）を安全に除去する（分数 1/3 を壊さない）\n_SLASH_ALT_RE = re.compile(r'(?<![0-9/])\\s+/\\s+(?![0-9])\\S+')\n\n_ALT_SLASH_PAIR_RE = re.compile(r\"\\b([^\\W\\d_]+)\\s*/\\s*([^\\W\\d_]+)\\b\")\n_STRAY_MARKS_RE = re.compile(r'<<[^>]*>>|<(?!gap\\b)[^>]*>')\n_MULTI_GAP_RE = re.compile(r'(?:<gap>\\s*){2,}')\n\n# 追加の stray marks（.., xx 等）を除去\n_EXTRA_STRAY_RE = re.compile(r'(?<!\\w)(?:\\.\\.+|xx+)(?!\\w)')\n\n# ḫ/Ḫ は念のため h/H に正規化（翻訳側に出た場合の保険）\n_HACEK_TRANS = str.maketrans({\"ḫ\": \"h\", \"Ḫ\": \"H\"})\n\ndef _month_repl(m: re.Match) -> str:\n    return f\"Month {_ROMAN2INT.get(m.group(1).upper(), m.group(1))}\"\n\n\nclass VectorizedPostprocessor:\n    def postprocess_batch(self, translations: List[str]) -> List[str]:\n        s = pd.Series(translations).fillna(\"\").astype(str)\n\n        s = _normalize_gaps_vec(s)\n        s = s.str.replace(_PN_RE, \"<gap>\", regex=True)\n        s = s.str.replace(_COMMODITY_RE, _commodity_repl, regex=True)\n\n        for pat, repl in _SHEKEL_REPLS:\n            s = s.str.replace(pat, repl, regex=True)\n\n        s = s.str.replace(_MIXED_FRAC_RE, _mixed_frac_repl, regex=True)\n        s = s.str.replace(_SIMPLE_FRAC_RE, _simple_frac_repl, regex=True)\n        s = s.str.replace(_DECIMAL_RE, lambda m: _canon_decimal_str(m.group(1)), regex=True)\n\n        s = s.str.replace(_SOFT_GRAM_RE, \" \", regex=True)\n        s = s.str.replace(_BARE_GRAM_RE, \" \", regex=True)\n        s = s.str.replace(_UNCERTAIN_RE, \"\", regex=True)\n\n        s = s.str.replace(_STRAY_MARKS_RE, \"\", regex=True)\n        s = s.str.replace(_EXTRA_STRAY_RE, \"\", regex=True)\n        # Keep the left alternative (e.g., \"you / she\" -> \"you\")\n        s = s.str.replace(_ALT_SLASH_PAIR_RE, r\"\\1\", regex=True)\n        s = s.str.replace(_SLASH_ALT_RE, \"\", regex=True)\n\n        # Curly quotes は削除せず、straight quote に正規化\n        s = s.str.replace(_CURLY_DQ_RE, '\"', regex=True)\n        s = s.str.replace(_CURLY_SQ_RE, \"'\", regex=True)\n\n        s = s.str.replace(_MONTH_RE, _month_repl, regex=True)\n        s = s.str.replace(_MULTI_GAP_RE, \"<gap>\", regex=True)\n\n        s = s.str.replace(\"<gap>\", \"\\x00GAP\\x00\", regex=False)\n        s = s.str.translate(_FORBIDDEN_TRANS)\n        s = s.str.replace(\"\\x00GAP\\x00\", \" <gap> \", regex=False)\n        s = s.str.translate(_HACEK_TRANS)\n\n        s = s.str.replace(_REPEAT_WORD_RE, r\"\\1\", regex=True)\n        for n in range(4, 1, -1):\n            pat = r\"\\b((?:\\w+\\s+){\" + str(n-1) + r\"}\\w+)(?:\\s+\\1\\b)+\"\n            s = s.str.replace(pat, r\"\\1\", regex=True)\n\n        s = s.str.replace(_PUNCT_SPACE_RE, r\"\\1\", regex=True)\n        s = s.str.replace(_REPEAT_PUNCT_RE, r\"\\1\", regex=True)\n        s = s.str.replace(_WS_RE, \" \", regex=True).str.strip()\n\n        return s.tolist()\n\n_preprocessor = OptimizedPreprocessor()\n_postprocessor = VectorizedPostprocessor()\n\ndef resolve_input_dir(input_dirs: tuple[str, ...]) -> str:\n    for d in input_dirs:\n        if Path(d).exists():\n            return d\n    raise FileNotFoundError(\n        \"Competition input directory not found. Tried: \" + \", \".join(input_dirs)\n    )\n\n\ndef build_inputs(test_df: pd.DataFrame) -> tuple[list[str], list[str], list[str]]:\n    if \"id\" not in test_df.columns:\n        raise ValueError(f\"test.csv must contain 'id'. columns={list(test_df.columns)}\")\n    ids = test_df[\"id\"].astype(str).tolist()\n\n    if \"task\" in test_df.columns:\n        tasks = test_df[\"task\"].astype(str).tolist()\n    else:\n        tasks = [\"akk2en\"] * len(test_df)\n\n    if \"source\" in test_df.columns:\n        sources = test_df[\"source\"].astype(str).tolist()\n    elif \"transliteration\" in test_df.columns:\n        sources = test_df[\"transliteration\"].astype(str).tolist()\n    else:\n        raise ValueError(\n            \"test.csv must contain either 'source' or 'transliteration'. \"\n            f\"columns={list(test_df.columns)}\"\n        )\n\n    inputs: list[str] = [\"\"] * len(sources)\n\n    idx_akk2en = [i for i, t in enumerate(tasks) if t == \"akk2en\"]\n    idx_en2akk = [i for i, t in enumerate(tasks) if t == \"en2akk\"]\n\n    src_akk = [sources[i] for i in idx_akk2en]\n    src_en = [sources[i] for i in idx_en2akk]\n\n    src_akk_pp = _preprocessor.preprocess_batch(src_akk) if src_akk else []\n    src_en_pp = _postprocessor.postprocess_batch(src_en) if src_en else []\n\n    for i, s_norm in zip(idx_akk2en, src_akk_pp):\n        inputs[i] = PREFIX_AKK_EN + s_norm\n    for i, s_norm in zip(idx_en2akk, src_en_pp):\n        inputs[i] = PREFIX_EN_AKK + s_norm\n\n    # Keep strict: unknown tasks are a hard error.\n    unknown = sorted(set(tasks) - {\"akk2en\", \"en2akk\"})\n    if unknown:\n        raise ValueError(f\"Unknown task(s): {unknown}\")\n\n    return ids, tasks, inputs\n","metadata":{"execution":{"iopub.execute_input":"2026-03-23T08:36:57.793225Z","iopub.status.busy":"2026-03-23T08:36:57.792729Z","iopub.status.idle":"2026-03-23T08:36:57.833502Z","shell.execute_reply":"2026-03-23T08:36:57.832761Z"},"papermill":{"duration":0.050044,"end_time":"2026-03-23T08:36:57.834892","exception":false,"start_time":"2026-03-23T08:36:57.784848","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def resolve_input_dir(input_dirs: tuple[str, ...]) -> str:\n    for d in input_dirs:\n        if Path(d).exists():\n            return d\n    raise FileNotFoundError('Competition input directory not found. Tried: ' + ', '.join(input_dirs))\n\ndef resolve_model_dir(model_dir: str) -> str:\n    # model_dir は Kaggle Dataset/Notebook/Working のいずれでも良い\n    candidates = [model_dir]\n    for d in candidates:\n        if Path(d).exists():\n            return d\n    raise FileNotFoundError(f'Model directory not found: {model_dir}')\n\ninput_dir = resolve_input_dir(cfg.input_dirs)\ntest_path = f'{input_dir}/test.csv'\nprint('Test:', test_path)\ntest_df = pd.read_csv(test_path)\nprint('Test rows:', len(test_df))\nprint('Columns:', list(test_df.columns))\n\nids, tasks, inputs = build_inputs(test_df)\nprint('Prepared inputs:', len(inputs))\n","metadata":{"execution":{"iopub.execute_input":"2026-03-23T08:36:57.849406Z","iopub.status.busy":"2026-03-23T08:36:57.848894Z","iopub.status.idle":"2026-03-23T08:36:57.880194Z","shell.execute_reply":"2026-03-23T08:36:57.87936Z"},"papermill":{"duration":0.040115,"end_time":"2026-03-23T08:36:57.881505","exception":false,"start_time":"2026-03-23T08:36:57.84139","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 5. Multi-model candidate generation + MBR\n# ==========================================\nclass TextDataset(Dataset):\n    def __init__(self, texts: list[str], tasks: list[str]):\n        if len(texts) != len(tasks):\n            raise ValueError(f'len(texts)={len(texts)} != len(tasks)={len(tasks)}')\n        self.texts = texts\n        self.tasks = tasks\n\n    def __len__(self) -> int:\n        return len(self.texts)\n\n    def __getitem__(self, idx: int) -> dict:\n        return {'index': idx, 'text': self.texts[idx], 'task': self.tasks[idx]}\n\n\nclass BucketBatchSampler(Sampler[list[int]]):\n    def __init__(self, texts: list[str], batch_size: int, num_buckets: int):\n        self.batch_size = batch_size\n        lengths = [max(1, len(t.split())) for t in texts]\n        sorted_idx = sorted(range(len(lengths)), key=lambda i: lengths[i])\n\n        bucket_size = max(1, len(sorted_idx) // max(1, num_buckets))\n        self.buckets = [\n            sorted_idx[i * bucket_size: None if i == num_buckets - 1 else (i + 1) * bucket_size]\n            for i in range(num_buckets)\n        ]\n\n    def __iter__(self):\n        for bucket in self.buckets:\n            for i in range(0, len(bucket), self.batch_size):\n                yield bucket[i:i + self.batch_size]\n\n    def __len__(self):\n        return sum(math.ceil(len(bucket) / self.batch_size) for bucket in self.buckets)\n\n\ndef make_dataloader(tokenizer, texts: list[str], tasks: list[str], *, max_input_length: int) -> DataLoader:\n    def collate_fn(batch: list[dict]):\n        b_indices = [x['index'] for x in batch]\n        b_texts = [x['text'] for x in batch]\n        b_tasks = [x['task'] for x in batch]\n        enc = tokenizer(\n            b_texts,\n            return_tensors='pt',\n            padding=True,\n            truncation=True,\n            max_length=max_input_length,\n        )\n        enc['indices'] = b_indices\n        enc['tasks'] = b_tasks\n        return enc\n\n    ds = TextDataset(texts, tasks)\n\n    if cfg.use_bucket_batching:\n        sampler = BucketBatchSampler(texts, batch_size=cfg.batch_size, num_buckets=cfg.num_buckets)\n        return DataLoader(\n            ds,\n            batch_sampler=sampler,\n            num_workers=cfg.num_workers,\n            collate_fn=collate_fn,\n            pin_memory=(cfg.device.type == 'cuda'),\n        )\n\n    return DataLoader(\n        ds,\n        batch_size=cfg.batch_size,\n        shuffle=False,\n        num_workers=cfg.num_workers,\n        collate_fn=collate_fn,\n        pin_memory=(cfg.device.type == 'cuda'),\n    )\n\n\ndef _move_to_device(batch: dict, device: torch.device) -> dict:\n    out = {}\n    for k, v in batch.items():\n        out[k] = v.to(device) if torch.is_tensor(v) else v\n    return out\n\n\ndef _bf16_ctx(device: torch.device, enabled: bool):\n    if enabled and device.type == 'cuda' and _cuda_bf16_supported():\n        return torch.autocast(device_type='cuda', dtype=torch.bfloat16)\n    return nullcontext()\n\n\ndef _adaptive_beam_size(attention_mask: torch.Tensor | None) -> int:\n    if not cfg.use_adaptive_beams or attention_mask is None:\n        return cfg.num_beams\n    med = float(attention_mask.sum(dim=1).float().median().item())\n    short = max(cfg.num_beam_cands, cfg.num_beams // 2)\n    return short if med < 100 else cfg.num_beams\n\n\ndef postprocess_batch(out_texts: list[str], tasks: list[str]) -> list[str]:\n    if len(out_texts) != len(tasks):\n        raise ValueError(f\"len(out_texts)={len(out_texts)} != len(tasks)={len(tasks)}\")\n    out: list[str] = [\"\" for _ in range(len(out_texts))]\n\n    idx_akk2en = [i for i, t in enumerate(tasks) if t == \"akk2en\"]\n    idx_en2akk = [i for i, t in enumerate(tasks) if t == \"en2akk\"]\n\n    if idx_akk2en:\n        texts = [out_texts[i] for i in idx_akk2en]\n        pp = _postprocessor.postprocess_batch(texts)\n        for i, t in zip(idx_akk2en, pp):\n            out[i] = t\n    if idx_en2akk:\n        texts = [out_texts[i] for i in idx_en2akk]\n        pp = _preprocessor.preprocess_batch(texts)\n        for i, t in zip(idx_en2akk, pp):\n            out[i] = t\n\n    unknown = sorted(set(tasks) - {\"akk2en\", \"en2akk\"})\n    if unknown:\n        raise ValueError(f\"Unknown task(s): {unknown}\")\n    return out\n\n\ndef _is_oom(e: Exception) -> bool:\n    msg = str(e).lower()\n    return ('out of memory' in msg) or ('cuda error: out of memory' in msg)\n\n\ndef generate_grouped(\n    model,\n    tokenizer,\n    input_ids: torch.Tensor,\n    attention_mask: torch.Tensor | None,\n    *,\n    num_return_sequences: int,\n    max_new_tokens: int,\n    gen_kwargs: dict,\n) -> list[list[str]]:\n    bs = int(input_ids.size(0))\n    try:\n        with _bf16_ctx(cfg.device, cfg.use_bf16_amp):\n            gen_ids = model.generate(\n                input_ids=input_ids,\n                attention_mask=attention_mask,\n                max_new_tokens=max_new_tokens,\n                **gen_kwargs,\n            )\n        texts = tokenizer.batch_decode(\n            gen_ids,\n            skip_special_tokens=True,\n            clean_up_tokenization_spaces=True,\n        )\n        return [\n            texts[i * num_return_sequences: (i + 1) * num_return_sequences]\n            for i in range(bs)\n        ]\n    except RuntimeError as e:\n        if _is_oom(e):\n            print(f'[OOM] generate: bs={bs} num_return={num_return_sequences} gen_kwargs={gen_kwargs}')\n            if torch.cuda.is_available():\n                torch.cuda.empty_cache()\n            if bs == 1:\n                raise\n            mid = bs // 2\n            left = generate_grouped(\n                model, tokenizer, input_ids[:mid], attention_mask[:mid] if attention_mask is not None else None,\n                num_return_sequences=num_return_sequences, max_new_tokens=max_new_tokens, gen_kwargs=gen_kwargs,\n            )\n            right = generate_grouped(\n                model, tokenizer, input_ids[mid:], attention_mask[mid:] if attention_mask is not None else None,\n                num_return_sequences=num_return_sequences, max_new_tokens=max_new_tokens, gen_kwargs=gen_kwargs,\n            )\n            return left + right\n        raise\n\n\n# --- MBR selection (chrF++相当) ---\nfrom sacrebleu.metrics import CHRF\n_chrf = CHRF(word_order=2)\n\n\ndef _sim(a: str, b: str) -> float:\n    return float(_chrf.sentence_score(a, [b]).score) / 100.0\n\n\ndef select_mbr(candidates: list[str]) -> str:\n    uniq: list[str] = []\n    seen = set()\n    for c in candidates:\n        c = (c or '').strip()\n        if not c:\n            continue\n        if c in seen:\n            continue\n        seen.add(c)\n        uniq.append(c)\n    if not uniq:\n        return '<gap>'\n    if len(uniq) == 1:\n        return uniq[0]\n\n    n = len(uniq)\n    sims = [[0.0] * n for _ in range(n)]\n    for i in range(n):\n        for j in range(i + 1, n):\n            s = _sim(uniq[i], uniq[j])\n            sims[i][j] = s\n            sims[j][i] = s\n\n    avg = []\n    for i in range(n):\n        avg.append((sum(sims[i]) - sims[i][i]) / (n - 1))\n\n    best_i = int(np.argmax(avg))\n    return uniq[best_i]\n\n\nmodel_specs = [\n    ('byt5-large', cfg.byt5_large_dir, cfg.max_input_length, cfg.max_new_tokens),\n    ('byt5-base', cfg.byt5_base_dir, cfg.max_input_length, cfg.max_new_tokens),\n    ('byt5-small', cfg.byt5_small_dir, cfg.max_input_length, cfg.max_new_tokens),\n    ('byt5-mbr-v6', cfg.byt5_mbr_v2_dir, cfg.max_input_length, cfg.max_new_tokens),\n    ('byt5-optimized-34x-v6', cfg.final_byt5_dir, cfg.max_input_length, cfg.max_new_tokens),\n    ('byt5-mbr-v2', cfg.legacy_byt5_mbr_v2_dir, 524, 384),\n    ('final-byt5', cfg.legacy_final_byt5_dir, 524, 384),\n]\n\nall_candidates: list[list[str]] = [[] for _ in range(len(inputs))]\n\nwith torch.inference_mode():\n    for model_name, model_dir_raw, model_max_input_length, model_max_new_tokens in model_specs:\n        model_dir = resolve_model_dir(model_dir_raw)\n        print('---')\n        print('Model:', model_name, model_dir)\n\n        tokenizer = AutoTokenizer.from_pretrained(model_dir)\n        model = AutoModelForSeq2SeqLM.from_pretrained(model_dir, attn_implementation='eager')\n        model.to(cfg.device)\n        model.eval()\n\n        if cfg.device.type == 'cuda':\n            try:\n                torch.set_float32_matmul_precision('high')\n            except Exception:\n                pass\n\n        if cfg.use_better_transformer and cfg.device.type == 'cuda':\n            try:\n                from optimum.bettertransformer import BetterTransformer\n                model = BetterTransformer.transform(model)\n                print(f'BetterTransformer applied: {model_name}')\n            except Exception as e:\n                print(f'BetterTransformer skipped: {model_name} ({e})')\n\n        dl = make_dataloader(tokenizer, inputs, tasks, max_input_length=model_max_input_length)\n\n        for batch in tqdm(dl, desc=f'Generating ({model_name})'):\n            batch = _move_to_device(batch, cfg.device)\n            sample_indices = batch['indices']\n            b_tasks = batch['tasks']\n            input_ids = batch['input_ids']\n            attention_mask = batch.get('attention_mask')\n\n            beam_size = _adaptive_beam_size(attention_mask)\n\n            beam_grouped = generate_grouped(\n                model,\n                tokenizer,\n                input_ids,\n                attention_mask,\n                num_return_sequences=cfg.num_beam_cands,\n                max_new_tokens=model_max_new_tokens,\n                gen_kwargs={\n                    'do_sample': False,\n                    'num_beams': beam_size,\n                    'num_return_sequences': cfg.num_beam_cands,\n                    'early_stopping': cfg.early_stopping,\n                    'length_penalty': cfg.length_penalty,\n                    'repetition_penalty': cfg.repetition_penalty,\n                },\n            )\n\n            sample_grouped = generate_grouped(\n                model,\n                tokenizer,\n                input_ids,\n                attention_mask,\n                num_return_sequences=cfg.num_sample_cands,\n                max_new_tokens=model_max_new_tokens,\n                gen_kwargs={\n                    'do_sample': True,\n                    'top_p': cfg.sample_top_p,\n                    'temperature': cfg.sample_temperature,\n                    'num_beams': 1,\n                    'num_return_sequences': cfg.num_sample_cands,\n                    'repetition_penalty': cfg.repetition_penalty,\n                },\n            )\n\n            bs = len(b_tasks)\n            for i in range(bs):\n                sample_index = sample_indices[i]\n                all_candidates[sample_index].extend(beam_grouped[i])\n                all_candidates[sample_index].extend(sample_grouped[i])\n\n        try:\n            from optimum.bettertransformer import BetterTransformer\n            model = BetterTransformer.reverse(model)\n        except Exception:\n            pass\n\n        del model\n        del tokenizer\n        gc.collect()\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n\n\npreds = []\nfor i, cands in enumerate(tqdm(all_candidates, desc='MBR selecting')):\n    task_list = [tasks[i]] * len(cands)\n    cands_pp = postprocess_batch(cands, task_list) if cands else []\n    preds.append(select_mbr(cands_pp))\n\n# Safety fallback\npreds = [('<gap>' if (p is None or str(p).strip() == '') else str(p)) for p in preds]\n\nprint('Preds:', len(preds))\nprint('Example:', preds[0][:200] if preds else '<empty>')\n","metadata":{"execution":{"iopub.execute_input":"2026-03-23T08:36:57.895875Z","iopub.status.busy":"2026-03-23T08:36:57.895653Z","iopub.status.idle":"2026-03-23T08:41:37.795464Z","shell.execute_reply":"2026-03-23T08:41:37.794732Z"},"papermill":{"duration":279.909577,"end_time":"2026-03-23T08:41:37.797711","exception":false,"start_time":"2026-03-23T08:36:57.888134","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# DIFF (2026-03-21): scoring error 対策（空文字→<gap> + validate_submission）は `[4-7-2]` を踏襲\nsub_df = pd.DataFrame({\"id\": ids, \"translation\": preds})\nsub_df[\"translation\"] = sub_df[\"translation\"].fillna(\"\").astype(str)\nsub_df.loc[sub_df[\"translation\"].str.strip() == \"\", \"translation\"] = \"<gap>\"\n\ndef validate_submission(df: pd.DataFrame, expected_rows: int):\n    # Kaggle の format error を避けるため、提出直前に最低限の形を検査する。\n    expected_columns = [\"id\", \"translation\"]\n    if list(df.columns) != expected_columns:\n        raise ValueError(f\"Submission columns must be {expected_columns}. got={list(df.columns)}\")\n    if len(df) != expected_rows:\n        raise ValueError(f\"Submission rows mismatch. expected={expected_rows}, got={len(df)}\")\n    if df[\"id\"].isna().any():\n        raise ValueError(\"Submission contains missing id values.\")\n    if df[\"id\"].duplicated().any():\n        dup_ids = df.loc[df[\"id\"].duplicated(), \"id\"].astype(str).head(5).tolist()\n        raise ValueError(f\"Submission contains duplicated id values. examples={dup_ids}\")\n    empty_mask = df[\"translation\"].fillna(\"\").astype(str).str.strip() == \"\"\n    if empty_mask.any():\n        bad_ids = df.loc[empty_mask, \"id\"].astype(str).head(5).tolist()\n        raise ValueError(f\"Submission contains empty translations. examples={bad_ids}\")\n\nvalidate_submission(sub_df, expected_rows=len(ids))\n\nsub_path = Path(cfg.output_dir) / \"submission.csv\"\nsub_df.to_csv(sub_path, index=False)\n\nprint(\"Submission preview:\")\ndisplay(sub_df.head())\nprint(\"Saved to:\", str(sub_path))\n","metadata":{"execution":{"iopub.execute_input":"2026-03-23T08:41:37.814609Z","iopub.status.busy":"2026-03-23T08:41:37.814124Z","iopub.status.idle":"2026-03-23T08:41:37.849166Z","shell.execute_reply":"2026-03-23T08:41:37.848313Z"},"papermill":{"duration":0.045178,"end_time":"2026-03-23T08:41:37.850545","exception":false,"start_time":"2026-03-23T08:41:37.805367","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}