{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":116062,"databundleVersionId":14875579},{"sourceType":"datasetVersion","sourceId":14187577,"datasetId":9045748,"databundleVersionId":14981293},{"sourceType":"kernelVersion","sourceId":287175447}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":539.137457,"end_time":"2026-01-02T08:09:25.498146","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-01-02T08:00:26.360689","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"075dfbd6f0874e49a9f8dcffd64ef606":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_5c34a5e5b6314bde8ff71240e2b86385","IPY_MODEL_cd201563ff664e0a8b5f8c6806b68057","IPY_MODEL_25c59c67c2f146d29a395cebdd1ec63d"],"layout":"IPY_MODEL_98857ae7848f439b96cdf8e664e0071b","tabbable":null,"tooltip":null}},"12ecba73ba67420e8a313650393c728c":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_fea0ca5158af422cad64c3a949cecfbd","placeholder":"​","style":"IPY_MODEL_47bf78a3ffb24b508ec2561fd5f3e116","tabbable":null,"tooltip":null,"value":"Grouping: 100%"}},"1c42586658774df784c72ccff63b18f1":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_c3cb57a53a3448478dfdc28d2fc54535","placeholder":"​","style":"IPY_MODEL_308febf8fa954334ac8b177828760fbb","tabbable":null,"tooltip":null,"value":" 279431/279431 [01:58&lt;00:00, 29800.63it/s]"}},"1f20170d3ee444f0a3e20402796a1382":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_8b5d93a2adb74bd587ce68c80e252c8e","max":279431,"min":0,"orientation":"horizontal","style":"IPY_MODEL_bbda2c935093412c9cb7a08027d60caf","tabbable":null,"tooltip":null,"value":279431}},"2331a0a2f7b746d594205abfafe71d05":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_7ef6320a390f44c595d871c64b4d09db","max":41244062,"min":0,"orientation":"horizontal","style":"IPY_MODEL_97d5ef12e6a74fdaaaa91ca37c7154d7","tabbable":null,"tooltip":null,"value":41244062}},"25c59c67c2f146d29a395cebdd1ec63d":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_4e687098297f45e4a7a176263d9a1067","placeholder":"​","style":"IPY_MODEL_5a2ec3472ab84317a6d3f5dacc9c34b1","tabbable":null,"tooltip":null,"value":" 40121/40121 [00:00&lt;00:00, 197222.91it/s]"}},"308febf8fa954334ac8b177828760fbb":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"34e296c7c5684649ab4bed69feb15254":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_12ecba73ba67420e8a313650393c728c","IPY_MODEL_2331a0a2f7b746d594205abfafe71d05","IPY_MODEL_373df637519b44ceae1cc278cd4c5c13"],"layout":"IPY_MODEL_89e9faa4c1d7422bba70938bcdcf6807","tabbable":null,"tooltip":null}},"373df637519b44ceae1cc278cd4c5c13":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_e2717c151e204e24bdd0a227d7d1a942","placeholder":"​","style":"IPY_MODEL_cef85ddaf2114c3291759690b4228903","tabbable":null,"tooltip":null,"value":" 41244062/41244062 [01:55&lt;00:00, 351731.67it/s]"}},"38bbeb997bfb464f84eca0cc35c4ed91":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"3972079dbd70454ab69d0fe8045e8429":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"47bf78a3ffb24b508ec2561fd5f3e116":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"4e687098297f45e4a7a176263d9a1067":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5a2ec3472ab84317a6d3f5dacc9c34b1":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"5c34a5e5b6314bde8ff71240e2b86385":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_dde43a565c5543fcb425973bc09955d1","placeholder":"​","style":"IPY_MODEL_dcc919edccb542988c64b9e1589cddb8","tabbable":null,"tooltip":null,"value":"100%"}},"5ed3f0ce37fc47fd805075c684b20445":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_b9e1061e01c54ac5a487af9e691b0c24","placeholder":"​","style":"IPY_MODEL_38bbeb997bfb464f84eca0cc35c4ed91","tabbable":null,"tooltip":null,"value":"Optimizing: 100%"}},"7ef6320a390f44c595d871c64b4d09db":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"89e9faa4c1d7422bba70938bcdcf6807":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8b5d93a2adb74bd587ce68c80e252c8e":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"97d5ef12e6a74fdaaaa91ca37c7154d7":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"98857ae7848f439b96cdf8e664e0071b":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b52b4b3fcd4a4d39a88710f2d3f8660f":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_5ed3f0ce37fc47fd805075c684b20445","IPY_MODEL_1f20170d3ee444f0a3e20402796a1382","IPY_MODEL_1c42586658774df784c72ccff63b18f1"],"layout":"IPY_MODEL_3972079dbd70454ab69d0fe8045e8429","tabbable":null,"tooltip":null}},"b9e1061e01c54ac5a487af9e691b0c24":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bbda2c935093412c9cb7a08027d60caf":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c3cb57a53a3448478dfdc28d2fc54535":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c80fccb3a091439fb7f8bd40aaff9874":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"cd201563ff664e0a8b5f8c6806b68057":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_d43b59c754ae40aa80942f90ee40d810","max":40121,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c80fccb3a091439fb7f8bd40aaff9874","tabbable":null,"tooltip":null,"value":40121}},"cef85ddaf2114c3291759690b4228903":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"d43b59c754ae40aa80942f90ee40d810":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dcc919edccb542988c64b9e1589cddb8":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"dde43a565c5543fcb425973bc09955d1":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e2717c151e204e24bdd0a227d7d1a942":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fea0ca5158af422cad64c3a949cecfbd":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"3b9b7d9c-27e1-4b59-bd0e-0dada219a68e","cell_type":"code","source":"import gc\nimport os\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\n\n# ==========================================\n# 1. OPTIMIZED OBO PARSER\n# ==========================================\n\n\ndef parse_obo_parents(go_obo_path):\n    print(f\"[1/5] Parsing OBO Ontology...\")\n    term_parents = defaultdict(set)\n    roots = {\"GO:0003674\", \"GO:0008150\", \"GO:0005575\"}\n\n    with open(go_obo_path, \"r\") as f:\n        cur_id = None\n        for line in f:\n            line = line.strip()\n\n            if line == \"[Term]\":\n                cur_id = None\n            elif line.startswith(\"id: \"):\n                cur_id = line.split(\"id: \")[1].strip()\n            elif line.startswith(\"is_a: \"):\n                pid = line.split()[1].strip()\n                if cur_id:\n                    term_parents[cur_id].add(pid)\n            elif line.startswith(\"relationship: part_of \"):\n                parts = line.split()\n                if len(parts) >= 3:\n                    pid = parts[2].strip()\n                    if cur_id:\n                        term_parents[cur_id].add(pid)\n\n    return term_parents, roots\n\n\ndef get_ancestors_map(term_parents):\n    print(\"[1/5] Building Ancestor Map...\")\n    ancestors = {}\n\n    def get_all_ancestors(term):\n        if term in ancestors:\n            return ancestors[term]\n\n        parents = term_parents.get(term, set())\n        all_anc = set(parents)\n\n        for p in parents:\n            all_anc |= get_all_ancestors(p)\n\n        ancestors[term] = all_anc\n        return all_anc\n\n    for term in tqdm(list(term_parents.keys())):\n        get_all_ancestors(term)\n\n    return ancestors\n\n\n# ==========================================\n# 2. LOGIC: PROPAGATION + NORMALIZATION\n# ==========================================\n\n\ndef process_predictions(df, ancestors_map, roots):\n    print(\"[3/5] Processing Predictions (Propagate + Normalize)...\")\n\n    # 1. Convert to Dict Structure for fast access\n    # { protein_id: { go_term: score } }\n    data_map = defaultdict(dict)\n    values = df.values  # protein, term, score\n\n    for pid, term, score in tqdm(values, desc=\"Grouping\"):\n        data_map[pid][term] = float(score)\n\n    new_rows = []\n\n    # 2. Iterate Per Protein\n    for pid, terms_dict in tqdm(data_map.items(), desc=\"Optimizing\"):\n        final_scores = terms_dict.copy()\n\n        # --- A. POSITIVE PROPAGATION ---\n        # Ensure Parent >= Child\n        current_terms = list(terms_dict.keys())\n\n        for term in current_terms:\n            s = terms_dict[term]\n            if term in ancestors_map:\n                for anc in ancestors_map[term]:\n                    final_scores[anc] = max(final_scores.get(anc, 0.0), s)\n\n        # --- B. FORCE ROOTS ---\n        # If we have any prediction, the roots must be 1.0\n        if len(final_scores) > 0:\n            for r in roots:\n                final_scores[r] = 1.0\n\n        # --- C. RANK NORMALIZATION (The Boost) ---\n        # Find the max score for this protein (excluding roots usually 1.0)\n        max_val = 0.0\n        for t, s in final_scores.items():\n            if t not in roots:\n                max_val = max(max_val, s)\n\n        # If the best prediction is weak (e.g., 0.3), scale everything up\n        # Target: Make the max score at least 0.95\n        if max_val > 0 and max_val < 0.95:\n            scale_factor = 0.95 / max_val\n            for t in final_scores:\n                if t not in roots:\n                    final_scores[t] = min(1.0, final_scores[t] * scale_factor)\n\n        # Collect\n        for go_term, score in final_scores.items():\n            # Optimization: Drop extremely low scores to reduce file size\n            if score >= 0.001:\n                new_rows.append((pid, go_term, score))\n\n    return pd.DataFrame(new_rows, columns=[\"protein_id\", \"go_term\", \"score\"])\n\n\n# ==========================================\n# 3. MAIN PIPELINE\n# ==========================================\n\n# Paths\nOBO_PATH = \"/kaggle/input/competitions/cafa-6-protein-function-prediction/Train/go-basic.obo\"\nSUBMISSION_INPUT = \"/kaggle/input/notebooks/kaitonmh/goa-negative-propagation/submission.tsv\"\nSUBMISSION_OUTPUT = \"submission.tsv\"\n\n# 1. Load Ontology\nterm_parents, roots = parse_obo_parents(OBO_PATH)\nancestors_map = get_ancestors_map(term_parents)\n\n# 2. Load Submission\nprint(f\"[2/5] Loading submission...\")\n# If OOM occurs, add nrows=10_000_000 or use chunks\nsubmission = pd.read_csv(\n    SUBMISSION_INPUT,\n    sep=\"\\t\",\n    header=None,\n    names=[\"protein_id\", \"go_term\", \"score\", \"key\"],\n)\nsubmission = submission[[\"protein_id\", \"go_term\", \"score\"]]\n\n# 3. Process\nfinal_df = process_predictions(submission, ancestors_map, roots)\n\n# 4. Save\nprint(f\"[4/5] Saving {len(final_df)} rows...\")\n# Sorting ensures deterministic file for hashing/checks\nfinal_df.sort_values([\"protein_id\", \"score\"], ascending=[True, False], inplace=True)\nfinal_df.to_csv(SUBMISSION_OUTPUT, sep=\"\\t\", index=False, header=False)\n\nprint(f\"[✅] Done. Saved to {SUBMISSION_OUTPUT}\")\nprint(final_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-05-09T12:42:21.728798Z","iopub.execute_input":"2026-05-09T12:42:21.729274Z","iopub.status.idle":"2026-05-09T12:51:30.250714Z","shell.execute_reply.started":"2026-05-09T12:42:21.729239Z","shell.execute_reply":"2026-05-09T12:51:30.249433Z"}},"outputs":[],"execution_count":null},{"id":"5859581f-a7a8-4333-bf02-48ba0e61b1af","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}