{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"accelerator":"GPU","colab":{"gpuType":"T4","provenance":[]},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":2859.017951,"end_time":"2025-11-04T06:14:02.051516","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-11-04T05:26:23.033565","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"001b139f03a74a44a2290095fab89fe2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_22e8eef252074ee598dd859015125853","IPY_MODEL_661c4a4820f849638518d4efba8fe5d0","IPY_MODEL_8494b456ea034bc4ab816f68bb012055"],"layout":"IPY_MODEL_2df63a3128c542408e2c1abb97a81046"}},"01c129d760c84e1688e8e9bfa349f0fb":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"038660a2af824fcbbecbff322e49d9d0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_f2bf17c43ca04a6fb3f030c14e15c151","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7a0212b019714b4b823f83c4d5454143","value":1}},"05611449bf7e42bd96d71415bc9b2449":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_063a36208f304a82bf193e9678169ba6","IPY_MODEL_b4901e1ee1404d89ab0444fd62dba10c","IPY_MODEL_7269d53a93e5442187b4437bf5e1830d"],"layout":"IPY_MODEL_959a8712a0e24d4796292c03b1981fbc"}},"063a36208f304a82bf193e9678169ba6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d6810287c3394657be7e124b608cc4e9","placeholder":"​","style":"IPY_MODEL_2afdfd1fa0b34460ac3f88f6e3d43bec","value":"data/weights/esmc_300m_2024_12_v0.pth: 100%"}},"0ee9ad1511544549b45a2102d0a15833":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_12c98284734447d08d85a939f7c37b78","IPY_MODEL_e9c59aea2bae4a25a427b2591a83db09","IPY_MODEL_ea95ade951954449a8e2ad0a73f61d92"],"layout":"IPY_MODEL_29799eaf8281445babff5047eb277a67"}},"0fcb4ff6dd724867926f2898e7f3b59a":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"12c98284734447d08d85a939f7c37b78":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0fcb4ff6dd724867926f2898e7f3b59a","placeholder":"​","style":"IPY_MODEL_1cf5d99c377742f4b4a20db4c126a14d","value":"config.json: 100%"}},"15572993fb1d425f869fdfd25140c59d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"156ba2d931a4415fb5531416b617395a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_f6481049ac614e44b0aec1e1c3f2e372","placeholder":"​","style":"IPY_MODEL_f31af35faf244b45970c895c83390430","value":" 1.31k/? [00:00&lt;00:00, 18.9kB/s]"}},"1a4ffcc6aeab4dddb86abe7a4f574089":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7b6cb4ed4dd04bfb9913b8c0d948c4a2","IPY_MODEL_519ea91d38db4da5a9fa8ae03be0f46c","IPY_MODEL_f32915e41b6c479ea68d469969c705ae"],"layout":"IPY_MODEL_69382ec9dcad4049b13fd0dcb5beb45c"}},"1cf5d99c377742f4b4a20db4c126a14d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2113145bfcfe4041ae373f8a7406006d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"22e8eef252074ee598dd859015125853":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_bf565f2ad19e4f148638cd9a944d7f60","placeholder":"​","style":"IPY_MODEL_fb88b0bf043748cb8b233494bfb43500","value":".gitattributes: "}},"29799eaf8281445babff5047eb277a67":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2afdfd1fa0b34460ac3f88f6e3d43bec":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2df63a3128c542408e2c1abb97a81046":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2fe03ffae31c463185bfd31f41581382":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4e7fc7523cbf4f2c858de690b5d0c188":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"519ea91d38db4da5a9fa8ae03be0f46c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_b6e0c7abbf2945e78ac5b8877c2845f8","max":4,"min":0,"orientation":"horizontal","style":"IPY_MODEL_4e7fc7523cbf4f2c858de690b5d0c188","value":4}},"574b1ab5cb5a4753b3779d5ffac87130":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6087c05520654f27895d84bc85404968":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"661c4a4820f849638518d4efba8fe5d0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_c96310f5e09048be91f74a31d49f394f","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2113145bfcfe4041ae373f8a7406006d","value":1}},"69382ec9dcad4049b13fd0dcb5beb45c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6bd23fef00594780956a29e12a938588":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7269d53a93e5442187b4437bf5e1830d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_574b1ab5cb5a4753b3779d5ffac87130","placeholder":"​","style":"IPY_MODEL_e0db1eacb4674670854abc1153a4dc3b","value":" 1.33G/1.33G [00:13&lt;00:00, 216MB/s]"}},"7a0212b019714b4b823f83c4d5454143":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7b6cb4ed4dd04bfb9913b8c0d948c4a2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6bd23fef00594780956a29e12a938588","placeholder":"​","style":"IPY_MODEL_ac7ebcb01fb740ed9110e1b50024c7ff","value":"Fetching 4 files: 100%"}},"8494b456ea034bc4ab816f68bb012055":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_01c129d760c84e1688e8e9bfa349f0fb","placeholder":"​","style":"IPY_MODEL_6087c05520654f27895d84bc85404968","value":" 1.52k/? [00:00&lt;00:00, 25.5kB/s]"}},"87c9dc008de64df59e13e94e9d51b671":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"959a8712a0e24d4796292c03b1981fbc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ac7ebcb01fb740ed9110e1b50024c7ff":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"b4901e1ee1404d89ab0444fd62dba10c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_2fe03ffae31c463185bfd31f41581382","max":1332095738,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c101058e22774cc5b702baea651eb449","value":1332095738}},"b6e0c7abbf2945e78ac5b8877c2845f8":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b7b70ad55d5c4710b906cc639b2bb5fd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_df32c96457ac423c82cda6eb400cef3d","IPY_MODEL_038660a2af824fcbbecbff322e49d9d0","IPY_MODEL_156ba2d931a4415fb5531416b617395a"],"layout":"IPY_MODEL_c7ed64c72d124f96b43e2576248a95bf"}},"bd8812a391b14bf3951951723c581288":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bf565f2ad19e4f148638cd9a944d7f60":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c101058e22774cc5b702baea651eb449":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c7ed64c72d124f96b43e2576248a95bf":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c96310f5e09048be91f74a31d49f394f":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"cac033e6cd424d49ba87cb1a956cb802":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d4d3e61067174eccacc0e6d3a610abcd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"d6810287c3394657be7e124b608cc4e9":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"df32c96457ac423c82cda6eb400cef3d":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_bd8812a391b14bf3951951723c581288","placeholder":"​","style":"IPY_MODEL_f316149b6e7a49dcbb6e1d53b8e281fd","value":"README.md: "}},"e0db1eacb4674670854abc1153a4dc3b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e46cff69d8b54f49bc00302fd858b09d":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e70f80b79f2145579451034f96df247b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e9c59aea2bae4a25a427b2591a83db09":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_15572993fb1d425f869fdfd25140c59d","max":2,"min":0,"orientation":"horizontal","style":"IPY_MODEL_d4d3e61067174eccacc0e6d3a610abcd","value":2}},"ea95ade951954449a8e2ad0a73f61d92":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_cac033e6cd424d49ba87cb1a956cb802","placeholder":"​","style":"IPY_MODEL_87c9dc008de64df59e13e94e9d51b671","value":" 2.00/2.00 [00:00&lt;00:00, 45.8B/s]"}},"f2bf17c43ca04a6fb3f030c14e15c151":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"f316149b6e7a49dcbb6e1d53b8e281fd":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f31af35faf244b45970c895c83390430":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f32915e41b6c479ea68d469969c705ae":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_e46cff69d8b54f49bc00302fd858b09d","placeholder":"​","style":"IPY_MODEL_e70f80b79f2145579451034f96df247b","value":" 4/4 [00:13&lt;00:00,  3.68s/it]"}},"f6481049ac614e44b0aec1e1c3f2e372":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fb88b0bf043748cb8b233494bfb43500":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I challenged myself to train a **model** that will predict the **functions of a protein in human body** based on its **residue sequence.**\n\nI fed the residue sequence to an already existing pre-trained **ESM Cambrian** protein representation model to get the representation of the protein sequences\n\nAnd I also created a **Transformer model** and trained it to ***predict the functions of proteins, based on the representation sequence*** outputted by the ESM Cambrian model.\n","metadata":{"id":"13TcGboadFF4"}},{"cell_type":"code","source":"!pip install esm\n!pip install torchmetrics","metadata":{"id":"87831175","outputId":"d2b511ec-91ef-461e-88a1-f7df96cbf573","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:01.931568Z","iopub.execute_input":"2025-11-08T14:22:01.932210Z","iopub.status.idle":"2025-11-08T14:22:08.750178Z","shell.execute_reply.started":"2025-11-08T14:22:01.932183Z","shell.execute_reply":"2025-11-08T14:22:08.749423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nfrom pathlib import Path\nfrom esm.models.esmc import ESMC\nfrom esm.sdk.api import ESMProtein, LogitsConfig\nimport pandas as pd\nimport torch\nimport torch.nn.functional as F\nimport os\nfrom Bio import SeqIO\nimport networkx as nx\nfrom torch import nn\nfrom collections import Counter\nimport numpy as np\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.model_selection import train_test_split\nimport random\nfrom torchmetrics.classification import MultilabelF1Score\n\n\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ntorch.set_default_device(device)","metadata":{"id":"7766b9d6","outputId":"b5eef1cf-e77e-41ed-877b-468a61529638","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:08.752162Z","iopub.execute_input":"2025-11-08T14:22:08.752576Z","iopub.status.idle":"2025-11-08T14:22:08.758360Z","shell.execute_reply.started":"2025-11-08T14:22:08.752541Z","shell.execute_reply":"2025-11-08T14:22:08.757729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This is the labels dataset showing the term(protein's function ids) assigned to each protein\nlabels_df = pd.read_csv(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\", sep='\\t')\nlabels_df.head()","metadata":{"id":"82de0c84","outputId":"06ec9011-1721-46fd-ec0d-a6e40d9ae807","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:08.759191Z","iopub.execute_input":"2025-11-08T14:22:08.759431Z","iopub.status.idle":"2025-11-08T14:22:08.996571Z","shell.execute_reply.started":"2025-11-08T14:22:08.759407Z","shell.execute_reply":"2025-11-08T14:22:08.995846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ModelArgs:\n    batch_size = 4 #temporarly - because of the limited GPU Ram\n    dim = 960\n    n_layers = 2\n    n_heads = 12\n    head_dim = dim//n_heads\n    max_seq_len = 3200 #temporarly - because of the limited GPU Ram\n    extra_tokens = 3\n    lr = 1e-5 #using low learning rate as the batch size is too small\n    out_dim = 1000\n\nargs = ModelArgs()","metadata":{"id":"dbc2d8e9","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:08.997414Z","iopub.execute_input":"2025-11-08T14:22:08.997718Z","iopub.status.idle":"2025-11-08T14:22:09.002127Z","shell.execute_reply.started":"2025-11-08T14:22:08.997680Z","shell.execute_reply":"2025-11-08T14:22:09.001226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#As there are many terms, we will shortlist only the top appearing ones, otherwise we would have just few times less output classes than observations\n\nprotein_to_terms = labels_df.groupby('EntryID')['term'].apply(list).to_dict()\n\nterm_counts = Counter()\nfor terms in protein_to_terms.values():\n    term_counts.update(terms)\ntop_terms = [t for t, _ in term_counts.most_common(args.out_dim)]\nmlb = MultiLabelBinarizer(classes=range(len(top_terms)))","metadata":{"id":"bf6ec92a","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:09.004507Z","iopub.execute_input":"2025-11-08T14:22:09.004776Z","iopub.status.idle":"2025-11-08T14:22:10.321294Z","shell.execute_reply.started":"2025-11-08T14:22:09.004758Z","shell.execute_reply":"2025-11-08T14:22:10.320712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for protein in protein_to_terms:\n    protein_to_terms[protein] = [t for t in protein_to_terms[protein] if t in top_terms]","metadata":{"id":"406375ec","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:10.321963Z","iopub.execute_input":"2025-11-08T14:22:10.322218Z","iopub.status.idle":"2025-11-08T14:22:14.465410Z","shell.execute_reply.started":"2025-11-08T14:22:10.322194Z","shell.execute_reply":"2025-11-08T14:22:14.464836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"term_to_idx = {term: idx for idx, term in enumerate(top_terms)}","metadata":{"id":"2d869672","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.466197Z","iopub.execute_input":"2025-11-08T14:22:14.466445Z","iopub.status.idle":"2025-11-08T14:22:14.470806Z","shell.execute_reply.started":"2025-11-08T14:22:14.466424Z","shell.execute_reply":"2025-11-08T14:22:14.470040Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Loading the sequences into dict, in the following format: {protein id: residue sequence}\nsequences = {}\n\nwith open(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta\", 'r') as f:\n    for record in SeqIO.parse(f, \"fasta\"):\n        protein_id = record.id.split('|')[1]\n\n        a_seq = str(record.seq)[:args.max_seq_len-args.extra_tokens] #as the GPU ram is too limited, we will truncate the protein length\n        sequences[protein_id] = a_seq","metadata":{"id":"6931c8da","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.471524Z","iopub.execute_input":"2025-11-08T14:22:14.471852Z","iopub.status.idle":"2025-11-08T14:22:14.918488Z","shell.execute_reply.started":"2025-11-08T14:22:14.471827Z","shell.execute_reply":"2025-11-08T14:22:14.917670Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Bidirectional Transformer model** LLM-alike, which takes as input an already processed hidden state(about which I will introduce in the following cells)","metadata":{"id":"evKtpvhso3bU"}},{"cell_type":"code","source":"class ProteinModel(nn.Module):\n  def __init__(self, args):\n    super().__init__()\n\n    self.dim = args.dim\n    self.out_dim = args.out_dim\n    self.n_layers = args.n_layers\n\n    self.layers = torch.nn.ModuleList()\n    for _ in range(self.n_layers):\n        self.layers.append(Block(args))\n\n    self.head = nn.Sequential(\n        nn.RMSNorm(self.dim),\n        nn.SiLU(),\n        nn.Linear(self.dim, self.dim),\n        nn.RMSNorm(self.dim),\n        nn.Linear(self.dim, self.out_dim),\n    )\n\n    self.apply(self._init_weights)\n\n  def _init_weights(self,module):\n    if isinstance(module, nn.Linear):\n        torch.nn.init.normal_(module.weight,mean=0.0,std=0.02)\n        if module.bias is not None:\n            torch.nn.init.zeros_(module.bias)\n\n  def forward(self,x,attn_pad_mask):\n    for layer in self.layers:\n      x = layer(x,attn_pad_mask)\n    return self.head(x[:,0])\n","metadata":{"id":"4f6fe2ab","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.919385Z","iopub.execute_input":"2025-11-08T14:22:14.919746Z","iopub.status.idle":"2025-11-08T14:22:14.926513Z","shell.execute_reply.started":"2025-11-08T14:22:14.919723Z","shell.execute_reply":"2025-11-08T14:22:14.925714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Block(nn.Module):\n    def __init__(self,args):\n        super().__init__()\n        self.dim = args.dim\n        self.attn_norm = nn.RMSNorm(self.dim)\n        self.attn = MHA(args)\n        self.mlp_norm = nn.RMSNorm(self.dim)\n        self.mlp = MLP(args)\n\n    def forward(self,x,attn_pad_mask):\n        x = x + self.attn(self.attn_norm(x), attn_pad_mask)\n        x = x + self.mlp(self.mlp_norm(x))\n        return x","metadata":{"id":"630ff794","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.927318Z","iopub.execute_input":"2025-11-08T14:22:14.927576Z","iopub.status.idle":"2025-11-08T14:22:14.943128Z","shell.execute_reply.started":"2025-11-08T14:22:14.927548Z","shell.execute_reply":"2025-11-08T14:22:14.942415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def precompute_freqs_cis(args):\n    dim = args.head_dim\n    max_seq_len = args.max_seq_len\n    base = 10_000.0\n    freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))\n    t = torch.arange(max_seq_len)\n    freqs = torch.outer(t, freqs)\n    freqs_cis = torch.polar(torch.ones_like(freqs), freqs)\n    return freqs_cis\n\n\ndef apply_rope(x,freqs_cis):\n    dtype = x.dtype\n    x = torch.view_as_complex(x.float().view(*x.shape[:-1], -1, 2))\n    freqs_cis = freqs_cis[:x.size(1)].view(1, x.size(1), 1, x.size(-1))\n    y = torch.view_as_real(x * freqs_cis).flatten(3)\n    return y[:,:,:,:3].to(dtype)","metadata":{"id":"738745e0","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.943882Z","iopub.execute_input":"2025-11-08T14:22:14.944075Z","iopub.status.idle":"2025-11-08T14:22:14.957224Z","shell.execute_reply.started":"2025-11-08T14:22:14.944060Z","shell.execute_reply":"2025-11-08T14:22:14.956604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MHA(nn.Module):\n    def __init__(self,args):\n        super().__init__()\n\n        self.dim = args.dim\n        self.n_heads = args.n_heads\n        self.head_dim = self.dim//self.n_heads\n        self.scale = self.head_dim ** -0.5\n\n        self.qkv = nn.Linear(self.dim,self.head_dim*self.n_heads*3,bias=False)\n\n        self.c_proj = nn.Linear(self.n_heads * self.head_dim, self.dim, bias=False)\n\n        self.register_buffer(\"freqs_cis\", precompute_freqs_cis(args), persistent=False)\n\n\n\n    def forward(self,x, attn_pad_mask):\n        B, S, d = x.size()\n\n        qkv = self.qkv(x)\n        q, k, v = qkv.split(self.n_heads * self.head_dim, dim=2)\n\n        q = q.view(B, S, self.n_heads, self.head_dim)\n        k = k.view(B, S, self.n_heads, self.head_dim)\n        v = v.view(B, S, self.n_heads, self.head_dim)\n\n        q = apply_rope(q,self.freqs_cis[:S])\n        k = apply_rope(k,self.freqs_cis[:S])\n\n        q = torch.einsum(\"bshd->bhsd\",q)\n        k = torch.einsum(\"bshd->bhds\",k)\n        v = torch.einsum(\"bshd->bhsd\",v)\n\n\n        attn = (q @ k) * self.scale\n        if attn_pad_mask is not None:\n            attn = attn.masked_fill(attn_pad_mask, float('-inf'))\n\n        attn = F.softmax(attn, dim=-1)\n\n        out = attn @ v\n\n        out = torch.einsum(\"bhsd->bshd\",out).contiguous()\n        out = self.c_proj(out.flatten(-2))\n\n        return out","metadata":{"id":"0192662a","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.957916Z","iopub.execute_input":"2025-11-08T14:22:14.958103Z","iopub.status.idle":"2025-11-08T14:22:14.967429Z","shell.execute_reply.started":"2025-11-08T14:22:14.958088Z","shell.execute_reply":"2025-11-08T14:22:14.966848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MLP(nn.Module):\n    def __init__(self,args):\n        super().__init__()\n        self.dim = args.dim\n        self.w1 = nn.Linear(self.dim,self.dim*2)\n        self.w2 = nn.Linear(self.dim*2,self.dim)\n    def forward(self,x):\n        return self.w2(F.silu(self.w1(x)))","metadata":{"id":"549f528a","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.968266Z","iopub.execute_input":"2025-11-08T14:22:14.968563Z","iopub.status.idle":"2025-11-08T14:22:14.983806Z","shell.execute_reply.started":"2025-11-08T14:22:14.968547Z","shell.execute_reply":"2025-11-08T14:22:14.983213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_loss(model, x, attn_pad_mask, targets, criterion, args):\n    out_dim = args.out_dim\n    logits = model(x,attn_pad_mask)\n    loss = criterion(logits,targets)\n\n    return loss, logits","metadata":{"id":"0fab233c","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.985800Z","iopub.execute_input":"2025-11-08T14:22:14.986017Z","iopub.status.idle":"2025-11-08T14:22:14.997875Z","shell.execute_reply.started":"2025-11-08T14:22:14.986002Z","shell.execute_reply":"2025-11-08T14:22:14.997193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = ProteinModel(args)\nopt = torch.optim.AdamW(model.parameters(),lr=8e-6)\nmodel.train()\n\n#ESM-C is a pre-trained protein representation model, from this model will get a hidden state of a protein, which is a sequence of residues\nclient = ESMC.from_pretrained(\"esmc_300m\").to(device)\nclient.eval()","metadata":{"id":"b41204f3","outputId":"9c19c000-6ed9-4d34-a402-8ee775c3ec91","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:14.998509Z","iopub.execute_input":"2025-11-08T14:22:14.998737Z","iopub.status.idle":"2025-11-08T14:22:16.485506Z","shell.execute_reply.started":"2025-11-08T14:22:14.998721Z","shell.execute_reply":"2025-11-08T14:22:16.484457Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Preparing the final training data sequences and the corresponding multi-hot encoding","metadata":{"id":"zUNWlT6k11m9"}},{"cell_type":"code","source":"shuffled_items = list(protein_to_terms.items())\nrandom.shuffle(shuffled_items)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:16.486475Z","iopub.execute_input":"2025-11-08T14:22:16.486781Z","iopub.status.idle":"2025-11-08T14:22:16.531964Z","shell.execute_reply.started":"2025-11-08T14:22:16.486755Z","shell.execute_reply":"2025-11-08T14:22:16.531273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_logits = []\nlabels_idx = []\n\nfor protein_id, terms in shuffled_items:\n    terms_idx = []\n    for t in terms:\n        if t in term_to_idx:\n            terms_idx.append(term_to_idx[t])\n\n    if len(terms_idx) < 1:\n        continue\n\n    if protein_id in sequences:\n        seq_id = sequences[protein_id]\n    else:\n        continue\n\n\n    sequence_logits.append(seq_id)\n    labels_idx.append(terms_idx)\ny_encoded = torch.tensor(mlb.fit_transform(labels_idx),dtype=torch.float32)","metadata":{"id":"bcadef77","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:16.532722Z","iopub.execute_input":"2025-11-08T14:22:16.533055Z","iopub.status.idle":"2025-11-08T14:22:17.381211Z","shell.execute_reply.started":"2025-11-08T14:22:16.533021Z","shell.execute_reply":"2025-11-08T14:22:17.380358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_pos = y_encoded.sum(dim=0)       # per class positive counts\nnum_neg = y_encoded.size(0) - num_pos\npos_weight = num_neg / num_pos    # inverse frequency\n\npos_weight = pos_weight.to(device)\n#For multiclass classification, as a single protein can have several functions (sigmoid loss)\ncriterion = nn.BCEWithLogitsLoss()#pos_weight=pos_weight)\n\nf1_score = MultilabelF1Score(num_labels=args.out_dim,average='micro').to(device)\nsigm = nn.Sigmoid()\n","metadata":{"id":"bdTtHP7aChFw","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:17.382072Z","iopub.execute_input":"2025-11-08T14:22:17.382299Z","iopub.status.idle":"2025-11-08T14:22:17.389602Z","shell.execute_reply.started":"2025-11-08T14:22:17.382276Z","shell.execute_reply":"2025-11-08T14:22:17.389014Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"UNDER CONSTRUCTION","metadata":{"id":"4D5X8ettNkuC"}},{"cell_type":"code","source":"for _ in range(3):\n    loss_list = []\n    \n    for b in range(0,len(y_encoded),args.batch_size):\n    \n        pre_x = sequence_logits[b:b+args.batch_size]\n        pre_x = client._tokenize(pre_x)\n        with torch.no_grad():\n            x = client(pre_x).hidden_states[20].float() # getting the 20th layer's output (out of 30 layers)\n    \n        torch.cuda.empty_cache()\n        targets = y_encoded[b:b+args.batch_size]\n        targets = targets * (1 - 0.0045) + 0.5 * 0.0045\n    \n        opt.zero_grad()\n        model.train()\n        attn_pad_mask = pre_x == 1\n        attn_pad_mask.unsqueeze_(1)\n        attn_pad_mask.unsqueeze_(1)\n        loss, logits = get_loss(model,x,attn_pad_mask,targets,criterion,args)\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        opt.step()\n        loss_list.append(loss.item())\n        if b%40 == 0:\n          acc = f1_score(sigm(logits),y_encoded[b:b+args.batch_size])\n          print(f'Average loss {sum(loss_list)/len(loss_list)} while accuracy is {acc}, at step {b}, epoch {_}')\n          loss_list = []\n        torch.cuda.empty_cache()\n","metadata":{"id":"1b2de1e9","outputId":"898766c9-1856-498a-f1c3-8b3287023dbc","trusted":true,"execution":{"iopub.status.busy":"2025-11-08T14:22:17.390568Z","iopub.execute_input":"2025-11-08T14:22:17.390891Z","execution_failed":"2025-11-08T14:40:18.786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"0fliZwIwz8sF","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"id":"YCCKNiDW4_IZ","trusted":true},"outputs":[],"execution_count":null}]}