{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"},{"sourceId":13489718,"sourceType":"datasetVersion","datasetId":8563555}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:39:01.591215Z","iopub.execute_input":"2025-10-24T15:39:01.591401Z","iopub.status.idle":"2025-10-24T15:39:03.740918Z","shell.execute_reply.started":"2025-10-24T15:39:01.591378Z","shell.execute_reply":"2025-10-24T15:39:03.739915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm import tqdm\na=0\nids = []\nfor n in tqdm(open('/kaggle/input/cafa-6-protein-function-prediction/Test/testsuperset.fasta')):\n    if n.startswith('>'):\n        a+=1\n        ids.append(n.split(' ')[0].replace('>',''))\nprint(a)\nopen('ids_test.txt','w').write('\\n'.join(ids))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:39:03.742419Z","iopub.execute_input":"2025-10-24T15:39:03.742743Z","iopub.status.idle":"2025-10-24T15:39:05.509699Z","shell.execute_reply.started":"2025-10-24T15:39:03.742722Z","shell.execute_reply":"2025-10-24T15:39:05.508966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_id_go_file_optimized(df_filtered, ids, output_file='id_go_pairs.txt', verbose=True):\n    \"\"\"\n    Optimized version using Polars operations for better performance with large datasets.\n    \n    Args:\n        df_filtered: Polars dataframe with 'db_object_id' and 'go_id' columns\n        ids: List of UniProt IDs to extract\n        output_file: Output file path\n        verbose: Whether to print warnings for missing IDs\n    \n    Returns:\n        Tuple of (found_ids, missing_ids)\n    \"\"\"\n    \n    print(f\"Filtering dataframe for {len(ids)} IDs...\")\n    \n    # Filter dataframe for all IDs at once (much faster than individual filters)\n    filtered_subset = df_filtered.filter(pl.col('db_object_id').is_in(ids))\n    \n    # Get unique IDs that were actually found\n    found_ids = filtered_subset['db_object_id'].unique().to_list()\n    \n    # Find missing IDs\n    missing_ids = list(set(ids) - set(found_ids))\n    \n    # Sort the subset by ID for organized output\n    filtered_subset = filtered_subset.sort('db_object_id')\n    \n    # Write to file with progress bar\n    with open(output_file, 'w') as f:\n        # Group by ID for progress tracking\n        grouped = filtered_subset.group_by('db_object_id', maintain_order=True)\n        \n        for (id_value,), group_df in tqdm(grouped, desc=\"Writing ID-GO pairs\", total=len(found_ids)):\n            for row in group_df.iter_rows():\n                go_id_idx = group_df.columns.index('go_id')\n                go_term = row[go_id_idx]\n                f.write(f\"{id_value}\\t{go_term}\\t1\\n\")\n    \n    # Print summary\n    print(f\"\\n✅ Completed processing {len(ids)} IDs\")\n    print(f\"   - Found: {len(found_ids)} IDs\")\n    print(f\"   - Missing: {len(missing_ids)} IDs\")\n    \n    if missing_ids and verbose:\n        print(f\"\\n⚠️  Missing IDs ({len(missing_ids)}):\")\n        for missing_id in sorted(missing_ids)[:10]:  # Show first 10\n            print(f\"   - {missing_id}\")\n        if len(missing_ids) > 10:\n            print(f\"   ... and {len(missing_ids) - 10} more\")\n    \n    # Save missing IDs to file for reference\n    if missing_ids:\n        missing_file = output_file.replace('.tsv', '_missing_ids.txt')\n        with open(missing_file, 'w') as f:\n            for missing_id in sorted(missing_ids):\n                f.write(f\"{missing_id}\\n\")\n        print(f\"\\n💾 Missing IDs saved to: {missing_file}\")\n    \n    return found_ids, missing_ids\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:39:22.771149Z","iopub.execute_input":"2025-10-24T15:39:22.771480Z","iopub.status.idle":"2025-10-24T15:39:22.782951Z","shell.execute_reply.started":"2025-10-24T15:39:22.771452Z","shell.execute_reply":"2025-10-24T15:39:22.780954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\ndf_filtered = pl.read_csv('/kaggle/input/goa-uniprot-all/selection_ids_goa_uniprot_all.csv')\ndf_filtered.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:41:07.963722Z","iopub.execute_input":"2025-10-24T15:41:07.964071Z","iopub.status.idle":"2025-10-24T15:41:08.349515Z","shell.execute_reply.started":"2025-10-24T15:41:07.964052Z","shell.execute_reply":"2025-10-24T15:41:08.348021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"[n for n in df_filtered['qualifier'].unique() if 'NOT' in n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:41:12.894056Z","iopub.execute_input":"2025-10-24T15:41:12.894341Z","iopub.status.idle":"2025-10-24T15:41:12.987924Z","shell.execute_reply.started":"2025-10-24T15:41:12.894326Z","shell.execute_reply":"2025-10-24T15:41:12.986744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_filtered.shape)\ndf_filtered = df_filtered.filter(~pl.col('qualifier').str.contains('NOT'))\nprint(df_filtered.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:41:52.885172Z","iopub.execute_input":"2025-10-24T15:41:52.885536Z","iopub.status.idle":"2025-10-24T15:41:53.085396Z","shell.execute_reply.started":"2025-10-24T15:41:52.885508Z","shell.execute_reply":"2025-10-24T15:41:53.079936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"found, missing = create_id_go_file_optimized(\n        df_filtered, \n        ids, \n        output_file='submission_goa.tsv',\n        verbose=True\n    )\n    \nprint(\"\\n✨ Done! Check the output files.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:42:17.203443Z","iopub.execute_input":"2025-10-24T15:42:17.203900Z","iopub.status.idle":"2025-10-24T15:42:38.778346Z","shell.execute_reply.started":"2025-10-24T15:42:17.203841Z","shell.execute_reply":"2025-10-24T15:42:38.777089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nkaggle_key = user_secrets.get_secret(\"kaggle_key\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:46:06.389662Z","iopub.execute_input":"2025-10-24T15:46:06.390086Z","iopub.status.idle":"2025-10-24T15:46:06.577330Z","shell.execute_reply.started":"2025-10-24T15:46:06.390050Z","shell.execute_reply":"2025-10-24T15:46:06.575567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport json\n\n# Set your Kaggle credentials as environment variables\n# You need your username and API key from kaggle.json\nos.environ['KAGGLE_USERNAME'] = 'mtinti'\nos.environ['KAGGLE_KEY'] = kaggle_key","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:49:19.779549Z","iopub.execute_input":"2025-10-24T15:49:19.779879Z","iopub.status.idle":"2025-10-24T15:49:19.785787Z","shell.execute_reply.started":"2025-10-24T15:49:19.779839Z","shell.execute_reply":"2025-10-24T15:49:19.784252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!kaggle kernels output sasaleaf/merge-of-2submission-lb-0-25 -p /kaggle/working/merge","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T15:49:22.569818Z","iopub.execute_input":"2025-10-24T15:49:22.570122Z","iopub.status.idle":"2025-10-24T15:51:22.306224Z","shell.execute_reply.started":"2025-10-24T15:49:22.570105Z","shell.execute_reply":"2025-10-24T15:51:22.304069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cat /kaggle/working/merge/submission.tsv 'submission_goa.tsv' > 'submission.tsv'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head 'submission.tsv'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"! rm -r /kaggle/working/merge\n! rm 'submission_goa.tsv'\n! rm 'submission_goa_missing_ids.txt'","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}