{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14173811,"sourceType":"datasetVersion","datasetId":9034760},{"sourceId":14187577,"sourceType":"datasetVersion","datasetId":9045748},{"sourceId":14207560,"sourceType":"datasetVersion","datasetId":9061916},{"sourceId":14219168,"sourceType":"datasetVersion","datasetId":9039112}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc\nfrom collections import defaultdict\n\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:44:23.487604Z","iopub.execute_input":"2025-12-17T04:44:23.488768Z","iopub.status.idle":"2025-12-17T04:44:23.496945Z","shell.execute_reply.started":"2025-12-17T04:44:23.488725Z","shell.execute_reply":"2025-12-17T04:44:23.495807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_train_terms(path):\n    mapping = defaultdict(list)\n    df = pd.read_csv(path, sep=\"\\t\", header=None, names=[\"protein\",\"go\",\"ont\"], dtype=str)\n    for _, r in tqdm(df.iterrows(), total=len(df)): \n        mapping[r.protein].append(r.go)\n    print(f\"[io] Read training annotations for {len(mapping)} proteins from {path}\")\n    return mapping\n\ndef parse_obo(go_obo_path):\n    parents = defaultdict(set)\n    children = defaultdict(set)\n    \n    if not os.path.exists(go_obo_path): \n        return parents, children\n        \n    with open(go_obo_path,\"r\") as f:\n        cur_id=None\n        for line in f:\n            line=line.strip()\n            if line==\"[Term]\": \n                cur_id=None\n            elif line.startswith(\"id: \"): \n                cur_id=line.split(\"id: \")[1].strip()\n            elif line.startswith(\"is_a: \"):\n                pid=line.split()[1].strip()\n                if cur_id: \n                    parents[cur_id].add(pid)\n                    children[pid].add(cur_id)\n            elif line.startswith(\"relationship: part_of \"):\n                parts=line.split(); \n                if len(parts)>=3:\n                    pid=parts[2].strip()\n                    if cur_id: \n                        parents[cur_id].add(pid)\n                        children[pid].add(cur_id)\n    print(f\"[io] Parsed OBO: {len(parents)} nodes with parents\")\n    return parents, children\n\ndef get_ancestors(go_id, parents):\n    ans=set()\n    stack=[go_id]\n    while stack:\n        cur=stack.pop()\n        for p in parents.get(cur,[]): \n            if p not in ans:\n                ans.add(p)\n                stack.append(p)\n    return ans\n\ndef get_descendants(go_id):\n    desc = set()\n    stack = [go_id]\n    while stack:\n        cur = stack.pop()\n        for child in children_map.get(cur, []):\n            if child not in desc:\n                desc.add(child)\n                stack.append(child)\n    return desc\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:44:23.499384Z","iopub.execute_input":"2025-12-17T04:44:23.500283Z","iopub.status.idle":"2025-12-17T04:44:23.534631Z","shell.execute_reply.started":"2025-12-17T04:44:23.500233Z","shell.execute_reply":"2025-12-17T04:44:23.533560Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_terms = read_train_terms(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\")\nparents_map, children_map = parse_obo(\"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:44:23.536025Z","iopub.execute_input":"2025-12-17T04:44:23.536352Z","iopub.status.idle":"2025-12-17T04:45:07.977228Z","shell.execute_reply.started":"2025-12-17T04:44:23.536323Z","shell.execute_reply":"2025-12-17T04:45:07.975604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_ver227.csv')\ngo_annotations = go_annotations.drop_duplicates()\nprint(f'[+] Dataset shape: {go_annotations.shape}')\ngo_annotations.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:07.980078Z","iopub.execute_input":"2025-12-17T04:45:07.980580Z","iopub.status.idle":"2025-12-17T04:45:11.022015Z","shell.execute_reply.started":"2025-12-17T04:45:07.980377Z","shell.execute_reply":"2025-12-17T04:45:11.020719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations.qualifier.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:11.022716Z","iopub.execute_input":"2025-12-17T04:45:11.023105Z","iopub.status.idle":"2025-12-17T04:45:11.257545Z","shell.execute_reply.started":"2025-12-17T04:45:11.023085Z","shell.execute_reply":"2025-12-17T04:45:11.256534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_ver228.tsv', sep='\\t')\ngo_annotations = go_annotations.drop_duplicates()\nprint(f'[+] Dataset shape: {go_annotations.shape}')\ngo_annotations.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:11.259273Z","iopub.execute_input":"2025-12-17T04:45:11.259627Z","iopub.status.idle":"2025-12-17T04:45:14.454377Z","shell.execute_reply.started":"2025-12-17T04:45:11.259598Z","shell.execute_reply":"2025-12-17T04:45:14.452913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations.qualifier.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:14.455596Z","iopub.execute_input":"2025-12-17T04:45:14.455900Z","iopub.status.idle":"2025-12-17T04:45:14.657358Z","shell.execute_reply.started":"2025-12-17T04:45:14.455878Z","shell.execute_reply":"2025-12-17T04:45:14.656062Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<p style=\"background-color:#e6f7ff; \n          padding:15px; \n          color:#111;\n          font-size:16px;\n          border-width:3px; \n          border-color:#d0eefc; \n          border-style:solid;\n          border-radius:6px\">\n🧬 <code>NOT</code> qualifiers explicitly indicate that a <code>protein</code> is <em>not</em> associated with a given <code>GO term</code>. These annotations serve as reliable <code>negative examples</code>.<br>\nIf a protein is not linked to a specific <code>GO term</code>, it is also not linked to any of its <code>descendants</code> within the Gene Ontology hierarchy.<br>\nBy propagating these <code>negative annotations</code> through the ontology, we can reduce <code>false positives</code> and achieve a <strong>more accurate leaderboard score</strong>.\n</p>\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndef compare_csv_overlap(file1_path, file2_path):\n    df1 = pd.read_csv(file1_path, dtype=str)\n    df2 = pd.read_csv(file2_path, dtype=str, sep='\\t')\n\n    keys = ['protein_id', 'go_term', 'qualifier']\n\n    if not set(keys).issubset(df1.columns) or not set(keys).issubset(df2.columns):\n        return\n\n    set1 = df1[keys].drop_duplicates()\n    set2 = df2[keys].drop_duplicates()\n\n    common_rows = pd.merge(set1, set2, on=keys, how='inner')\n\n    count_common = len(common_rows)\n    count_file1 = len(set1)\n    count_file2 = len(set2)\n\n    print(f\"--- RESULTS ---\")\n    print(f\"Total unique in File 1: {count_file1}\")\n    print(f\"Total unique in File 2: {count_file2}\")\n    print(f\"---------------------------------\")\n    print(f\"Common count: {count_common}\")\n    \n    if count_file1 > 0:\n        print(f\"Rate in File 1: {(count_common / count_file1) * 100:.2f}%\")\n        print(f\"Rate in File 2: {(count_common / count_file2) * 100:.2f}%\")\n\ncompare_csv_overlap('/kaggle/input/protein-go-annotations/goa_uniprot_ver227.csv', '/kaggle/input/protein-go-annotations/goa_uniprot_ver228.tsv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:14.658898Z","iopub.execute_input":"2025-12-17T04:45:14.659236Z","iopub.status.idle":"2025-12-17T04:45:25.215321Z","shell.execute_reply.started":"2025-12-17T04:45:14.659208Z","shell.execute_reply":"2025-12-17T04:45:25.213729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"[1/3] Filtering Negative Annotations ..\")\nnegative_annots = go_annotations[go_annotations['qualifier'].str.contains('NOT', na=False)]\nnegative_annots = negative_annots.drop(columns=['qualifier']).drop_duplicates()\n\nprint(f\"[2/3] Propagate Negative Terms ..\")\nnegative_annots = negative_annots.groupby('protein_id')['go_term'].apply(list).to_dict()\n\npropagated={}\nfor p in tqdm(negative_annots.keys()):\n    terms=set(negative_annots[p])\n    extra=set()\n    for t in list(terms): \n        extra |= get_descendants(t)\n    propagated[p] = sorted(terms | extra)\n        \nnegative_annots = propagated\n\nprint(f\"[3/3] Extract Unique Keys ..\")\nrows = [(protein_id, go_term) for protein_id, terms in negative_annots.items() for go_term in terms]\nnegative_df = pd.DataFrame(rows, columns=[\"protein_id\", \"go_term\"])\nnegative_df['pred_key'] = negative_df.protein_id.apply(str) + '_' + negative_df.go_term.apply(str)\nnegative_keys = set(negative_df['pred_key'])\n\ndel negative_df\ngc.collect()\n\nprint(f\"Total unique negative protein-GO pairs: {len(negative_keys)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:25.219383Z","iopub.execute_input":"2025-12-17T04:45:25.219763Z","iopub.status.idle":"2025-12-17T04:45:27.209328Z","shell.execute_reply.started":"2025-12-17T04:45:25.219741Z","shell.execute_reply":"2025-12-17T04:45:27.207986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"[1/4] Loading GOA Annotations ..\")\ngo_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_ver227.csv')\n\nprint(f\"[2/4] Removing unwanted annotations ..\")\ngo_annotations = go_annotations[~go_annotations['qualifier'].str.contains('NOT', na=False)]\ngo_annotations.drop(columns=['qualifier'], inplace=True)\ngo_annotations = go_annotations.drop_duplicates()\n\nprint(f\"[3/4] Set Ground-Truth Score ..\")\ngo_annotations['score'] = round(1.0, 3)\n\nprint(f\"[4/4] Setting Key ..\")\ngo_annotations['pred_key'] = go_annotations['protein_id'].astype(str) + '_' + go_annotations['go_term'].astype(str)\ngo_annotations = go_annotations[~go_annotations['pred_key'].isin(negative_keys)]\ngoa_pred_keys = set(go_annotations['pred_key'])\nprint(f\"[+] Total unique ground truth protein-GO pairs: {len(goa_pred_keys)}\")\nprint(f\"[✅] Done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:27.210562Z","iopub.execute_input":"2025-12-17T04:45:27.210909Z","iopub.status.idle":"2025-12-17T04:45:34.507306Z","shell.execute_reply.started":"2025-12-17T04:45:27.210879Z","shell.execute_reply":"2025-12-17T04:45:34.504599Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Post-processing Submissions","metadata":{}},{"cell_type":"code","source":"def load_submission(path, chunksize=50000, num_rows = None):\n    total = None\n    if num_rows:\n        total = int(num_rows / chunksize) + 1 \n    chunks = []\n    for chunk in tqdm(pd.read_csv(path, sep='\\t', header=None, chunksize=chunksize), total = total):\n        chunk['pred_key'] = chunk[0].astype(str) + '_' + chunk[1].astype(str)\n        chunks.append(chunk)\n    df = pd.concat(chunks, ignore_index=True)\n    df.columns = ['protein_id', 'go_term', 'score', 'pred_key']\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:34.508934Z","iopub.execute_input":"2025-12-17T04:45:34.509292Z","iopub.status.idle":"2025-12-17T04:45:34.522307Z","shell.execute_reply.started":"2025-12-17T04:45:34.509262Z","shell.execute_reply":"2025-12-17T04:45:34.517422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Loading submission ..\")\nsubmission = load_submission('/kaggle/input/cafa-6-submission/submission.tsv')\nsubmission.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:34.524201Z","iopub.execute_input":"2025-12-17T04:45:34.524584Z","iopub.status.idle":"2025-12-17T04:45:46.110520Z","shell.execute_reply.started":"2025-12-17T04:45:34.524552Z","shell.execute_reply":"2025-12-17T04:45:46.109252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(f\"[1/2] Removing Ground-Truth from submission ..\")\n# submission = submission[~submission.pred_key.isin(goa_pred_keys)]\n\n# print(f\"[2/2] Removing Negatives from submission ..\")\n# submission = submission[~submission.pred_key.isin(negative_keys)]\n# submission.drop(columns=['pred_key'], inplace=True)\n# submission.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:46.111251Z","iopub.execute_input":"2025-12-17T04:45:46.111595Z","iopub.status.idle":"2025-12-17T04:45:50.296721Z","shell.execute_reply.started":"2025-12-17T04:45:46.111572Z","shell.execute_reply":"2025-12-17T04:45:50.295221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'[⏳] Saving Submission ...')\nsubmission.to_csv('submission.tsv',sep='\\t', index=False, header=None)\nprint(f\"[✅] Done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:45:50.298974Z","iopub.execute_input":"2025-12-17T04:45:50.299542Z","iopub.status.idle":"2025-12-17T04:46:08.299935Z","shell.execute_reply.started":"2025-12-17T04:45:50.299508Z","shell.execute_reply":"2025-12-17T04:46:08.298346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submission.tsv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-17T04:46:08.301095Z","iopub.execute_input":"2025-12-17T04:46:08.301510Z","iopub.status.idle":"2025-12-17T04:46:08.538917Z","shell.execute_reply.started":"2025-12-17T04:46:08.301479Z","shell.execute_reply":"2025-12-17T04:46:08.537255Z"}},"outputs":[],"execution_count":null}]}