{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"},{"sourceId":14173811,"sourceType":"datasetVersion","datasetId":9034760},{"sourceId":14187577,"sourceType":"datasetVersion","datasetId":9045748},{"sourceId":14207560,"sourceType":"datasetVersion","datasetId":9061916},{"sourceId":14219168,"sourceType":"datasetVersion","datasetId":9039112}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc\nfrom collections import defaultdict\n\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:43:49.776475Z","iopub.execute_input":"2025-12-28T16:43:49.776779Z","iopub.status.idle":"2025-12-28T16:43:52.261802Z","shell.execute_reply.started":"2025-12-28T16:43:49.776746Z","shell.execute_reply":"2025-12-28T16:43:52.260760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_train_terms(path):\n    mapping = defaultdict(list)\n    df = pd.read_csv(path, sep=\"\\t\", header=None, names=[\"protein\",\"go\",\"ont\"], dtype=str)\n    for _, r in tqdm(df.iterrows(), total=len(df)): \n        mapping[r.protein].append(r.go)\n    print(f\"[io] Read training annotations for {len(mapping)} proteins from {path}\")\n    return mapping\n\ndef parse_obo(go_obo_path):\n    parents = defaultdict(set)\n    children = defaultdict(set)\n    \n    if not os.path.exists(go_obo_path): \n        return parents, children\n        \n    with open(go_obo_path,\"r\") as f:\n        cur_id=None\n        for line in f:\n            line=line.strip()\n            if line==\"[Term]\": \n                cur_id=None\n            elif line.startswith(\"id: \"): \n                cur_id=line.split(\"id: \")[1].strip()\n            elif line.startswith(\"is_a: \"):\n                pid=line.split()[1].strip()\n                if cur_id: \n                    parents[cur_id].add(pid)\n                    children[pid].add(cur_id)\n            elif line.startswith(\"relationship: part_of \"):\n                parts=line.split(); \n                if len(parts)>=3:\n                    pid=parts[2].strip()\n                    if cur_id: \n                        parents[cur_id].add(pid)\n                        children[pid].add(cur_id)\n    print(f\"[io] Parsed OBO: {len(parents)} nodes with parents\")\n    return parents, children\n\ndef get_ancestors(go_id, parents):\n    ans=set()\n    stack=[go_id]\n    while stack:\n        cur=stack.pop()\n        for p in parents.get(cur,[]): \n            if p not in ans:\n                ans.add(p)\n                stack.append(p)\n    return ans\n\ndef get_descendants(go_id):\n    desc = set()\n    stack = [go_id]\n    while stack:\n        cur = stack.pop()\n        for child in children_map.get(cur, []):\n            if child not in desc:\n                desc.add(child)\n                stack.append(child)\n    return desc\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:43:52.263808Z","iopub.execute_input":"2025-12-28T16:43:52.264297Z","iopub.status.idle":"2025-12-28T16:43:52.276497Z","shell.execute_reply.started":"2025-12-28T16:43:52.264232Z","shell.execute_reply":"2025-12-28T16:43:52.275639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_terms = read_train_terms(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\")\nparents_map, children_map = parse_obo(\"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:43:52.277989Z","iopub.execute_input":"2025-12-28T16:43:52.278348Z","iopub.status.idle":"2025-12-28T16:44:23.123434Z","shell.execute_reply.started":"2025-12-28T16:43:52.278313Z","shell.execute_reply":"2025-12-28T16:44:23.122345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_ver227.csv')\ngo_annotations = go_annotations.drop_duplicates()\nprint(f'[+] Dataset shape: {go_annotations.shape}')\ngo_annotations.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:23.124480Z","iopub.execute_input":"2025-12-28T16:44:23.125281Z","iopub.status.idle":"2025-12-28T16:44:26.101555Z","shell.execute_reply.started":"2025-12-28T16:44:23.125224Z","shell.execute_reply":"2025-12-28T16:44:26.100649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations.qualifier.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:26.103116Z","iopub.execute_input":"2025-12-28T16:44:26.103459Z","iopub.status.idle":"2025-12-28T16:44:26.304012Z","shell.execute_reply.started":"2025-12-28T16:44:26.103430Z","shell.execute_reply":"2025-12-28T16:44:26.302800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_ver228.tsv', sep='\\t')\ngo_annotations = go_annotations.drop_duplicates()\nprint(f'[+] Dataset shape: {go_annotations.shape}')\ngo_annotations.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:26.305616Z","iopub.execute_input":"2025-12-28T16:44:26.306311Z","iopub.status.idle":"2025-12-28T16:44:29.777054Z","shell.execute_reply.started":"2025-12-28T16:44:26.306275Z","shell.execute_reply":"2025-12-28T16:44:29.776107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations.qualifier.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:29.779862Z","iopub.execute_input":"2025-12-28T16:44:29.780160Z","iopub.status.idle":"2025-12-28T16:44:29.975563Z","shell.execute_reply.started":"2025-12-28T16:44:29.780136Z","shell.execute_reply":"2025-12-28T16:44:29.974537Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<p style=\"background-color:#e6f7ff; \n          padding:15px; \n          color:#111;\n          font-size:16px;\n          border-width:3px; \n          border-color:#d0eefc; \n          border-style:solid;\n          border-radius:6px\">\n🧬 <code>NOT</code> qualifiers explicitly indicate that a <code>protein</code> is <em>not</em> associated with a given <code>GO term</code>. These annotations serve as reliable <code>negative examples</code>.<br>\nIf a protein is not linked to a specific <code>GO term</code>, it is also not linked to any of its <code>descendants</code> within the Gene Ontology hierarchy.<br>\nBy propagating these <code>negative annotations</code> through the ontology, we can reduce <code>false positives</code> and achieve a <strong>more accurate leaderboard score</strong>.\n</p>\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndef compare_csv_overlap(file1_path, file2_path):\n    df1 = pd.read_csv(file1_path, dtype=str)\n    df2 = pd.read_csv(file2_path, dtype=str, sep='\\t')\n\n    keys = ['protein_id', 'go_term', 'qualifier']\n\n    if not set(keys).issubset(df1.columns) or not set(keys).issubset(df2.columns):\n        return\n\n    set1 = df1[keys].drop_duplicates()\n    set2 = df2[keys].drop_duplicates()\n\n    common_rows = pd.merge(set1, set2, on=keys, how='inner')\n\n    count_common = len(common_rows)\n    count_file1 = len(set1)\n    count_file2 = len(set2)\n\n    print(f\"--- RESULTS ---\")\n    print(f\"Total unique in File 1: {count_file1}\")\n    print(f\"Total unique in File 2: {count_file2}\")\n    print(f\"---------------------------------\")\n    print(f\"Common count: {count_common}\")\n    \n    if count_file1 > 0:\n        print(f\"Rate in File 1: {(count_common / count_file1) * 100:.2f}%\")\n        print(f\"Rate in File 2: {(count_common / count_file2) * 100:.2f}%\")\n\ncompare_csv_overlap('/kaggle/input/protein-go-annotations/goa_uniprot_ver227.csv', '/kaggle/input/protein-go-annotations/goa_uniprot_ver228.tsv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:29.976548Z","iopub.execute_input":"2025-12-28T16:44:29.977151Z","iopub.status.idle":"2025-12-28T16:44:38.608741Z","shell.execute_reply.started":"2025-12-28T16:44:29.977116Z","shell.execute_reply":"2025-12-28T16:44:38.607812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"[1/3] Filtering Negative Annotations ..\")\nnegative_annots = go_annotations[go_annotations['qualifier'].str.contains('NOT', na=False)]\nnegative_annots = negative_annots.drop(columns=['qualifier']).drop_duplicates()\n\nprint(f\"[2/3] Propagate Negative Terms ..\")\nnegative_annots = negative_annots.groupby('protein_id')['go_term'].apply(list).to_dict()\n\npropagated={}\nfor p in tqdm(negative_annots.keys()):\n    terms=set(negative_annots[p])\n    extra=set()\n    for t in list(terms): \n        extra |= get_descendants(t)\n    propagated[p] = sorted(terms | extra)\n        \nnegative_annots = propagated\n\nprint(f\"[3/3] Extract Unique Keys ..\")\nrows = [(protein_id, go_term) for protein_id, terms in negative_annots.items() for go_term in terms]\nnegative_df = pd.DataFrame(rows, columns=[\"protein_id\", \"go_term\"])\nnegative_df['pred_key'] = negative_df.protein_id.apply(str) + '_' + negative_df.go_term.apply(str)\nnegative_keys = set(negative_df['pred_key'])\n\ndel negative_df\ngc.collect()\n\nprint(f\"Total unique negative protein-GO pairs: {len(negative_keys)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:38.609742Z","iopub.execute_input":"2025-12-28T16:44:38.610183Z","iopub.status.idle":"2025-12-28T16:44:40.039223Z","shell.execute_reply.started":"2025-12-28T16:44:38.610154Z","shell.execute_reply":"2025-12-28T16:44:40.037791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"[1/4] Loading GOA Annotations ..\")\ngo_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_ver227.csv')\n\nprint(f\"[2/4] Removing unwanted annotations ..\")\ngo_annotations = go_annotations[~go_annotations['qualifier'].str.contains('NOT', na=False)]\ngo_annotations.drop(columns=['qualifier'], inplace=True)\ngo_annotations = go_annotations.drop_duplicates()\n\nprint(f\"[3/4] Set Ground-Truth Score ..\")\ngo_annotations['score'] = round(1.0, 3)\n\nprint(f\"[4/4] Setting Key ..\")\ngo_annotations['pred_key'] = go_annotations['protein_id'].astype(str) + '_' + go_annotations['go_term'].astype(str)\ngo_annotations = go_annotations[~go_annotations['pred_key'].isin(negative_keys)]\ngoa_pred_keys = set(go_annotations['pred_key'])\nprint(f\"[+] Total unique ground truth protein-GO pairs: {len(goa_pred_keys)}\")\nprint(f\"[✅] Done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:40.040523Z","iopub.execute_input":"2025-12-28T16:44:40.041023Z","iopub.status.idle":"2025-12-28T16:44:45.309211Z","shell.execute_reply.started":"2025-12-28T16:44:40.040983Z","shell.execute_reply":"2025-12-28T16:44:45.308228Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Post-processing Submissions","metadata":{}},{"cell_type":"code","source":"def load_submission(path, chunksize=50000, num_rows = None):\n    total = None\n    if num_rows:\n        total = int(num_rows / chunksize) + 1 \n    chunks = []\n    for chunk in tqdm(pd.read_csv(path, sep='\\t', header=None, chunksize=chunksize), total = total):\n        chunk['pred_key'] = chunk[0].astype(str) + '_' + chunk[1].astype(str)\n        chunks.append(chunk)\n    df = pd.concat(chunks, ignore_index=True)\n    df.columns = ['protein_id', 'go_term', 'score', 'pred_key']\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:45.310138Z","iopub.execute_input":"2025-12-28T16:44:45.310436Z","iopub.status.idle":"2025-12-28T16:44:45.317626Z","shell.execute_reply.started":"2025-12-28T16:44:45.310406Z","shell.execute_reply":"2025-12-28T16:44:45.316372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Loading submission ..\")\nsubmission = load_submission('/kaggle/input/privatescore/submission.tsv')\nsubmission.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:44:45.318597Z","iopub.execute_input":"2025-12-28T16:44:45.318915Z","iopub.status.idle":"2025-12-28T16:45:29.050273Z","shell.execute_reply.started":"2025-12-28T16:44:45.318880Z","shell.execute_reply":"2025-12-28T16:45:29.049317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" print(f\"[1/2] Removing Ground-Truth from submission ..\")\n submission = submission[~submission.pred_key.isin(goa_pred_keys)]\n\n print(f\"[2/2] Removing Negatives from submission ..\")\n submission = submission[~submission.pred_key.isin(negative_keys)]\n submission.drop(columns=['pred_key'], inplace=True)\n submission.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:45:29.051313Z","iopub.execute_input":"2025-12-28T16:45:29.051707Z","iopub.status.idle":"2025-12-28T16:46:01.965747Z","shell.execute_reply.started":"2025-12-28T16:45:29.051669Z","shell.execute_reply":"2025-12-28T16:46:01.964780Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'[⏳] Saving Submission ...')\nsubmission.to_csv('submission.tsv',sep='\\t', index=False, header=None)\nprint(f\"[✅] Done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:46:01.966799Z","iopub.execute_input":"2025-12-28T16:46:01.967339Z","iopub.status.idle":"2025-12-28T16:47:47.415616Z","shell.execute_reply.started":"2025-12-28T16:46:01.967308Z","shell.execute_reply":"2025-12-28T16:47:47.414303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submission.tsv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-28T16:47:47.416531Z","iopub.execute_input":"2025-12-28T16:47:47.417138Z","iopub.status.idle":"2025-12-28T16:47:47.609433Z","shell.execute_reply.started":"2025-12-28T16:47:47.417110Z","shell.execute_reply":"2025-12-28T16:47:47.607764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}