{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13482672,"sourceType":"datasetVersion","datasetId":8559888},{"sourceId":13502493,"sourceType":"datasetVersion","datasetId":8572953},{"sourceId":13680623,"sourceType":"datasetVersion","datasetId":8699749},{"sourceId":14145354,"sourceType":"datasetVersion","datasetId":9014792},{"sourceId":14148649,"sourceType":"datasetVersion","datasetId":9017190}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 📦 Imports","metadata":{}},{"cell_type":"code","source":"import os, gc\nfrom collections import defaultdict\n\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:09.98545Z","iopub.execute_input":"2025-11-10T23:50:09.985776Z","iopub.status.idle":"2025-11-10T23:50:12.991996Z","shell.execute_reply.started":"2025-11-10T23:50:09.985751Z","shell.execute_reply":"2025-11-10T23:50:12.990218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"## ƒ Utility Functions","metadata":{}},{"cell_type":"code","source":"def read_train_terms(path):\n    mapping = defaultdict(list)\n    df = pd.read_csv(path, sep=\"\\t\", header=None, names=[\"protein\",\"go\",\"ont\"], dtype=str)\n    for _, r in tqdm(df.iterrows(), total=len(df)): \n        mapping[r.protein].append(r.go)\n    print(f\"[io] Read training annotations for {len(mapping)} proteins from {path}\")\n    return mapping\n\ndef parse_obo(go_obo_path):\n    parents = defaultdict(set)\n    children = defaultdict(set)\n    \n    if not os.path.exists(go_obo_path): \n        return parents, children\n        \n    with open(go_obo_path,\"r\") as f:\n        cur_id=None\n        for line in f:\n            line=line.strip()\n            if line==\"[Term]\": \n                cur_id=None\n            elif line.startswith(\"id: \"): \n                cur_id=line.split(\"id: \")[1].strip()\n            elif line.startswith(\"is_a: \"):\n                pid=line.split()[1].strip()\n                if cur_id: \n                    parents[cur_id].add(pid)\n                    children[pid].add(cur_id)\n            elif line.startswith(\"relationship: part_of \"):\n                parts=line.split(); \n                if len(parts)>=3:\n                    pid=parts[2].strip()\n                    if cur_id: \n                        parents[cur_id].add(pid)\n                        children[pid].add(cur_id)\n    print(f\"[io] Parsed OBO: {len(parents)} nodes with parents\")\n    return parents, children\n\ndef get_ancestors(go_id, parents):\n    ans=set()\n    stack=[go_id]\n    while stack:\n        cur=stack.pop()\n        for p in parents.get(cur,[]): \n            if p not in ans:\n                ans.add(p)\n                stack.append(p)\n    return ans\n\ndef get_descendants(go_id):\n    desc = set()\n    stack = [go_id]\n    while stack:\n        cur = stack.pop()\n        for child in children_map.get(cur, []):\n            if child not in desc:\n                desc.add(child)\n                stack.append(child)\n    return desc\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:12.994432Z","iopub.execute_input":"2025-11-10T23:50:12.994956Z","iopub.status.idle":"2025-11-10T23:50:13.010727Z","shell.execute_reply.started":"2025-11-10T23:50:12.994918Z","shell.execute_reply":"2025-11-10T23:50:13.009071Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"## 🥇 Competition Data","metadata":{}},{"cell_type":"code","source":"train_terms = read_train_terms(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\")\nparents_map, children_map = parse_obo(\"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:13.011844Z","iopub.execute_input":"2025-11-10T23:50:13.012127Z","iopub.status.idle":"2025-11-10T23:50:45.18491Z","shell.execute_reply.started":"2025-11-10T23:50:13.012103Z","shell.execute_reply":"2025-11-10T23:50:45.183598Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"## 🧬 Load GOA Uniprot Data","metadata":{}},{"cell_type":"code","source":"go_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_all.csv')\ngo_annotations = go_annotations.drop_duplicates()\nprint(f'[+] Dataset shape: {go_annotations.shape}')\ngo_annotations.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:45.186093Z","iopub.execute_input":"2025-11-10T23:50:45.186403Z","iopub.status.idle":"2025-11-10T23:50:48.755662Z","shell.execute_reply.started":"2025-11-10T23:50:45.186378Z","shell.execute_reply":"2025-11-10T23:50:48.75462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations.qualifier.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:48.758144Z","iopub.execute_input":"2025-11-10T23:50:48.758434Z","iopub.status.idle":"2025-11-10T23:50:48.959625Z","shell.execute_reply.started":"2025-11-10T23:50:48.758411Z","shell.execute_reply":"2025-11-10T23:50:48.958751Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<p style=\"background-color:#e6f7ff; \n          padding:15px; \n          color:#111;\n          font-size:16px;\n          border-width:3px; \n          border-color:#d0eefc; \n          border-style:solid;\n          border-radius:6px\">\n🧬 <code>NOT</code> qualifiers explicitly indicate that a <code>protein</code> is <em>not</em> associated with a given <code>GO term</code>. These annotations serve as reliable <code>negative examples</code>.<br>\nIf a protein is not linked to a specific <code>GO term</code>, it is also not linked to any of its <code>descendants</code> within the Gene Ontology hierarchy.<br>\nBy propagating these <code>negative annotations</code> through the ontology, we can reduce <code>false positives</code> and achieve a <strong>more accurate leaderboard score</strong>.\n</p>\n","metadata":{}},{"cell_type":"code","source":"print(f\"[1/3] Filtering Negative Annotations ..\")\nnegative_annots = go_annotations[go_annotations['qualifier'].str.contains('NOT', na=False)]\nnegative_annots = negative_annots.drop(columns=['qualifier']).drop_duplicates()\n\nprint(f\"[2/3] Propagate Negative Terms ..\")\nnegative_annots = negative_annots.groupby('protein_id')['go_term'].apply(list).to_dict()\n\npropagated={}\nfor p in tqdm(negative_annots.keys()):\n    terms=set(negative_annots[p])\n    extra=set()\n    for t in list(terms): \n        extra |= get_descendants(t)\n    propagated[p] = sorted(terms | extra)\n        \nnegative_annots = propagated\n\nprint(f\"[3/3] Extract Unique Keys ..\")\nrows = [(protein_id, go_term) for protein_id, terms in negative_annots.items() for go_term in terms]\nnegative_df = pd.DataFrame(rows, columns=[\"protein_id\", \"go_term\"])\nnegative_df['pred_key'] = negative_df.protein_id.apply(str) + '_' + negative_df.go_term.apply(str)\nnegative_keys = set(negative_df['pred_key'])\n\ndel negative_df\ngc.collect()\n\nprint(f\"Total unique negative protein-GO pairs: {len(negative_keys)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:48.960647Z","iopub.execute_input":"2025-11-10T23:50:48.961006Z","iopub.status.idle":"2025-11-10T23:50:50.65797Z","shell.execute_reply.started":"2025-11-10T23:50:48.960966Z","shell.execute_reply":"2025-11-10T23:50:50.656542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"[1/4] Loading GOA Annotations ..\")\ngo_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_all.csv')\n\nprint(f\"[2/4] Removing unwanted annotations ..\")\ngo_annotations = go_annotations[~go_annotations['qualifier'].str.contains('NOT', na=False)]\ngo_annotations.drop(columns=['qualifier'], inplace=True)\ngo_annotations = go_annotations.drop_duplicates()\n\nprint(f\"[3/4] Set Ground-Truth Score ..\")\ngo_annotations['score'] = round(1.0, 3)\n\nprint(f\"[4/4] Setting Key ..\")\ngo_annotations['pred_key'] = go_annotations['protein_id'].astype(str) + '_' + go_annotations['go_term'].astype(str)\ngo_annotations = go_annotations[~go_annotations['pred_key'].isin(negative_keys)]\ngoa_pred_keys = set(go_annotations['pred_key'])\nprint(f\"[+] Total unique ground truth protein-GO pairs: {len(goa_pred_keys)}\")\nprint(f\"[✅] Done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:50.659063Z","iopub.execute_input":"2025-11-10T23:50:50.660412Z","iopub.status.idle":"2025-11-10T23:50:56.738661Z","shell.execute_reply.started":"2025-11-10T23:50:50.660379Z","shell.execute_reply":"2025-11-10T23:50:56.737477Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"## ⏳ Load Submissions","metadata":{}},{"cell_type":"code","source":"def load_submission(path, chunksize=50000, num_rows = None):\n    total = None\n    if num_rows:\n        total = int(num_rows / chunksize) + 1 \n    chunks = []\n    for chunk in tqdm(pd.read_csv(path, sep='\\t', header=None, chunksize=chunksize), total = total):\n        chunk['pred_key'] = chunk[0].astype(str) + '_' + chunk[1].astype(str)\n        chunks.append(chunk)\n    df = pd.concat(chunks, ignore_index=True)\n    df.columns = ['protein_id', 'go_term', 'score', 'pred_key']\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:56.739818Z","iopub.execute_input":"2025-11-10T23:50:56.740126Z","iopub.status.idle":"2025-11-10T23:50:56.747589Z","shell.execute_reply.started":"2025-11-10T23:50:56.7401Z","shell.execute_reply":"2025-11-10T23:50:56.746115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!wc -l /kaggle/input/submiss/0235.tsv\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T06:07:38.198166Z","iopub.execute_input":"2025-12-14T06:07:38.198390Z","iopub.status.idle":"2025-12-14T06:07:42.653677Z","shell.execute_reply.started":"2025-12-14T06:07:38.198371Z","shell.execute_reply":"2025-12-14T06:07:42.652409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!wc -l /kaggle/input/submiss/0270.tsv\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T06:07:42.656524Z","iopub.execute_input":"2025-12-14T06:07:42.656804Z","iopub.status.idle":"2025-12-14T06:07:50.024512Z","shell.execute_reply.started":"2025-12-14T06:07:42.656776Z","shell.execute_reply":"2025-12-14T06:07:50.023338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"[1/2] Loading 1st submission ..\")\nA = load_submission('/kaggle/input/submiss/0235.tsv', num_rows = 21627761)\nA.drop(A.index[A['score'] < 0.04], inplace=True)\nA['score'] = A['score'].clip(upper=1.0)\n\nprint(f\"[2/2] Loading 2nd submission ..\")\nB = load_submission('/kaggle/input/submiss/0270.tsv', num_rows = 49797219 )\nB.dropna(inplace=True)\n\nA.shape, B.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:50:56.748421Z","iopub.execute_input":"2025-11-10T23:50:56.748662Z","iopub.status.idle":"2025-11-10T23:54:53.9933Z","shell.execute_reply.started":"2025-11-10T23:50:56.748644Z","shell.execute_reply":"2025-11-10T23:54:53.991734Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"##  🧹 Post-Processing","metadata":{}},{"cell_type":"code","source":"print(f\"[1/4] Removing Ground-Truth from A ..\")\nA = A[~A.pred_key.isin(goa_pred_keys)]\nprint(f\"[2/4] Removing Ground-Truth from B ..\")\nB = B[~B.pred_key.isin(goa_pred_keys)]\n\nprint(f\"[3/4] Removing Negatives from A ..\")\nA = A[~A.pred_key.isin(negative_keys)]\nprint(f\"[4/4] Removing Negatives from B ..\")\nB = B[~B.pred_key.isin(negative_keys)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:54:53.99472Z","iopub.execute_input":"2025-11-10T23:54:53.995118Z","iopub.status.idle":"2025-11-10T23:55:39.328674Z","shell.execute_reply.started":"2025-11-10T23:54:53.995084Z","shell.execute_reply":"2025-11-10T23:55:39.327564Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"## 🔗 Merge Submission","metadata":{}},{"cell_type":"code","source":"print(f\"[1/3] Intersection keys ..\")\nA_keys = set(A.pred_key)\nB_keys = set(B.pred_key)\nintersect_keys = A_keys & B_keys\n\n# Takes a while\nprint(f\"[2/3] Intersection ..\")\n# LeaderBoard Scores\nwa = 0.250\nwb = 0.211\n\nA_inter = A[A.pred_key.isin(intersect_keys)].copy()\nB_inter = B[B.pred_key.isin(intersect_keys)].copy()\n\ninter = A_inter.merge(\n    B_inter[['pred_key','score']],\n    on='pred_key',\n    suffixes=('_a','_b')\n)\n\n\nprint(f\"[3/3] Weighted average Sum ..\")\ninter['score'] = (inter['score_a'] * wa + inter['score_b'] * wb) / (wa + wb)\ninter.drop(columns = ['score_a', 'score_b'], inplace=True)\nprint(f\"[✅] Done.\")\ninter.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:55:39.329797Z","iopub.execute_input":"2025-11-10T23:55:39.330134Z","iopub.status.idle":"2025-11-10T23:57:32.594186Z","shell.execute_reply.started":"2025-11-10T23:55:39.330106Z","shell.execute_reply":"2025-11-10T23:57:32.593376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Takes a while\nprint(f\"[1/2] Add Missing Rows ..\")\nAnotB = A[~A.pred_key.isin(B_keys)]\nBnotA = B[~B.pred_key.isin(A_keys)]\n\n\nprint(f\"[2/2] Merging ..\")\nsubmission = pd.concat([go_annotations, inter, AnotB, BnotA], axis=0)\nsubmission.drop(columns=['pred_key'], inplace=True)\nsubmission.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:57:32.595185Z","iopub.execute_input":"2025-11-10T23:57:32.595446Z","iopub.status.idle":"2025-11-10T23:59:39.927253Z","shell.execute_reply.started":"2025-11-10T23:57:32.595424Z","shell.execute_reply":"2025-11-10T23:59:39.926257Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"## 📤 Submit","metadata":{}},{"cell_type":"code","source":"print(f'[⏳] Saving Submission ...')\nsubmission.to_csv('submission.tsv',sep='\\t', index=False, header=None)\nprint(f\"[✅] Done.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-10T23:59:39.928394Z","iopub.execute_input":"2025-11-10T23:59:39.928718Z","iopub.status.idle":"2025-11-11T00:00:47.133549Z","shell.execute_reply.started":"2025-11-10T23:59:39.928665Z","shell.execute_reply":"2025-11-11T00:00:47.132128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submission.tsv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-11T00:00:47.137083Z","iopub.execute_input":"2025-11-11T00:00:47.137384Z","iopub.status.idle":"2025-11-11T00:00:47.660103Z","shell.execute_reply.started":"2025-11-11T00:00:47.137361Z","shell.execute_reply":"2025-11-11T00:00:47.658923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}