{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31239,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# An Exploratory Analysis of Test superset","metadata":{}},{"cell_type":"markdown","source":"<strong>What's this about:</strong>\n<div style=\"line-height:24px; font-size:16px\">\n    <ul style=\"list-style:circle\">\n        <li>Examine the overlap between train and test proteinss</li>\n        <li>Visualize the distribution of the three knowledge groups in the test superset</li>\n        <li>Visualize ontology coverage patterns among training proteins</li>\n    </ul>\n</div>","metadata":{"execution":{"iopub.execute_input":"2025-12-18T19:21:24.808453Z","iopub.status.busy":"2025-12-18T19:21:24.807869Z","iopub.status.idle":"2025-12-18T19:21:24.861535Z","shell.execute_reply":"2025-12-18T19:21:24.859149Z","shell.execute_reply.started":"2025-12-18T19:21:24.808397Z"}}},{"cell_type":"code","source":"!python -V","metadata":{"execution":{"iopub.status.busy":"2025-12-25T21:05:01.774739Z","iopub.execute_input":"2025-12-25T21:05:01.775052Z","iopub.status.idle":"2025-12-25T21:05:01.914723Z","shell.execute_reply.started":"2025-12-25T21:05:01.775021Z","shell.execute_reply":"2025-12-25T21:05:01.913326Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\nfrom typing import Iterator\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-12-25T21:05:04.971211Z","iopub.execute_input":"2025-12-25T21:05:04.971656Z","iopub.status.idle":"2025-12-25T21:05:08.464290Z","shell.execute_reply.started":"2025-12-25T21:05:04.971599Z","shell.execute_reply":"2025-12-25T21:05:08.463228Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CAFA6_DIR = Path(\"/kaggle/input/cafa-6-protein-function-prediction\")\nTRAIN_FASTA = CAFA6_DIR / \"Train\" / \"train_sequences.fasta\"\nTEST_FASTA = CAFA6_DIR / \"Test\" / \"testsuperset.fasta\"\nTRAIN_TERMS = CAFA6_DIR / \"Train\" / \"train_terms.tsv\"","metadata":{"execution":{"iopub.status.busy":"2025-12-25T21:05:09.767411Z","iopub.execute_input":"2025-12-25T21:05:09.767938Z","iopub.status.idle":"2025-12-25T21:05:09.773698Z","shell.execute_reply.started":"2025-12-25T21:05:09.767909Z","shell.execute_reply":"2025-12-25T21:05:09.772275Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FIG_DPI = 140\nsns.set_theme(style=\"whitegrid\", font_scale=1.1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T21:05:11.692502Z","iopub.execute_input":"2025-12-25T21:05:11.692862Z","iopub.status.idle":"2025-12-25T21:05:11.698285Z","shell.execute_reply.started":"2025-12-25T21:05:11.692835Z","shell.execute_reply":"2025-12-25T21:05:11.697192Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Reading fasta files helpers:","metadata":{}},{"cell_type":"code","source":"def parse_entry_id_from_header(header: str) -> str:\n    header = header.strip()\n    parts = header.split(\"|\")\n    if len(parts) >= 2 and parts[1]:\n        return parts[1]\n    first = header.split()[0] if header else \"\"\n    if first:\n        return first\n    raise ValueError(f\"Unexpected FASTA header format: '{header}'\")\n\ndef iter_fasta(fasta_path: str | Path) -> Iterator[tuple[str, str]]:\n    header, seq = None, []\n    with open(fasta_path) as f:\n        for line in f:\n            line = line.strip()\n            if not line:\n                continue\n            if line.startswith(\">\"):\n                if header is not None:\n                    entry_id = parse_entry_id_from_header(header)\n                    yield entry_id, \"\".join(seq)\n                header = line[1:].strip()\n                seq = []\n            else:\n                seq.append(line)\n        if header is not None:\n            entry_id = parse_entry_id_from_header(header)\n            yield entry_id, \"\".join(seq)\n\ndef read_fasta_entry_ids(fasta_path: str | Path) -> np.ndarray:\n    return np.fromiter(\n        (entry_id for entry_id, _ in iter_fasta(fasta_path)),\n        dtype=object\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T21:05:13.995849Z","iopub.execute_input":"2025-12-25T21:05:13.996480Z","iopub.status.idle":"2025-12-25T21:05:14.004658Z","shell.execute_reply.started":"2025-12-25T21:05:13.996452Z","shell.execute_reply":"2025-12-25T21:05:14.003587Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Helpers to read and process train terms:","metadata":{}},{"cell_type":"code","source":"def normalize_ont_label(values: pd.Series) -> pd.Series:\n    \"\"\"\n    CAFA5 and 6 use different ontology labels, this function normalizes them\n    \"\"\"\n    values_low = values.astype(str).str.strip().str.lower()\n    mapping = {\n        \"p\": \"BPO\", \"bpo\": \"BPO\", \"biological_process\": \"BPO\",\n        \"f\": \"MFO\", \"mfo\": \"MFO\", \"molecular_function\": \"MFO\",\n        \"c\": \"CCO\", \"cco\": \"CCO\", \"cellular_component\": \"CCO\",\n    }\n    mapped = values_low.map(mapping)\n    return mapped.fillna(values.astype(str).str.strip().str.upper())\n\n\ndef read_train_terms(train_terms_path: Path) -> pd.DataFrame:\n    \"\"\"\n    Read the train terms file and normalize the ontology labels\n    \"\"\"\n    train_terms = pd.read_csv(\n        train_terms_path, sep=\"\\t\", header=None,\n        names=[\"protein_id\", \"term_id\", \"ontology\"], dtype=str,\n    )\n    train_terms = train_terms[\n        train_terms[\"protein_id\"].ne(\"EntryID\")\n    ].copy()\n    \n    train_terms[\"protein_id\"] = train_terms[\"protein_id\"].str.strip()\n    train_terms[\"term_id\"] = train_terms[\"term_id\"].str.strip()\n    train_terms[\"ontology\"] = normalize_ont_label(train_terms[\"ontology\"])\n    return train_terms\n\n\ndef build_ontology_presence(train_terms: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"\n    Build a presence matrix for the ontologies in the train terms file\n    \"\"\"\n    presence = (\n        train_terms[[\"protein_id\", \"ontology\"]]\n        .drop_duplicates()\n        .assign(has_term=True)\n        .pivot(index=\"protein_id\", columns=\"ontology\", values=\"has_term\")\n        .notna()\n    )\n    return presence.reindex(columns=[\"MFO\", \"BPO\", \"CCO\"], fill_value=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T21:05:16.550001Z","iopub.execute_input":"2025-12-25T21:05:16.550357Z","iopub.status.idle":"2025-12-25T21:05:16.559227Z","shell.execute_reply.started":"2025-12-25T21:05:16.550333Z","shell.execute_reply":"2025-12-25T21:05:16.558259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ids = read_fasta_entry_ids(TRAIN_FASTA)\ntest_ids = read_fasta_entry_ids(TEST_FASTA)\n\ntrain_terms = read_train_terms(TRAIN_TERMS)\npresence = build_ontology_presence(train_terms)\n\ntrain_set = set(train_ids.tolist())\ntest_set = set(test_ids.tolist())\n\ntest_in_trainseq = np.array([pid in train_set for pid in test_ids], dtype=bool)\n\nprint(\"train sequences:\", len(train_ids))\nprint(\"test sequences:\", len(test_ids))\nprint(\"test ∩ train (sequence ids):\", int(test_in_trainseq.sum()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T21:05:20.542844Z","iopub.execute_input":"2025-12-25T21:05:20.543185Z","iopub.status.idle":"2025-12-25T21:05:24.602945Z","shell.execute_reply.started":"2025-12-25T21:05:20.543157Z","shell.execute_reply":"2025-12-25T21:05:24.601996Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"💡 Note: all train proteins are also present in test","metadata":{}},{"cell_type":"markdown","source":"No we classify all test proteins into 3 groups used in LB evaluation:\n- no_knowledge: The protein has no training annotations in any GO subontology (not in train)\n- limited_knowledge: The protein has training annotations, but none in the given GO subontology.\n- partial_knowledge: The protein has training annotations in the given GO subontology, and is therefore a candidate for partial-knowledge evaluation, where only newly added terms will be scored after excluding known annotations.","metadata":{}},{"cell_type":"code","source":"def classify_by_ontology(\n    test_ids: np.ndarray,\n    presence: pd.DataFrame,\n    ont: str\n) -> pd.Series:\n    test_index = pd.Index(test_ids, name=\"protein_id\")\n    has_any = test_index.isin(presence.index)\n    has_ont = np.zeros(test_index.size, dtype=bool)\n    common = test_index[has_any]\n    if common.size:\n        has_ont[has_any] = presence.loc[common, ont].to_numpy()\n\n    labels = pd.Series(\"no_knowledge\", index=test_index, dtype=object)\n    labels[has_any & (~has_ont)] = \"limited_knowledge\"\n    labels[has_ont] = \"partial_knowledge\"\n    return labels\n\n\nlabels_mfo = classify_by_ontology(test_ids, presence, \"MFO\")\nlabels_bpo = classify_by_ontology(test_ids, presence, \"BPO\")\nlabels_cco = classify_by_ontology(test_ids, presence, \"CCO\")\n\nsummary = pd.DataFrame({\n    \"protein_id\": test_ids,\n    \"MFO\": labels_mfo.values,\n    \"BPO\": labels_bpo.values,\n    \"CCO\": labels_cco.values,\n})\n\ngroups = [\"no_knowledge\", \"limited_knowledge\", \"partial_knowledge\"]\ncounts = pd.concat(\n    {\n        \"MFO\": labels_mfo.value_counts(),\n        \"BPO\": labels_bpo.value_counts(),\n        \"CCO\": labels_cco.value_counts(),\n    },\n    axis=1,\n).T.reindex(index=[\"MFO\", \"BPO\", \"CCO\"], columns=groups, fill_value=0).astype(int)\n\ncounts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T21:05:30.351892Z","iopub.execute_input":"2025-12-25T21:05:30.352235Z","iopub.status.idle":"2025-12-25T21:05:30.866018Z","shell.execute_reply.started":"2025-12-25T21:05:30.352198Z","shell.execute_reply":"2025-12-25T21:05:30.865171Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"💡 Note:\n- The number of no_knowledge proteins is identical across all subontologies, which is expected because this group consists simply of proteins that have no training annotations at all, independent of ontology.\n\n- The partial_knowledge group reflects the number of proteins present in the training data (= proteins with annotations in the given subontology)\n\n- limited_knowledge represents proteins that have annotations in other subontologies but lack any annotation in the subontology being evaluated.","metadata":{}},{"cell_type":"code","source":"order_groups = [\"no_knowledge\", \"limited_knowledge\", \"partial_knowledge\"]\npalette = {\n    \"no_knowledge\": \"#9aa0a6\",\n    \"limited_knowledge\": \"#fbbc04\",\n    \"partial_knowledge\": \"#34a853\",\n}\n\nont_list = [\"MFO\", \"BPO\", \"CCO\"]\nvalues_mat = counts.reindex(index=ont_list, columns=order_groups).to_numpy()\n\nfig, ax = plt.subplots(figsize=(5, 3), dpi=FIG_DPI)\nbottom = np.zeros(len(ont_list), dtype=float)\n\nfor group_idx, grp in enumerate(order_groups):\n    values = values_mat[:, group_idx].astype(float)\n    ax.bar(ont_list, values, bottom=bottom, color=palette[grp], label=grp)\n    bottom += values\n\nax.set_title(\n    \"CAFA6 test proteins grouped by training knowledge (per ontology)\",\n    fontsize=10\n)\nax.set_ylabel(\"Number of test proteins\", fontsize=8)\nax.tick_params(axis=\"both\", labelsize=8)\n\nax.legend(\n    title=None,\n    frameon=True,\n    fontsize=8,\n    loc=\"upper center\",\n    bbox_to_anchor=(0.5, -0.18),\n    ncol=3,\n)\n\nsns.despine(ax=ax)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T21:07:36.125575Z","iopub.execute_input":"2025-12-25T21:07:36.125921Z","iopub.status.idle":"2025-12-25T21:07:36.357359Z","shell.execute_reply.started":"2025-12-25T21:07:36.125896Z","shell.execute_reply":"2025-12-25T21:07:36.356490Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Below, I plot the distribution of ontology coverage among training proteins:\n","metadata":{}},{"cell_type":"code","source":"test_common = pd.Index(test_ids).intersection(presence.index)\ntrain_presence = presence.astype(int)\n\npattern = (\n    train_presence[\"MFO\"].astype(str)\n    + train_presence[\"BPO\"].astype(str)\n    + train_presence[\"CCO\"].astype(str)\n)\npattern_counts = pattern.value_counts().rename_axis(\"pattern\").reset_index(name=\"n\")\n\npattern_counts[\"MFO\"] = pattern_counts[\"pattern\"].str[0].astype(int)\npattern_counts[\"BPO\"] = pattern_counts[\"pattern\"].str[1].astype(int)\npattern_counts[\"CCO\"] = pattern_counts[\"pattern\"].str[2].astype(int)\n\nheat = pattern_counts.set_index([\"MFO\", \"BPO\", \"CCO\"])[\"n\"].sort_values(ascending=False)\n\nfig, ax = plt.subplots(figsize=(7.2, 4.6), dpi=FIG_DPI)\nmatrix = pattern_counts[[\"MFO\", \"BPO\", \"CCO\"]].to_numpy()\ny_labels = pattern_counts[\"pattern\"].to_numpy()\nsns.heatmap(matrix, cmap=\"Blues\", cbar=False, ax=ax, linewidths=0.5, linecolor=\"white\")\n\nfor i, n in enumerate(pattern_counts[\"n\"].to_numpy()):\n    ax.text(3.05, i + 0.5, f\"{int(n)}\", va=\"center\", ha=\"left\", fontsize=10)\n\nax.set_xticklabels([\"MFO\", \"BPO\", \"CCO\"], rotation=0)\nax.set_yticks([])\nax.set_title(\"Ontology coverage among training proteins (MFO/BPO/CCO)\")\nax.yaxis.label.set_visible(False)\nax.set_xlim(0, 3.6)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-25T21:07:58.222475Z","iopub.execute_input":"2025-12-25T21:07:58.222891Z","iopub.status.idle":"2025-12-25T21:07:58.578868Z","shell.execute_reply.started":"2025-12-25T21:07:58.222865Z","shell.execute_reply":"2025-12-25T21:07:58.578050Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"💡 Each row corresponds to a binary pattern across the three GO subontologies (dark blue = annotation present, light blue = annotation absent). The values on the right indicate the number of proteins following each annotation pattern. For example, the top row shows that 36,775 proteins have at least one annotation in all three subontologies.","metadata":{}}]}