{"cells": [{"cell_type": "markdown", "metadata": {}, "source": "# CAFA-6 Super Ensemble v14 - Three Sources\n\n**Historial:** v10=0.373, v11=0.368, v12=0.373, v13=0.371\n\n**v14 = 3 fuentes diversas:**\n1. quick-submission (0.358)\n2. goa-negative (0.250)\n3. proven-ensemble (0.358)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "import os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nprint('v14 - Three Sources!')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "class CFG:\n    OBO_PATH = \"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\"\n    \n    # v14: 3 fuentes diversas\n    SUBMISSIONS = [\n        (\"/kaggle/input/cafa-quick-submission-0-358-score/submission.tsv\", 0.358),\n        (\"/kaggle/input/cafa-6-blend-goa-negative-propagation/submission.tsv\", 0.250),\n        (\"/kaggle/input/cafa-6-proven-ensemble-submission/submission.tsv\", 0.358),\n    ]\n    \n    # CAFA5 technique\n    LOW_SCORE_CLIP = 0.10\n    \n    # NO bonuses (como v10)\n    DIVERSITY_BONUS = 0.0\n    MULTI_AGREEMENT_BONUS = 0.0\n    \n    # Standard threshold\n    SCORE_THRESHOLD = 0.10\n    \n    ROOTS = {'GO:0003674', 'GO:0008150', 'GO:0005575'}\n\nprint(f'{len(CFG.SUBMISSIONS)} submissions')\nprint('v14: Three diverse sources')\nprint(f'  - LOW_SCORE_CLIP: {CFG.LOW_SCORE_CLIP}')\nprint(f'  - SCORE_THRESHOLD: {CFG.SCORE_THRESHOLD}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "def parse_obo(obo_path):\n    parents = defaultdict(set)\n    if not os.path.exists(obo_path):\n        return parents\n    with open(obo_path, 'r') as f:\n        cur_id = None\n        for line in f:\n            line = line.strip()\n            if line == '[Term]':\n                cur_id = None\n            elif line.startswith('id: '):\n                cur_id = line.split('id: ')[1].strip()\n            elif line.startswith('is_a: '):\n                pid = line.split()[1].strip()\n                if cur_id:\n                    parents[cur_id].add(pid)\n            elif line.startswith('relationship: part_of '):\n                parts = line.split()\n                if len(parts) >= 3:\n                    pid = parts[2].strip()\n                    if cur_id:\n                        parents[cur_id].add(pid)\n    print(f'[OBO] {len(parents)} terms')\n    return parents\n\ndef get_ancestors(term, parents_map, cache):\n    if term in cache:\n        return cache[term]\n    ancestors = set()\n    stack = [term]\n    while stack:\n        cur = stack.pop()\n        for p in parents_map.get(cur, []):\n            if p not in ancestors:\n                ancestors.add(p)\n                stack.append(p)\n    cache[term] = ancestors\n    return ancestors\n\ndef calculate_weights(lb_scores):\n    scores = np.array(lb_scores)\n    exp_scores = np.exp(scores * 10)\n    weights = exp_scores / exp_scores.sum()\n    return weights\n\nprint('Functions defined!')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "print('[1/5] Loading GO Ontology...')\nparents_map = parse_obo(CFG.OBO_PATH)"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "print('\\n[2/5] Loading submissions...')\nprint('Available inputs:')\nfor d in os.listdir('/kaggle/input'):\n    print(f'  {d}')\n\nvalid_subs = [(p, s) for p, s in CFG.SUBMISSIONS if os.path.exists(p)]\nprint(f'\\nFound {len(valid_subs)}/{len(CFG.SUBMISSIONS)}')\n\nif not valid_subs:\n    raise ValueError('No submissions!')\n\nlb_scores = [s for _, s in valid_subs]\nweights = calculate_weights(lb_scores)\n\nprint(f'\\nWeights based on LB scores:')\nfor (path, lb), w in zip(valid_subs, weights):\n    print(f'  {os.path.basename(os.path.dirname(path))}: LB={lb:.3f} -> weight={w:.3f}')\n\nsubmissions = []\nfor idx, ((path, lb_score), weight) in enumerate(zip(valid_subs, weights)):\n    print(f'\\nLoading {idx+1}/{len(valid_subs)}: {os.path.basename(os.path.dirname(path))}')\n    df = pd.read_csv(path, sep='\\t', header=None, names=['protein_id', 'go_term', 'score'])\n    df['protein_id'] = df['protein_id'].astype(str)\n    df['go_term'] = df['go_term'].astype(str)\n    df['score'] = pd.to_numeric(df['score'], errors='coerce').fillna(0).clip(0, 1.0)\n    \n    # CAFA5 TECHNIQUE: Clip low scores to 0 (ProtBoost winner)\n    before_clip = len(df)\n    df.loc[df['score'] < CFG.LOW_SCORE_CLIP, 'score'] = 0\n    df = df[df['score'] > 0]  # Remove zeros\n    after_clip = len(df)\n    print(f'  After clipping < {CFG.LOW_SCORE_CLIP}: {before_clip:,} -> {after_clip:,} ({before_clip-after_clip:,} removed)')\n    \n    df['weight'] = weight\n    df['source_idx'] = idx\n    print(f'  Rows: {len(df):,}, LB: {lb_score:.3f}, Weight: {weight:.3f}')\n    submissions.append(df)\n    \nprint(f'\\nTotal rows loaded: {sum(len(s) for s in submissions):,}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "print('\\n[3/5] Computing weighted ensemble (no bonuses, like v10)...')\n\nall_preds = pd.concat(submissions, ignore_index=True)\nall_preds['weighted_score'] = all_preds['score'] * all_preds['weight']\n\nagg = all_preds.groupby(['protein_id', 'go_term']).agg({\n    'weighted_score': 'sum',\n    'weight': 'sum',\n    'source_idx': 'nunique'\n}).reset_index()\n\nagg.columns = ['protein_id', 'go_term', 'weighted_sum', 'weight_sum', 'n_models']\nagg['score'] = agg['weighted_sum'] / agg['weight_sum']\n\n# NO BONUSES (v10 style)\nagg['score'] = agg['score'].clip(0, 1.0)\n\nensemble = agg[['protein_id', 'go_term', 'score']].copy()\nprint(f'Ensemble: {len(ensemble):,} predictions')\nprint(f'  2 models agree: {(agg[\"n_models\"] == 2).sum():,}')\nprint(f'  Single model: {(agg[\"n_models\"] == 1).sum():,}')\n\ndel all_preds, agg, submissions\ngc.collect()"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "print(f'\\n[4/5] GO propagation (fast, no consistency check)...')\nancestor_cache = {}\npropagated_rows = []\n\nprotein_groups = ensemble.groupby('protein_id')\nn_proteins = len(protein_groups)\n\nfor i, (pid, group) in enumerate(protein_groups):\n    if i % 50000 == 0:\n        print(f'  {i:,}/{n_proteins:,} proteins...')\n    \n    terms_dict = dict(zip(group['go_term'], group['score']))\n    final_scores = terms_dict.copy()\n    \n    # Propagate to ancestors\n    for term, score in list(terms_dict.items()):\n        for ancestor in get_ancestors(term, parents_map, ancestor_cache):\n            final_scores[ancestor] = max(final_scores.get(ancestor, 0), score)\n    \n    # Set roots to 1.0\n    for root in CFG.ROOTS:\n        final_scores[root] = 1.0\n    \n    # NO normalization (v10 style)\n    \n    # Collect results\n    for term, score in final_scores.items():\n        if score >= CFG.SCORE_THRESHOLD:\n            propagated_rows.append((pid, term, round(score, 6)))\n\ndel ensemble\ngc.collect()\n\nfinal_df = pd.DataFrame(propagated_rows, columns=['protein_id', 'go_term', 'score'])\ndel propagated_rows\ngc.collect()\n\nprint(f'Propagated: {len(final_df):,}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "print('\\n[5/5] Saving...')\nfinal_df = final_df.sort_values(['protein_id', 'score'], ascending=[True, False])\nfinal_df.to_csv('submission.tsv', sep='\\t', index=False, header=False)\n\nprint(f'\\n{\"=\" * 60}')\nprint('DONE - Super Ensemble v14 (Three Sources)')\nprint(f'Predictions: {len(final_df):,}')\nprint(f'Proteins: {final_df[\"protein_id\"].nunique():,}')\nprint(f'{\"=\" * 60}')\nprint('\\nv14 Config:')\nprint(f'  - Sources: 3 kernels (0.358 + 0.250 + 0.358)')\nprint(f'  - LOW_SCORE_CLIP: {CFG.LOW_SCORE_CLIP}')\nprint(f'  - SCORE_THRESHOLD: {CFG.SCORE_THRESHOLD}')\nprint(f'  - NO bonuses')"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.10.0"}}, "nbformat": 4, "nbformat_minor": 4}