{"cells":[{"cell_type":"markdown","metadata":{},"source":"# CAFA-6 Super Ensemble v2\n\n**v1 Score: 0.366** | Target: Beat 0.40+\n\n## Mejoras v2:\n1. **4 submissions** en lugar de 2\n2. **Parent-child constraint** (p_child <= p_parent)\n3. **Multi-agreement bonus** (+0.04 si 3+ modelos)\n4. **Adjusted normalization** (0.92 vs 0.95)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict\nfrom tqdm.auto import tqdm\nprint('v2 loaded!')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class CFG:\n    OBO_PATH = \"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\"\n    GOA_PATH = \"/kaggle/input/goa-uniprot-all/selection_ids_goa_uniprot_all.csv\"\n    \n    # 4 submissions\n    SUBMISSIONS = [\n        (\"/kaggle/input/cafa-6-blend-goa-negative-propagation/submission.tsv\", 0.250),\n        (\"/kaggle/input/merge-of-2submission-lb-0-25/submission.tsv\", 0.250),\n        (\"/kaggle/input/cafa-6-protein-function-blend/submission.tsv\", 0.220),\n        (\"/kaggle/input/cafa-6-protein-weighted-ensemble/submission.tsv\", 0.210),\n    ]\n    \n    DIVERSITY_BONUS = 0.02\n    MULTI_AGREEMENT_BONUS = 0.04\n    MIN_AGREEMENT = 2\n    SCORE_THRESHOLD = 0.001\n    NORMALIZE_MAX = 0.92\n    ROOTS = {'GO:0003674', 'GO:0008150', 'GO:0005575'}\n\nprint(f'{len(CFG.SUBMISSIONS)} submissions configured')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def parse_obo(obo_path):\n    parents = defaultdict(set)\n    children = defaultdict(set)\n    if not os.path.exists(obo_path):\n        return parents, children\n    with open(obo_path, 'r') as f:\n        cur_id = None\n        for line in f:\n            line = line.strip()\n            if line == '[Term]':\n                cur_id = None\n            elif line.startswith('id: '):\n                cur_id = line.split('id: ')[1].strip()\n            elif line.startswith('is_a: '):\n                pid = line.split()[1].strip()\n                if cur_id:\n                    parents[cur_id].add(pid)\n                    children[pid].add(cur_id)\n            elif line.startswith('relationship: part_of '):\n                parts = line.split()\n                if len(parts) >= 3:\n                    pid = parts[2].strip()\n                    if cur_id:\n                        parents[cur_id].add(pid)\n                        children[pid].add(cur_id)\n    print(f'[OBO] {len(parents)} terms')\n    return parents, children\n\ndef get_ancestors(term, parents_map, cache=None):\n    if cache is not None and term in cache:\n        return cache[term]\n    ancestors = set()\n    stack = [term]\n    while stack:\n        cur = stack.pop()\n        for p in parents_map.get(cur, []):\n            if p not in ancestors:\n                ancestors.add(p)\n                stack.append(p)\n    if cache is not None:\n        cache[term] = ancestors\n    return ancestors\n\ndef get_descendants(term, children_map, cache=None):\n    if cache is not None and term in cache:\n        return cache[term]\n    descendants = set()\n    stack = [term]\n    while stack:\n        cur = stack.pop()\n        for c in children_map.get(cur, []):\n            if c not in descendants:\n                descendants.add(c)\n                stack.append(c)\n    if cache is not None:\n        cache[term] = descendants\n    return descendants\n\ndef calculate_weights(lb_scores):\n    scores = np.array(lb_scores)\n    exp_scores = np.exp(scores * 15)\n    return exp_scores / exp_scores.sum()\n\ndef enforce_parent_child_constraint(terms_dict, parents_map, ancestor_cache):\n    \"\"\"p_child <= p_parent\"\"\"\n    term_depths = {t: len(get_ancestors(t, parents_map, ancestor_cache)) for t in terms_dict}\n    sorted_terms = sorted(terms_dict.keys(), key=lambda t: term_depths.get(t, 0), reverse=True)\n    for term in sorted_terms:\n        child_score = terms_dict[term]\n        for parent in parents_map.get(term, []):\n            if parent in terms_dict and terms_dict[parent] < child_score:\n                terms_dict[parent] = child_score\n    return terms_dict\n\nprint('Functions defined!')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"print('=' * 60)\nprint('[1/7] Loading GO Ontology...')\nparents_map, children_map = parse_obo(CFG.OBO_PATH)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"print('\\n[2/7] Loading GOA...')\nnegative_keys = set()\npositive_df = None\n\nif os.path.exists(CFG.GOA_PATH):\n    goa = pd.read_csv(CFG.GOA_PATH)\n    pid_col = 'db_object_id' if 'db_object_id' in goa.columns else 'protein_id'\n    term_col = 'go_id' if 'go_id' in goa.columns else 'go_term'\n    \n    if 'qualifier' in goa.columns:\n        neg_goa = goa[goa['qualifier'].str.contains('NOT', na=False)]\n        desc_cache = {}\n        neg_expanded = []\n        for _, row in tqdm(neg_goa.iterrows(), total=len(neg_goa), desc='Neg prop'):\n            pid, term = str(row[pid_col]), str(row[term_col])\n            if pid and term and term != 'nan':\n                neg_expanded.append((pid, term))\n                for desc in get_descendants(term, children_map, desc_cache):\n                    neg_expanded.append((pid, desc))\n        negative_keys = {f'{p}_{t}' for p, t in neg_expanded}\n        print(f'[GOA-] {len(negative_keys):,}')\n    \n    pos_goa = goa[~goa['qualifier'].str.contains('NOT', na=False)] if 'qualifier' in goa.columns else goa\n    positive_df = pd.DataFrame({\n        'protein_id': pos_goa[pid_col].astype(str),\n        'go_term': pos_goa[term_col].astype(str),\n        'score': 1.0\n    }).drop_duplicates()\n    positive_df = positive_df[positive_df['go_term'] != 'nan']\n    print(f'[GOA+] {len(positive_df):,}')\n    del goa; gc.collect()\nelse:\n    print('GOA not found')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"print('\\n[3/7] Loading submissions...')\nprint('Available inputs:')\nfor d in os.listdir('/kaggle/input'):\n    print(f'  {d}')\n\nvalid_subs = [(p, s) for p, s in CFG.SUBMISSIONS if os.path.exists(p)]\nprint(f'\\nFound {len(valid_subs)}/{len(CFG.SUBMISSIONS)}')\n\nlb_scores = [s for _, s in valid_subs]\nweights = calculate_weights(lb_scores)\n\nsubmissions = []\nfor (path, lb_score), weight in zip(valid_subs, weights):\n    df = pd.read_csv(path, sep='\\t', header=None, names=['protein_id', 'go_term', 'score'])\n    df['protein_id'] = df['protein_id'].astype(str)\n    df['go_term'] = df['go_term'].astype(str)\n    df['score'] = pd.to_numeric(df['score'], errors='coerce').fillna(0).clip(0, 1.0)\n    df['weight'] = weight\n    print(f'  {os.path.basename(os.path.dirname(path))}: {len(df):,}, w={weight:.3f}')\n    submissions.append(df)\n\nprint(f'Total: {sum(len(s) for s in submissions):,}')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"print('\\n[4/7] Weighted ensemble...')\nall_preds = pd.concat(submissions, ignore_index=True)\nall_preds['key'] = all_preds['protein_id'] + '_' + all_preds['go_term']\n\nif negative_keys:\n    before = len(all_preds)\n    all_preds = all_preds[~all_preds['key'].isin(negative_keys)]\n    print(f'Removed {before - len(all_preds):,} negatives')\n\nall_preds['weighted_score'] = all_preds['score'] * all_preds['weight']\nagg = all_preds.groupby(['protein_id', 'go_term']).agg({\n    'weighted_score': 'sum',\n    'weight': 'sum',\n    'score': 'count'\n}).reset_index()\nagg.columns = ['protein_id', 'go_term', 'weighted_sum', 'weight_sum', 'n_models']\nagg['score'] = agg['weighted_sum'] / agg['weight_sum']\n\n# Diversity bonuses\nagg.loc[agg['n_models'] >= CFG.MIN_AGREEMENT, 'score'] += CFG.DIVERSITY_BONUS\nagg.loc[agg['n_models'] >= 3, 'score'] += CFG.MULTI_AGREEMENT_BONUS\nagg['score'] = agg['score'].clip(0, 1.0)\n\nensemble = agg[['protein_id', 'go_term', 'score']].copy()\nprint(f'Ensemble: {len(ensemble):,}')\nprint(f'  2+ agree: {(agg[\"n_models\"] >= 2).sum():,}')\nprint(f'  3+ agree: {(agg[\"n_models\"] >= 3).sum():,}')\nprint(f'  4 agree: {(agg[\"n_models\"] >= 4).sum():,}')\n\ndel all_preds, submissions, agg; gc.collect()"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"print('\\n[5/7] GO propagation + parent-child constraint...')\nancestor_cache = {}\npropagated_rows = []\n\nprotein_preds = ensemble.groupby('protein_id').apply(\n    lambda g: dict(zip(g['go_term'], g['score']))\n).to_dict()\n\nfor pid, terms_dict in tqdm(protein_preds.items(), desc='Processing'):\n    final_scores = terms_dict.copy()\n    \n    # Propagate to ancestors\n    for term, score in list(terms_dict.items()):\n        for ancestor in get_ancestors(term, parents_map, ancestor_cache):\n            final_scores[ancestor] = max(final_scores.get(ancestor, 0), score)\n    \n    # Parent-child constraint\n    final_scores = enforce_parent_child_constraint(final_scores, parents_map, ancestor_cache)\n    \n    # Force roots\n    for root in CFG.ROOTS:\n        final_scores[root] = 1.0\n    \n    # Normalization\n    max_non_root = max((s for t, s in final_scores.items() if t not in CFG.ROOTS), default=0)\n    if 0 < max_non_root < CFG.NORMALIZE_MAX:\n        scale = CFG.NORMALIZE_MAX / max_non_root\n        for t in final_scores:\n            if t not in CFG.ROOTS:\n                final_scores[t] = min(1.0, final_scores[t] * scale)\n    \n    for term, score in final_scores.items():\n        if score >= CFG.SCORE_THRESHOLD:\n            propagated_rows.append((pid, term, round(score, 6)))\n\nfinal_df = pd.DataFrame(propagated_rows, columns=['protein_id', 'go_term', 'score'])\nprint(f'Propagated: {len(final_df):,}')\n\ndel ensemble, protein_preds; gc.collect()"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"print('\\n[6/7] Inject GOA positives...')\nif positive_df is not None and len(positive_df) > 0:\n    test_proteins = set(final_df['protein_id'].unique())\n    positive_relevant = positive_df[positive_df['protein_id'].isin(test_proteins)]\n    before = len(final_df)\n    final_df = pd.concat([final_df, positive_relevant], ignore_index=True)\n    final_df = final_df.groupby(['protein_id', 'go_term'], as_index=False)['score'].max()\n    print(f'GOA+: {len(final_df):,} (delta: {len(final_df) - before:,})')"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"print('\\n[7/7] Saving...')\nfinal_df = final_df.sort_values(['protein_id', 'score'], ascending=[True, False])\nfinal_df.to_csv('submission.tsv', sep='\\t', index=False, header=False)\n\nprint(f'\\n{\"=\" * 60}')\nprint('DONE - Super Ensemble v2')\nprint(f'Predictions: {len(final_df):,}')\nprint(f'Proteins: {final_df[\"protein_id\"].nunique():,}')\nprint(f'GO terms: {final_df[\"go_term\"].nunique():,}')\nprint(f'{\"=\" * 60}')"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":4}