{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"},{"sourceId":13482672,"sourceType":"datasetVersion","datasetId":8559888},{"sourceId":13502493,"sourceType":"datasetVersion","datasetId":8572953},{"sourceId":13680623,"sourceType":"datasetVersion","datasetId":8699749},{"sourceId":282897090,"sourceType":"kernelVersion"},{"sourceId":283250975,"sourceType":"kernelVersion"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#CELL1\nimport os, gc\nfrom collections import defaultdict\n\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport numpy as np\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import StandardScaler\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:34:46.898839Z","iopub.execute_input":"2025-12-12T00:34:46.899035Z","iopub.status.idle":"2025-12-12T00:34:49.803238Z","shell.execute_reply.started":"2025-12-12T00:34:46.899017Z","shell.execute_reply":"2025-12-12T00:34:49.802494Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"code","source":"#CELL2\ndef read_train_terms(path):\n    mapping = defaultdict(list)\n    df = pd.read_csv(path, sep=\"\\t\", header=None, names=[\"protein\",\"go\",\"ont\"], dtype=str)\n    for _, r in tqdm(df.iterrows(), total=len(df)): \n        mapping[r.protein].append(r.go)\n    print(f\"[io] Read training annotations for {len(mapping)} proteins from {path}\")\n    return mapping\n\ndef parse_obo(go_obo_path):\n    parents = defaultdict(set)\n    children = defaultdict(set)\n    \n    if not os.path.exists(go_obo_path): \n        return parents, children\n        \n    with open(go_obo_path,\"r\") as f:\n        cur_id=None\n        for line in f:\n            line=line.strip()\n            if line==\"[Term]\": \n                cur_id=None\n            elif line.startswith(\"id: \"): \n                cur_id=line.split(\"id: \")[1].strip()\n            elif line.startswith(\"is_a: \"):\n                pid=line.split()[1].strip()\n                if cur_id: \n                    parents[cur_id].add(pid)\n                    children[pid].add(cur_id)\n            elif line.startswith(\"relationship: part_of \"):\n                parts=line.split(); \n                if len(parts)>=3:\n                    pid=parts[2].strip()\n                    if cur_id: \n                        parents[cur_id].add(pid)\n                        children[pid].add(cur_id)\n    print(f\"[io] Parsed OBO: {len(parents)} nodes with parents\")\n    return parents, children\n\ndef get_ancestors(go_id, parents):\n    ans=set()\n    stack=[go_id]\n    while stack:\n        cur=stack.pop()\n        for p in parents.get(cur,[]): \n            if p not in ans:\n                ans.add(p)\n                stack.append(p)\n    return ans\n\ndef get_descendants(go_id, children_map):\n    desc = set()\n    stack = [go_id]\n    while stack:\n        cur = stack.pop()\n        for child in children_map.get(cur, []):\n            if child not in desc:\n                desc.add(child)\n                stack.append(child)\n    return desc\n\ndef compute_go_depth(go_term, parents_map, cache={}):\n    \"\"\"Compute depth of GO term in hierarchy\"\"\"\n    if go_term in cache:\n        return cache[go_term]\n    \n    if go_term not in parents_map or len(parents_map[go_term]) == 0:\n        cache[go_term] = 0\n        return 0\n    \n    max_depth = 0\n    for parent in parents_map[go_term]:\n        depth = compute_go_depth(parent, parents_map, cache) + 1\n        max_depth = max(max_depth, depth)\n    \n    cache[go_term] = max_depth\n    return max_depth\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:34:49.804986Z","iopub.execute_input":"2025-12-12T00:34:49.805411Z","iopub.status.idle":"2025-12-12T00:34:49.815924Z","shell.execute_reply.started":"2025-12-12T00:34:49.805385Z","shell.execute_reply":"2025-12-12T00:34:49.815157Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"code","source":"print(\"=\"*80)\nprint(\"CAFA-6 STACKING META-LEARNER PIPELINE\")\nprint(\"=\"*80)\n\ntrain_terms = read_train_terms(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\")\nparents_map, children_map = parse_obo(\"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:34:49.816768Z","iopub.execute_input":"2025-12-12T00:34:49.817021Z","iopub.status.idle":"2025-12-12T00:35:13.933142Z","shell.execute_reply.started":"2025-12-12T00:34:49.816996Z","shell.execute_reply":"2025-12-12T00:35:13.932314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"code","source":"#CELL3\nprint(\"\\n[Computing GO Statistics for Meta-Features]\")\n\n# Compute depth for all GO terms\nprint(\"Computing GO depths...\")\ngo_depths = {}\nall_go_terms = set(parents_map.keys()) | set(children_map.keys())\nfor go_term in tqdm(all_go_terms, desc=\"GO depths\"):\n    go_depths[go_term] = compute_go_depth(go_term, parents_map)\n\n# Compute specificity (number of descendants)\nprint(\"Computing GO specificity...\")\ngo_specificity = {}\nfor go_term in tqdm(all_go_terms, desc=\"GO specificity\"):\n    go_specificity[go_term] = len(get_descendants(go_term, children_map))\n\n# Compute GO term frequency in training data\nprint(\"Computing GO term frequency...\")\ngo_frequency = defaultdict(int)\nfor protein, terms in train_terms.items():\n    for term in terms:\n        go_frequency[term] += 1\n\ntotal_annotations = sum(go_frequency.values())\ngo_frequency_norm = {k: v/total_annotations for k, v in go_frequency.items()}\n\nprint(f\"[✓] GO Statistics computed\")\nprint(f\"    - GO terms with depth: {len(go_depths)}\")\nprint(f\"    - GO terms with specificity: {len(go_specificity)}\")\nprint(f\"    - GO terms with frequency: {len(go_frequency)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:35:13.934042Z","iopub.execute_input":"2025-12-12T00:35:13.934628Z","iopub.status.idle":"2025-12-12T00:35:14.421423Z","shell.execute_reply.started":"2025-12-12T00:35:13.934600Z","shell.execute_reply":"2025-12-12T00:35:14.420715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"go_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_all.csv')\ngo_annotations = go_annotations.drop_duplicates()\nprint(f'[+] Dataset shape: {go_annotations.shape}')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:35:14.422107Z","iopub.execute_input":"2025-12-12T00:35:14.422281Z","iopub.status.idle":"2025-12-12T00:35:16.626273Z","shell.execute_reply.started":"2025-12-12T00:35:14.422266Z","shell.execute_reply":"2025-12-12T00:35:16.625486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\\n[Processing Negative Annotations]\")\nnegative_annots = go_annotations[go_annotations['qualifier'].str.contains('NOT', na=False)]\nnegative_annots = negative_annots.drop(columns=['qualifier']).drop_duplicates()\nnegative_annots = negative_annots.groupby('protein_id')['go_term'].apply(list).to_dict()\n\npropagated={}\nfor p in tqdm(negative_annots.keys(), desc=\"Propagating negatives\"):\n    terms=set(negative_annots[p])\n    extra=set()\n    for t in list(terms): \n        extra |= get_descendants(t, children_map)\n    propagated[p] = sorted(terms | extra)\n        \nnegative_annots = propagated\n\nrows = [(protein_id, go_term) for protein_id, terms in negative_annots.items() for go_term in terms]\nnegative_df = pd.DataFrame(rows, columns=[\"protein_id\", \"go_term\"])\nnegative_df['pred_key'] = negative_df.protein_id.apply(str) + '_' + negative_df.go_term.apply(str)\nnegative_keys = set(negative_df['pred_key'])\n\ndel negative_df\ngc.collect()\n\nprint(f\"Total unique negative protein-GO pairs: {len(negative_keys)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:35:16.627100Z","iopub.execute_input":"2025-12-12T00:35:16.627398Z","iopub.status.idle":"2025-12-12T00:35:17.825110Z","shell.execute_reply.started":"2025-12-12T00:35:16.627376Z","shell.execute_reply":"2025-12-12T00:35:17.824365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\\n[Creating Ground Truth]\")\ngo_annotations = pd.read_csv('/kaggle/input/protein-go-annotations/goa_uniprot_all.csv')\ngo_annotations = go_annotations[~go_annotations['qualifier'].str.contains('NOT', na=False)]\ngo_annotations.drop(columns=['qualifier'], inplace=True)\ngo_annotations = go_annotations.drop_duplicates()\ngo_annotations['score'] = round(1.0, 3)\ngo_annotations['pred_key'] = go_annotations['protein_id'].astype(str) + '_' + go_annotations['go_term'].astype(str)\ngo_annotations = go_annotations[~go_annotations['pred_key'].isin(negative_keys)]\ngoa_pred_keys = set(go_annotations['pred_key'])\n\nprint(f\"Ground truth pairs: {len(goa_pred_keys)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:35:17.827059Z","iopub.execute_input":"2025-12-12T00:35:17.827316Z","iopub.status.idle":"2025-12-12T00:35:22.193238Z","shell.execute_reply.started":"2025-12-12T00:35:17.827299Z","shell.execute_reply":"2025-12-12T00:35:22.192588Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"code","source":"def load_submission(path, chunksize=50000, num_rows=None):\n    total = None\n    if num_rows:\n        total = int(num_rows / chunksize) + 1 \n    chunks = []\n    for chunk in tqdm(pd.read_csv(path, sep='\\t', header=None, chunksize=chunksize), total=total):\n        chunk['pred_key'] = chunk[0].astype(str) + '_' + chunk[1].astype(str)\n        chunks.append(chunk)\n    df = pd.concat(chunks, ignore_index=True)\n    df.columns = ['protein_id', 'go_term', 'score', 'pred_key']\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:35:22.193931Z","iopub.execute_input":"2025-12-12T00:35:22.194176Z","iopub.status.idle":"2025-12-12T00:35:22.199368Z","shell.execute_reply.started":"2025-12-12T00:35:22.194158Z","shell.execute_reply":"2025-12-12T00:35:22.198617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\\n[Loading Base Model Predictions]\")\nprint(f\"Model A...\")\nA = load_submission('/kaggle/input/cafa-6-protein-function-starter-eda-model/submission.tsv')\n\nprint(f\"Model B...\")\nB = load_submission('/kaggle/input/k/daovanda2405/cafa-6-protein-function-starter-eda-model/submission.tsv')\n\nprint(f\"A shape: {A.shape}, B shape: {B.shape}\")\n\n# Remove ground truth\nA = A[~A.pred_key.isin(goa_pred_keys)]\nB = B[~B.pred_key.isin(goa_pred_keys)]\n\n# Soft negative filtering\nprint(f\"\\n[Soft Negative Filtering]\")\nA_neg_mask = A['pred_key'].isin(negative_keys)\nB_neg_mask = B['pred_key'].isin(negative_keys)\nA.loc[A_neg_mask, 'score'] *= 0.1\nB.loc[B_neg_mask, 'score'] *= 0.1\nA = A[A['score'] >= 0.001]\nB = B[B['score'] >= 0.001]\n\nprint(f\"After filtering - A: {A.shape}, B: {B.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:35:22.199990Z","iopub.execute_input":"2025-12-12T00:35:22.200191Z","iopub.status.idle":"2025-12-12T00:36:45.104801Z","shell.execute_reply.started":"2025-12-12T00:35:22.200177Z","shell.execute_reply":"2025-12-12T00:36:45.104064Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"markdown","source":"##  🧹 Post-Processing","metadata":{}},{"cell_type":"code","source":"def create_meta_features(row, go_depths, go_specificity, go_frequency_norm):\n    \"\"\"Extract meta-features from a prediction row\"\"\"\n    features = {}\n    \n    # Score features\n    features['score_A'] = row.get('score_A', 0)\n    features['score_B'] = row.get('score_B', 0)\n    features['score_mean'] = (features['score_A'] + features['score_B']) / 2\n    features['score_max'] = max(features['score_A'], features['score_B'])\n    features['score_min'] = min(features['score_A'], features['score_B'])\n    features['score_diff'] = abs(features['score_A'] - features['score_B'])\n    features['score_std'] = np.std([features['score_A'], features['score_B']])\n    features['score_product'] = features['score_A'] * features['score_B']\n    features['score_ratio'] = features['score_max'] / (features['score_min'] + 0.001)\n    \n    # GO term features\n    go_term = row['go_term']\n    features['go_depth'] = go_depths.get(go_term, 0)\n    features['go_specificity'] = go_specificity.get(go_term, 0)\n    features['go_frequency'] = go_frequency_norm.get(go_term, 0)\n    \n    # Agreement features\n    features['models_agree'] = 1 if features['score_diff'] < 0.1 else 0\n    features['both_high_conf'] = 1 if (features['score_A'] > 0.7 and features['score_B'] > 0.7) else 0\n    \n    return features\n\nprint(f\"\\n[Creating Meta-Features Dataset]\")\n\n# Merge A and B\nA_keys = set(A.pred_key)\nB_keys = set(B.pred_key)\nintersect_keys = A_keys & B_keys\n\nprint(f\"Intersection: {len(intersect_keys)} predictions\")\n\n# Get intersection\nA_inter = A[A.pred_key.isin(intersect_keys)].copy()\nB_inter = B[B.pred_key.isin(intersect_keys)].copy()\n\nmerged = A_inter.merge(\n    B_inter[['pred_key', 'score']],\n    on='pred_key',\n    suffixes=('_A', '_B')\n)\n\nprint(f\"Creating meta-features for {len(merged)} predictions...\")\n\n# Create features\nmeta_features_list = []\nfor idx, row in tqdm(merged.iterrows(), total=len(merged), desc=\"Meta-features\"):\n    features = create_meta_features(row, go_depths, go_specificity, go_frequency_norm)\n    features['pred_key'] = row['pred_key']\n    features['protein_id'] = row['protein_id']\n    features['go_term'] = row['go_term']\n    meta_features_list.append(features)\n\nmeta_df = pd.DataFrame(meta_features_list)\n\nprint(f\"[✓] Meta-features created: {meta_df.shape}\")\nprint(f\"Features: {[col for col in meta_df.columns if col not in ['pred_key', 'protein_id', 'go_term']]}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:36:45.105705Z","iopub.execute_input":"2025-12-12T00:36:45.105979Z","iopub.status.idle":"2025-12-12T00:42:32.287239Z","shell.execute_reply.started":"2025-12-12T00:36:45.105950Z","shell.execute_reply":"2025-12-12T00:42:32.286148Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"code","source":"print(f\"\\n[Preparing Meta-Learner Training Data]\")\n\n# Strategy: Use ALL predictions (not just intersection) and label with ground truth\n# This ensures we have positive samples\n\n# Merge A and B on pred_key (outer join to keep all)\nprint(\"Merging all predictions from A and B...\")\nall_A = A[['pred_key', 'protein_id', 'go_term', 'score']].copy()\nall_A.columns = ['pred_key', 'protein_id', 'go_term', 'score_A']\n\nall_B = B[['pred_key', 'protein_id', 'go_term', 'score']].copy()\nall_B.columns = ['pred_key', 'protein_id', 'go_term', 'score_B']\n\n# Outer merge to keep all predictions\nall_merged = all_A.merge(all_B, on=['pred_key', 'protein_id', 'go_term'], how='outer')\n\n# Fill missing scores with 0 (for predictions only in one model)\nall_merged['score_A'] = all_merged['score_A'].fillna(0)\nall_merged['score_B'] = all_merged['score_B'].fillna(0)\n\nprint(f\"Total merged predictions: {len(all_merged)}\")\n\n# Sample for training (to save memory and time)\n# We'll use a stratified sample\nall_merged['label'] = all_merged['pred_key'].isin(goa_pred_keys).astype(int)\n\npositive_count = all_merged['label'].sum()\nnegative_count = len(all_merged) - positive_count\n\nprint(f\"Full dataset class distribution:\")\nprint(f\"  Positive: {positive_count} ({100*positive_count/len(all_merged):.2f}%)\")\nprint(f\"  Negative: {negative_count} ({100*negative_count/len(all_merged):.2f}%)\")\n\nif positive_count == 0:\n    print(\"\\n⚠️  WARNING: Still no positive samples!\")\n    print(\"This suggests predictions are completely disjoint from ground truth.\")\n    print(\"Falling back to heuristic approach...\")\n    \n    # For intersection only (where both models have predictions)\n    meta_df = all_merged[(all_merged['score_A'] > 0) & (all_merged['score_B'] > 0)].copy()\n    \n    # Heuristic scoring\n    meta_df['meta_score'] = (\n        0.4 * meta_df['score_A'] + \n        0.4 * meta_df['score_B'] + \n        0.2 * ((meta_df['score_A'] + meta_df['score_B']) / 2)\n    )\n    \n    # Agreement bonus\n    meta_df['score_diff'] = abs(meta_df['score_A'] - meta_df['score_B'])\n    agreement_bonus = (1 - meta_df['score_diff']) * 0.1\n    meta_df['meta_score'] = (meta_df['meta_score'] + agreement_bonus).clip(0, 1)\n    \n    USE_META_LEARNER = False\n    \nelse:\n    print(f\"\\n✓ Found {positive_count} positive samples!\")\n    \n    # Sample for training (stratified)\n    max_samples = 500000  # Limit for memory\n    if len(all_merged) > max_samples:\n        print(f\"Sampling {max_samples} examples for training...\")\n        \n        positives = all_merged[all_merged['label'] == 1]\n        negatives = all_merged[all_merged['label'] == 0]\n        \n        # Keep all positives if possible, sample negatives proportionally\n        if len(positives) < max_samples * 0.1:  # At least 10% positive\n            n_neg = min(len(negatives), max_samples - len(positives))\n            negatives_sample = negatives.sample(n=n_neg, random_state=42)\n            training_df = pd.concat([positives, negatives_sample], ignore_index=True)\n        else:\n            training_df = all_merged.sample(n=max_samples, random_state=42)\n    else:\n        training_df = all_merged.copy()\n    \n    print(f\"Training on {len(training_df)} samples\")\n    \n    # Create meta-features for training data\n    print(\"Creating meta-features...\")\n    meta_features_list = []\n    for idx, row in tqdm(training_df.iterrows(), total=len(training_df), desc=\"Features\"):\n        features = create_meta_features(row, go_depths, go_specificity, go_frequency_norm)\n        features['pred_key'] = row['pred_key']\n        features['protein_id'] = row['protein_id']\n        features['go_term'] = row['go_term']\n        features['label'] = row['label']\n        meta_features_list.append(features)\n    \n    meta_df = pd.DataFrame(meta_features_list)\n    \n    # Split features and labels\n    feature_cols = [col for col in meta_df.columns if col not in ['pred_key', 'protein_id', 'go_term', 'label', 'meta_score']]\n    \n    X = meta_df[feature_cols].values\n    y = meta_df['label'].values\n    \n    print(f\"Training data shape: X={X.shape}, y={y.shape}\")\n    print(f\"Final positive: {y.sum()} ({100*y.sum()/len(y):.2f}%)\")\n    print(f\"Final negative: {len(y) - y.sum()} ({100*(len(y)-y.sum())/len(y):.2f}%)\")\n    \n    USE_META_LEARNER = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:42:32.287950Z","iopub.status.idle":"2025-12-12T00:42:32.288237Z","shell.execute_reply.started":"2025-12-12T00:42:32.288109Z","shell.execute_reply":"2025-12-12T00:42:32.288126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nif USE_META_LEARNER:\n    print(f\"\\n[Training Meta-Learner]\")\n    \n    # Standardize features\n    scaler = StandardScaler()\n    X_scaled = scaler.fit_transform(X)\n    \n    # Train logistic regression\n    print(\"Training Logistic Regression...\")\n    meta_model = LogisticRegression(\n        max_iter=1000,\n        class_weight='balanced',  # Handle class imbalance\n        C=1.0,\n        random_state=42,\n        n_jobs=-1\n    )\n    \n    meta_model.fit(X_scaled, y)\n    \n    print(f\"[✓] Meta-model trained!\")\n    print(f\"Model coefficients shape: {meta_model.coef_.shape}\")\n    \n    # Show feature importance\n    feature_importance = pd.DataFrame({\n        'feature': feature_cols,\n        'importance': abs(meta_model.coef_[0])\n    }).sort_values('importance', ascending=False)\n    \n    print(f\"\\nTop 10 Most Important Features:\")\n    print(feature_importance.head(10))\n    \nelse:\n    print(f\"\\n[Skipping Meta-Learner Training]\")\n    print(\"Using heuristic-based approach (already computed in Cell 11)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:42:32.289488Z","iopub.status.idle":"2025-12-12T00:42:32.289807Z","shell.execute_reply.started":"2025-12-12T00:42:32.289633Z","shell.execute_reply":"2025-12-12T00:42:32.289648Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"$$ $$","metadata":{}},{"cell_type":"code","source":"\nprint(f\"\\n[Generating Final Scores]\")\n\nif USE_META_LEARNER:\n    print(\"Predicting with trained meta-learner...\")\n    \n    # Need to create features for ALL predictions (not just training sample)\n    print(\"Creating features for all predictions...\")\n    \n    # Get intersection where both models have predictions\n    intersection_df = all_merged[(all_merged['score_A'] > 0) & (all_merged['score_B'] > 0)].copy()\n    \n    # Create features\n    pred_features_list = []\n    for idx, row in tqdm(intersection_df.iterrows(), total=len(intersection_df), desc=\"Prediction features\"):\n        features = create_meta_features(row, go_depths, go_specificity, go_frequency_norm)\n        features['pred_key'] = row['pred_key']\n        features['protein_id'] = row['protein_id']\n        features['go_term'] = row['go_term']\n        pred_features_list.append(features)\n    \n    pred_df = pd.DataFrame(pred_features_list)\n    \n    # Extract features\n    X_pred = pred_df[feature_cols].values\n    X_pred_scaled = scaler.transform(X_pred)\n    \n    # Predict\n    pred_df['meta_score'] = meta_model.predict_proba(X_pred_scaled)[:, 1]\n    \n    meta_predictions = pred_df[['protein_id', 'go_term', 'meta_score']].copy()\n    meta_predictions.columns = ['protein_id', 'go_term', 'score']\n    \n    print(f\"Meta-learner predictions: {len(meta_predictions)}\")\n    \nelse:\n    print(\"Using heuristic scores...\")\n    meta_predictions = meta_df[['protein_id', 'go_term', 'meta_score']].copy()\n    meta_predictions.columns = ['protein_id', 'go_term', 'score']\n\nmeta_predictions['pred_key'] = meta_predictions['protein_id'].astype(str) + '_' + meta_predictions['go_term'].astype(str)\n\nprint(f\"Final meta-predictions shape: {meta_predictions.shape}\")\nprint(f\"Score distribution:\\n{meta_predictions['score'].describe()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:42:32.291083Z","iopub.status.idle":"2025-12-12T00:42:32.291319Z","shell.execute_reply.started":"2025-12-12T00:42:32.291214Z","shell.execute_reply":"2025-12-12T00:42:32.291224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\\n[Adding Non-Overlapping Predictions]\")\n\n# Predictions only in A\nAnotB = A[~A.pred_key.isin(B_keys)].copy()\nAnotB = AnotB[['protein_id', 'go_term', 'score', 'pred_key']]\n\n# Predictions only in B\nBnotA = B[~B.pred_key.isin(A_keys)].copy()\nBnotA = BnotA[['protein_id', 'go_term', 'score', 'pred_key']]\n\nprint(f\"A only: {len(AnotB)}\")\nprint(f\"B only: {len(BnotA)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:42:32.292688Z","iopub.status.idle":"2025-12-12T00:42:32.293121Z","shell.execute_reply.started":"2025-12-12T00:42:32.292847Z","shell.execute_reply":"2025-12-12T00:42:32.292860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def positive_propagation(df, parents_map, threshold=0.85, decay_factor=0.92):\n    \"\"\"Propagate high-confidence predictions to ancestors\"\"\"\n    print(f\"[Positive Propagation] Starting with {len(df)} predictions\")\n    print(f\"[Positive Propagation] Threshold: {threshold}, Decay: {decay_factor}\")\n    \n    high_conf = df[df['score'] >= threshold].copy()\n    print(f\"[Positive Propagation] High-confidence: {len(high_conf)}\")\n    \n    new_predictions = []\n    existing_keys = set(df['pred_key'])\n    \n    for idx, row in tqdm(high_conf.iterrows(), total=len(high_conf), desc=\"Propagating\"):\n        ancestors = get_ancestors(row['go_term'], parents_map)\n        \n        for ancestor in ancestors:\n            pred_key = f\"{row['protein_id']}_{ancestor}\"\n            \n            if pred_key not in existing_keys:\n                new_predictions.append({\n                    'protein_id': row['protein_id'],\n                    'go_term': ancestor,\n                    'score': round(row['score'] * decay_factor, 3),\n                    'pred_key': pred_key\n                })\n                existing_keys.add(pred_key)\n    \n    print(f\"[Positive Propagation] Added {len(new_predictions)} ancestors\")\n    \n    if new_predictions:\n        new_df = pd.DataFrame(new_predictions)\n        result = pd.concat([df, new_df], ignore_index=True)\n        return result\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:42:32.294044Z","iopub.status.idle":"2025-12-12T00:42:32.294311Z","shell.execute_reply.started":"2025-12-12T00:42:32.294151Z","shell.execute_reply":"2025-12-12T00:42:32.294160Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\\n[Combining All Predictions]\")\n\n# Combine: Ground truth + Meta predictions + A-only + B-only\nsubmission = pd.concat([\n    go_annotations[['protein_id', 'go_term', 'score', 'pred_key']],\n    meta_predictions,\n    AnotB,\n    BnotA\n], axis=0, ignore_index=True)\n\nprint(f\"Before deduplication: {submission.shape}\")\n\n# Remove duplicates (keep first = ground truth priority)\nsubmission = submission.drop_duplicates(subset=['protein_id', 'go_term'], keep='first')\n\nprint(f\"After deduplication: {submission.shape}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:42:32.295555Z","iopub.status.idle":"2025-12-12T00:42:32.295758Z","shell.execute_reply.started":"2025-12-12T00:42:32.295659Z","shell.execute_reply":"2025-12-12T00:42:32.295668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\\n[Applying Positive Propagation]\")\nsubmission = positive_propagation(\n    submission,\n    parents_map,\n    threshold=0.85,\n    decay_factor=0.92\n)\n\n# Final cleanup\nsubmission.drop(columns=['pred_key'], inplace=True, errors='ignore')\nsubmission = submission.drop_duplicates(subset=['protein_id', 'go_term'], keep='first')\n\nprint(f\"\\n{'='*80}\")\nprint(f\"FINAL SUBMISSION SHAPE: {submission.shape}\")\nprint(f\"{'='*80}\")\n\n# ============================================================================\n# CELL 18: Save Submission\n# ============================================================================\n\nprint(f'\\n[Saving Submission]')\nsubmission.to_csv('submission.tsv', sep='\\t', index=False, header=None)\nprint(f\"[✓] Saved to submission.tsv\")\n\n\nprint(f\"\\n[Final Statistics]\")\nprint(f\"\\nScore Distribution:\")\nprint(submission['score'].describe())\n\nprint(f\"\\nScore Quartiles:\")\nfor q in [0.25, 0.50, 0.75, 0.90, 0.95, 0.99]:\n    val = submission['score'].quantile(q)\n    count = len(submission[submission['score'] >= val])\n    print(f\"  >= {val:.3f} (Q{q*100:.0f}): {count} predictions\")\n\nprint(f\"\\n[Preview Top 20 Predictions]\")\n!head -20 submission.tsv\n\nprint(f\"\\n[✓✓✓ Pipeline Complete! ✓✓✓]\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T00:42:32.296892Z","iopub.status.idle":"2025-12-12T00:42:32.297112Z","shell.execute_reply.started":"2025-12-12T00:42:32.297005Z","shell.execute_reply":"2025-12-12T00:42:32.297014Z"}},"outputs":[],"execution_count":null}]}