{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# CAFA 6 - V5: Ensemble (Combining K-mer + Taxonomy + Global Frequency)\nimport pandas as pd\nimport numpy as np\nfrom collections import Counter, defaultdict\n\n# Custom FASTA parser (no BioPython needed)\ndef parse_fasta(filepath):\n    seqs = {}\n    current_id = None\n    current_seq = []\n    with open(filepath, 'r') as f:\n        for line in f:\n            line = line.strip()\n            if line.startswith('>'):\n                if current_id:\n                    seqs[current_id] = ''.join(current_seq)\n                header = line[1:]\n                current_id = header.split('|')[1] if '|' in header else header.split()[0]\n                current_seq = []\n            else:\n                current_seq.append(line)\n        if current_id:\n            seqs[current_id] = ''.join(current_seq)\n    return seqs\n\nDATA_PATH = '/kaggle/input/cafa-6-protein-function-prediction'\n\nprint('='*60)\nprint('Version 5: Ensemble (K-mer + Taxonomy + Global)')\nprint('='*60)\n\n# Load data\nprint('\\nLoading data...')\ntrain_terms = pd.read_csv(f'{DATA_PATH}/Train/train_terms.tsv', sep='\\t')\ntrain_taxonomy = pd.read_csv(f'{DATA_PATH}/Train/train_taxonomy.tsv',\n                             sep='\\t', header=None, names=['EntryID', 'taxon'])\n\ntest_taxonomy = pd.read_csv(f'{DATA_PATH}/Test/testsuperset-taxon-list.tsv',\n                           sep='\\t', header=None, names=['EntryID', 'taxon'])\ntest_proteins = test_taxonomy['EntryID'].unique()\nprint(f'Train: {len(train_terms)} annotations, Test: {len(test_proteins)} proteins')\n\n# Load sequences\nprint('Loading protein sequences...')\ntrain_seqs = parse_fasta(f'{DATA_PATH}/Train/train_sequences.fasta')\ntest_seqs = parse_fasta(f'{DATA_PATH}/Test/testsuperset.fasta')\nprint(f'Loaded {len(train_seqs)} train and {len(test_seqs)} test sequences')\n\n# Build k-mer index\nprint('Building k-mer index...')\nK = 3  # k-mer size\nkmer_index = defaultdict(set)\n\nfor prot_id, seq in train_seqs.items():\n    for i in range(len(seq) - K + 1):\n        kmer = seq[i:i+K]\n        kmer_index[kmer].add(prot_id)\n\nprint(f'Built k-mer index with {len(kmer_index)} unique (K)-mers')\n\n# Merge and build taxonomy model\ntrain_data = train_terms.merge(train_taxonomy, on='EntryID', how='left')\naspect_map = {'P': 'BPO', 'F': 'MFO', 'C': 'CCO'}\n\n# Build taxonomy-based term frequencies\ntaxonomy_terms = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))\nfor _, row in train_data.iterrows():\n    taxon = row.get('taxon', 'Unknown')\n    ontology = aspect_map.get(row['aspect'], row['aspect'])\n    taxonomy_terms[taxon][ontology][row['term']] += 1\n\n# Build protein-term index for k-mer similarity\nprotein_terms = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))\nfor _, row in train_data.iterrows():\n    prot_id = row['EntryID']\n    ontology = aspect_map.get(row['aspect'], row['aspect'])\n    protein_terms[prot_id][ontology][row['term']] += 1\n\nprint(f'Built models for {len(taxonomy_terms)} taxa and {len(protein_terms)} proteins')\n\n# Calculate global statistics\nontology_stats = {}\nfor aspect, ontology in aspect_map.items():\n    terms = train_terms[train_terms['aspect'] == aspect]['term'].values\n    counts = Counter(terms)\n    total = len(terms)\n    probs = {t: c/total for t, c in counts.items()}\n    ontology_stats[ontology] = {\n        'counts': counts, 'probs': probs,\n        'top': [t for t, _ in counts.most_common(20)]\n    }\nprint(f\"Ontology stats: {len(ontology_stats)} unique terms\")\n\nprint('\\nGenerating predictions with ensemble...')\npredictions = []\n\nfor idx, protein in enumerate(test_proteins):\n    if (idx + 1) % 1000 == 0:\n        print(f'  Processed {idx + 1}/{len(test_proteins)} proteins')\n    \n    # Get taxonomy\n    taxon_row = test_taxonomy[test_taxonomy['EntryID'] == protein]\n    taxon = taxon_row['taxon'].iloc[0] if len(taxon_row) > 0 else 'Unknown'\n    \n    # Get sequence\n    test_seq = test_seqs.get(protein, '')\n    \n    # Find similar proteins using k-mer overlap\n    similar_proteins = Counter()\n    if test_seq:\n        for i in range(len(test_seq) - K + 1):\n            kmer = test_seq[i:i+K]\n            for train_prot in kmer_index.get(kmer, set()):\n                similar_proteins[train_prot] += 1\n    \n    # Get top 10 similar proteins\n    top_similar = [p for p, _ in similar_proteins.most_common(10)]\n    \n    # For each ontology, combine predictions from 3 strategies\n    for ontology in ['BPO', 'MFO', 'CCO']:\n        stats = ontology_stats[ontology]\n        predictions_for_ont = defaultdict(float)\n        \n        # Strategy 1: K-mer similarity (weight: 0.4)\n        if top_similar:\n            for sim_prot in top_similar:\n                if sim_prot in protein_terms and ontology in protein_terms[sim_prot]:\n                    for term, count in protein_terms[sim_prot][ontology].items():\n                        # Weight by similarity score\n                        sim_score = similar_proteins[sim_prot] / len(test_seq) if test_seq else 0\n                        predictions_for_ont[term] += sim_score * 0.4\n        \n        # Strategy 2: Taxonomy (weight: 0.3)\n        if taxon in taxonomy_terms and ontology in taxonomy_terms[taxon]:\n            taxon_terms = taxonomy_terms[taxon][ontology]\n            for term, count in taxon_terms.items():\n                freq = count / sum(taxon_terms.values())\n                predictions_for_ont[term] += freq * 0.3\n        \n        # Strategy 3: Global frequency (weight: 0.3)\n        for i, term in enumerate(stats['top'][:10]):\n            predictions_for_ont[term] += stats['probs'][term] * 0.3\n        \n        # Get top 5 predictions for this ontology\n        top_preds = sorted(predictions_for_ont.items(), key=lambda x: x[1], reverse=True)[:5]\n        \n        for term, score in top_preds:\n            # Normalize confidence to [0.05, 0.95]\n            conf = min(0.95, max(0.05, score))\n            predictions.append({\n                'Protein_Id': protein, 'GO_Term': term, 'Confidence': conf\n            })\n\nsubmission = pd.DataFrame(predictions)\nprint(f'\\nSubmission: {len(submission)} predictions')\nprint(f\"Confidence: {submission['Confidence'].min():.3f} - {submission['Confidence'].max():.3f}\")\nprint('\\nSample:')\nprint(submission.head(15))\n\nsubmission.to_csv('submission.tsv', index=False, sep='\\t')\nprint('\\nSaved to submission.tsv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T08:21:50.719209Z","iopub.execute_input":"2025-12-15T08:21:50.719567Z","iopub.status.idle":"2025-12-15T08:23:42.005071Z","shell.execute_reply.started":"2025-12-15T08:21:50.719540Z","shell.execute_reply":"2025-12-15T08:23:42.003938Z"}},"outputs":[],"execution_count":null}]}