{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CAFA6 — Combined ProtBERT Embedding + Multi-label Classifier\n**Author:** Wasiq Ali\n\n**Goal:** Combine ideas from multiple CAFA6 notebooks (ProtBERT embeddings + classifiers) into a single unified approach.\n","metadata":{}},{"cell_type":"code","source":"!pip install Bio","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:03.914694Z","iopub.execute_input":"2025-10-19T15:23:03.915862Z","iopub.status.idle":"2025-10-19T15:23:08.044355Z","shell.execute_reply.started":"2025-10-19T15:23:03.915829Z","shell.execute_reply":"2025-10-19T15:23:08.043107Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. import library","metadata":{}},{"cell_type":"code","source":"# 1. Setup\nimport os, gc, json, glob\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom Bio import SeqIO\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.metrics import roc_auc_score\nimport numpy as np\nimport lightgbm as lgb\nimport gc, time\nimport torch\nfrom transformers import AutoTokenizer, AutoModel\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint('Device:', DEVICE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:14.454199Z","iopub.execute_input":"2025-10-19T15:23:14.454522Z","iopub.status.idle":"2025-10-19T15:23:14.461181Z","shell.execute_reply.started":"2025-10-19T15:23:14.454499Z","shell.execute_reply":"2025-10-19T15:23:14.460244Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. load the dataset","metadata":{}},{"cell_type":"code","source":"import os\n\n# Step 1: Kaggle competition dataset ka base path\nbase_path = \"/kaggle/input/cafa-6-protein-function-prediction\"\n\n# Step 2: Directory walk se har file ka full path print karo\nfor dirname, _, filenames in os.walk(base_path):\n    for filename in filenames:\n        full_path = os.path.join(dirname, filename)\n        print(full_path)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:16.636078Z","iopub.execute_input":"2025-10-19T15:23:16.636769Z","iopub.status.idle":"2025-10-19T15:23:16.646464Z","shell.execute_reply.started":"2025-10-19T15:23:16.636739Z","shell.execute_reply":"2025-10-19T15:23:16.645454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom Bio import SeqIO\n\n# 1. Define paths\nINPUT_DIR = '/kaggle/input/cafa-6-protein-function-prediction'\nTRAIN_PATH = os.path.join(INPUT_DIR, 'Train/train_terms.tsv')\nTEST_PATH = os.path.join(INPUT_DIR, 'Test/testsuperset.fasta')\nTEST_PATHS = os.path.join(INPUT_DIR, 'Test/testsuperset-taxon-list.tsv')\nSAMPLE_PATH = os.path.join(INPUT_DIR, 'sample_submission.tsv')\n\n# 2. Load train data (.tsv)\ntrain_df = pd.read_csv(TRAIN_PATH, sep='\\t')\nprint(\"✅ Train data loaded:\", train_df.shape)\n\n# 2. Load train data (.tsv)\ntest_df = pd.read_csv(TEST_PATHS, sep='\\t')\nprint(\"✅ Train data loaded:\", train_df.shape)\n\n# 3. Load test sequences (.fasta)\ntest_sequences = list(SeqIO.parse(TEST_PATH, \"fasta\"))\nprint(\"✅ Test sequences loaded:\", len(test_sequences))\n\n# 4. Manually read valid lines from sample_submission.tsv\nsample_data = []\nwith open(SAMPLE_PATH, 'r') as file:\n    for line in file:\n        parts = line.strip().split('\\t')\n        if len(parts) == 3:\n            sample_data.append(parts)\n\n# Convert to DataFrame\nsample_sub = pd.DataFrame(sample_data, columns=['ProteinID', 'GO_term', 'score'])\nprint(\"✅ Sample submission loaded:\", sample_sub.shape)\n\n# Preview\nprint(sample_sub.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:18.767592Z","iopub.execute_input":"2025-10-19T15:23:18.768261Z","iopub.status.idle":"2025-10-19T15:23:22.015531Z","shell.execute_reply.started":"2025-10-19T15:23:18.768231Z","shell.execute_reply":"2025-10-19T15:23:22.014548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. explore the dataset","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:22.016803Z","iopub.execute_input":"2025-10-19T15:23:22.017094Z","iopub.status.idle":"2025-10-19T15:23:22.025979Z","shell.execute_reply.started":"2025-10-19T15:23:22.017074Z","shell.execute_reply":"2025-10-19T15:23:22.025084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\n# Step 1: Assume train_df is already loaded and looks like:\n# EntryID     term        aspect\n# Q5W0B1      GO:0000785  C\n# Q5W0B1      GO:0004842  F\n# Q5W0B1      GO:0051865  P\n\n# ✅ Step 2: Create 'labels' column — semicolon separated GO terms\nnew_df = train_df.groupby('EntryID')['term'].apply(lambda x: ';'.join(x)).reset_index()\nnew_df.columns = ['EntryID', 'labels']\n\n# ✅ Step 3: Use your original label parsing + binarization code\ndef parse_labels(x):\n    if pd.isna(x): return []\n    return [t.strip() for t in str(x).split(';') if t.strip()]\n\nnew_df['labels_list'] = new_df['labels'].apply(parse_labels)\n\nmlb = MultiLabelBinarizer()\nY = mlb.fit_transform(new_df['labels_list'])\n\n# ✅ Step 4: Output confirmation\nprint('✅ Num proteins:', new_df.shape[0])\nprint('✅ Num classes:', len(mlb.classes_))\nprint('✅ Y shape:', Y.shape)\nprint(new_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:24.725071Z","iopub.execute_input":"2025-10-19T15:23:24.72542Z","iopub.status.idle":"2025-10-19T15:23:29.660697Z","shell.execute_reply.started":"2025-10-19T15:23:24.725394Z","shell.execute_reply":"2025-10-19T15:23:29.659898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Load fasta file and extract sequences\nfasta_path = \"/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta\"\nseq_records = SeqIO.parse(fasta_path, \"fasta\")\n\n# Step 2: Convert to DataFrame\nseq_data = []\nfor record in seq_records:\n    seq_data.append({\n        \"EntryID\": record.id,            # e.g. Q5W0B1\n        \"sequence\": str(record.seq)      # e.g. MAASEYKLYG...\n    })\n\nseq_df = pd.DataFrame(seq_data)\n\n# Step 3: Merge with your original train_df\ntrain_df = pd.merge(train_df, seq_df, on=\"EntryID\", how=\"left\")\n\n# (Optional) Check if any sequences are missing\nmissing = train_df[train_df['sequence'].isna()]\nprint(\"Missing sequences:\", len(missing))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:29.661923Z","iopub.execute_input":"2025-10-19T15:23:29.662259Z","iopub.status.idle":"2025-10-19T15:23:30.451373Z","shell.execute_reply.started":"2025-10-19T15:23:29.662237Z","shell.execute_reply":"2025-10-19T15:23:30.450383Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Sequence","metadata":{}},{"cell_type":"code","source":"def simple_features(sequences):\n    features = []\n    for seq in sequences:\n        length = len(seq)\n        a_count = seq.count('A') / length\n        g_count = seq.count('G') / length\n        c_count = seq.count('C') / length\n        t_count = seq.count('T') / length\n        features.append([length, a_count, g_count, c_count, t_count])\n    return np.array(features)\n\ntrain_emb = simple_features(train_df['sequence'].astype(str).tolist())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:30.452708Z","iopub.execute_input":"2025-10-19T15:23:30.453064Z","iopub.status.idle":"2025-10-19T15:23:32.051813Z","shell.execute_reply.started":"2025-10-19T15:23:30.45301Z","shell.execute_reply":"2025-10-19T15:23:32.051099Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. merging","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom Bio import SeqIO\nimport numpy as np\n\n# 1. Define paths\nINPUT_DIR = '/kaggle/input/cafa-6-protein-function-prediction'\nTRAIN_TERMS_PATH = os.path.join(INPUT_DIR, 'Train', 'train_terms.tsv')\nTRAIN_FASTA_PATH = os.path.join(INPUT_DIR, 'Train', 'train_sequences.fasta')\nTEST_FASTA_PATH = os.path.join(INPUT_DIR, 'Test', 'testsuperset.fasta')\nSAMPLE_SUB_PATH = os.path.join(INPUT_DIR, 'sample_submission.tsv')\n\n# 2. Load train terms (.tsv)\ntrain_df = pd.read_csv(TRAIN_TERMS_PATH, sep='\\t')\nprint(\"✅ Train data loaded:\", train_df.shape)\nprint(\"Columns:\", train_df.columns.tolist())\n\n# 3. Load sample submission safely\nsample_data = []\nwith open(SAMPLE_SUB_PATH, 'r') as f:\n    for line in f:\n        parts = line.strip().split('\\t')\n        if len(parts) == 3:\n            sample_data.append(parts)\n\nsample_sub = pd.DataFrame(sample_data, columns=['ID', 'GO_term', 'score'])\nprint(\"✅ Sample submission loaded:\", sample_sub.shape)\nprint(\"Sample sub columns:\", sample_sub.columns.tolist())\n\n# 4. Load FASTA sequences into DataFrame\ndef load_fasta_to_df(fasta_path):\n    ids = []\n    seqs = []\n    for rec in SeqIO.parse(fasta_path, \"fasta\"):\n        ids.append(rec.id)\n        seqs.append(str(rec.seq))\n    return pd.DataFrame({'id': ids, 'sequence': seqs})\n\ntrain_seqs_df = load_fasta_to_df(TRAIN_FASTA_PATH)\ntest_seqs_df = load_fasta_to_df(TEST_FASTA_PATH)\nprint(\"✅ Train sequences DF:\", train_seqs_df.shape)\nprint(\"✅ Test sequences DF:\", test_seqs_df.shape)\n\n# 5. Merge train_df with train_seq\ntrain_df = pd.merge(train_df.rename(columns={'EntryID': 'id'}), train_seqs_df, on='id', how='left')\nprint(\"🔎 After merge train_df:\", train_df.shape)\nprint(\"Missing sequences in train:\", train_df['sequence'].isna().sum())\n\n# 6. Merge sample_sub (test) with sequences\ntest_df = pd.merge(sample_sub.rename(columns={'ID': 'id'}), test_seqs_df, on='id', how='left')\nprint(\"🔎 After merge test_df:\", test_df.shape)\nprint(\"Missing sequences in test:\", test_df['sequence'].isna().sum())\n\n# 7. Feature extraction function\ndef simple_features(sequences):\n    feats = []\n    for seq in sequences:\n        if not isinstance(seq, str) or len(seq) == 0:\n            feats.append([0, 0, 0, 0, 0])\n            continue\n        length = len(seq)\n        a = seq.count('A') / length\n        g = seq.count('G') / length\n        c = seq.count('C') / length\n        t = seq.count('T') / length\n        feats.append([length, a, g, c, t])\n    return np.array(feats)\n\n# 8. Build features for train & test\ntrain_emb = simple_features(train_df['sequence'].astype(str).tolist())\ntest_emb = simple_features(test_df['sequence'].astype(str).tolist())\nprint(\"✅ train_feats shape:\", train_emb.shape)\nprint(\"✅ test_feats shape:\", test_emb.shape)\n\n# 9. (Optional) Save features\nnp.save('train_feats.npy', train_emb)\nnp.save('test_feats.npy', test_emb)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:32.374792Z","iopub.execute_input":"2025-10-19T15:23:32.375152Z","iopub.status.idle":"2025-10-19T15:23:36.225375Z","shell.execute_reply.started":"2025-10-19T15:23:32.375127Z","shell.execute_reply":"2025-10-19T15:23:36.224432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check columns to identify the correct ID field\nprint(\"train_df columns:\", train_df.columns.tolist())\nprint(\"test_df columns:\", test_df.columns.tolist())\nprint(\"train_seqs_df columns:\", train_seqs_df.columns.tolist())\nprint(\"test_seqs_df columns:\", test_seqs_df.columns.tolist())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:23:36.227094Z","iopub.execute_input":"2025-10-19T15:23:36.227444Z","iopub.status.idle":"2025-10-19T15:23:36.233091Z","shell.execute_reply.started":"2025-10-19T15:23:36.227425Z","shell.execute_reply":"2025-10-19T15:23:36.232222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from Bio import SeqIO\n\nfasta_path = \"/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta\"\nall_seq_ids = [rec.id for rec in SeqIO.parse(fasta_path, \"fasta\")]\nprint(\"🔹 Total IDs in fasta:\", len(all_seq_ids))\n\nterms_ids = set(grouped['seq_id'])\nprint(\"🔹 Total IDs in train_terms.tsv:\", len(terms_ids))\n\ncommon_ids = terms_ids.intersection(all_seq_ids)\nprint(\"✅ Common IDs between fasta & terms:\", len(common_ids))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:24:36.912956Z","iopub.execute_input":"2025-10-19T15:24:36.913664Z","iopub.status.idle":"2025-10-19T15:24:37.310814Z","shell.execute_reply.started":"2025-10-19T15:24:36.913638Z","shell.execute_reply":"2025-10-19T15:24:37.309919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize both fasta and terms IDs (strip prefixes)\ndef clean_id(x):\n    return x.split('|')[-1].strip().upper()\n\nfasta_clean = [clean_id(i) for i in all_seq_ids]\nterms_clean = [clean_id(i) for i in grouped['seq_id']]\n\n# Mapping again\nid_to_idx = {sid: i for i, sid in enumerate(fasta_clean)}\n\nlabel_indices = [id_to_idx[sid] for sid in terms_clean if sid in id_to_idx]\nprint(\"✅ Found matching embeddings for\", len(label_indices), \"labeled sequences\")\n\ntrain_red = train_red[label_indices]\nprint(\"Filtered train_red shape:\", train_red.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:24:39.514451Z","iopub.execute_input":"2025-10-19T15:24:39.515179Z","iopub.status.idle":"2025-10-19T15:24:39.583478Z","shell.execute_reply.started":"2025-10-19T15:24:39.515149Z","shell.execute_reply":"2025-10-19T15:24:39.582683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"train_red:\", train_red.shape)\nprint(\"Y:\", Y.shape)\nassert train_red.shape[0] == Y.shape[0], \"❌ Still mismatched!\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:24:42.837205Z","iopub.execute_input":"2025-10-19T15:24:42.838107Z","iopub.status.idle":"2025-10-19T15:24:42.862515Z","shell.execute_reply.started":"2025-10-19T15:24:42.838065Z","shell.execute_reply":"2025-10-19T15:24:42.861491Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Reduce dimension\n","metadata":{}},{"cell_type":"code","source":"# 6. Reduce dimension\nfrom sklearn.decomposition import TruncatedSVD\nsvd = TruncatedSVD(n_components=5, random_state=42)\ntrain_red = svd.fit_transform(train_emb)\ntest_red = svd.transform(test_emb)\nprint(train_red.shape, test_red.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:24:57.924255Z","iopub.execute_input":"2025-10-19T15:24:57.924877Z","iopub.status.idle":"2025-10-19T15:24:58.612499Z","shell.execute_reply.started":"2025-10-19T15:24:57.924852Z","shell.execute_reply":"2025-10-19T15:24:58.611525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nimport numpy as np\n\nprint(\"Applying PCA compression to reduce embedding size...\")\n\n# Convert dtype\ntrain_red = train_red.astype(np.float32)\ntest_red = test_red.astype(np.float32)\n\n# Determine valid number of components\nn_samples, n_features = train_red.shape\nn_components = min(256, n_samples, n_features // 2)\nprint(f\"Using PCA n_components={n_components} (auto-adjusted)\")\n\n# Apply PCA\npca = PCA(n_components=n_components, random_state=42)\ntrain_red = pca.fit_transform(train_red)\ntest_red = pca.transform(test_red)\n\nprint(\"✅ PCA compression complete:\", train_red.shape, test_red.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:25:03.374131Z","iopub.execute_input":"2025-10-19T15:25:03.374434Z","iopub.status.idle":"2025-10-19T15:25:03.691657Z","shell.execute_reply.started":"2025-10-19T15:25:03.374415Z","shell.execute_reply":"2025-10-19T15:25:03.690725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"train_red shape:\", train_red.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:26:33.879111Z","iopub.execute_input":"2025-10-19T15:26:33.880229Z","iopub.status.idle":"2025-10-19T15:26:33.885112Z","shell.execute_reply.started":"2025-10-19T15:26:33.880196Z","shell.execute_reply":"2025-10-19T15:26:33.883966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"train_red shape:\", test_red.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:26:36.065973Z","iopub.execute_input":"2025-10-19T15:26:36.066334Z","iopub.status.idle":"2025-10-19T15:26:36.071396Z","shell.execute_reply.started":"2025-10-19T15:26:36.06631Z","shell.execute_reply":"2025-10-19T15:26:36.070414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell A: Load / detect embeddings and inspect shapes\nimport os, sys, numpy as np, pandas as pd, gc\nfrom pathlib import Path\n\n# Try to use existing variables if present; otherwise load .npy files if available\ndef get_var(name, globals_):\n    return globals_.get(name, None)\n\n# Use the current notebook globals\ng = globals()\n\ntrain_red = get_var('train_red', g)\ntest_red  = get_var('test_red', g)\n\n# Try to load .npy if variables missing\nif train_red is None or test_red is None:\n    # change these paths if your files are elsewhere\n    train_path = '/kaggle/input/embeddings/train_embeddings.npy'\n    test_path  = '/kaggle/input/embeddings/test_embeddings.npy'\n    if os.path.exists(train_path) and os.path.exists(test_path):\n        print(\"Loading embeddings from .npy files...\")\n        train_red = np.load(train_path)\n        test_red  = np.load(test_path)\n    else:\n        raise FileNotFoundError(\"train_red/test_red not found in session and default .npy paths missing. \"\n                                \"Either create variables train_red/test_red earlier or upload .npy embeddings to /kaggle/input/embeddings/\")\n\nprint(\"train_red shape:\", train_red.shape, \" dtype:\", train_red.dtype)\nprint(\"test_red  shape:\", test_red.shape,  \" dtype:\", test_red.dtype)\n\n# Force float32 to reduce memory\ntrain_red = train_red.astype('float32', copy=False)\ntest_red  = test_red.astype('float32', copy=False)\n\ngc.collect()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T14:55:57.210864Z","iopub.execute_input":"2025-10-19T14:55:57.211165Z","iopub.status.idle":"2025-10-19T14:55:57.710005Z","shell.execute_reply.started":"2025-10-19T14:55:57.211145Z","shell.execute_reply":"2025-10-19T14:55:57.709315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell B: Create Y and mlb from train_terms.tsv (only run if you do not already have Y)\nimport pandas as pd\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\nterms_path = '/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv'\nseq_path   = '/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta'\n\n# Load train_terms.tsv: assume two cols: sequence_id \\t term (or similar). We'll parse safely.\ndf_terms = pd.read_csv(terms_path, sep='\\t', header=None, names=['seq_id','term'])\n# Group terms per seq id\ngrouped = df_terms.groupby('seq_id')['term'].apply(list).reset_index()\nprint(\"Unique train sequences with terms:\", len(grouped))\n\n# Create MultiLabelBinarizer\nmlb = MultiLabelBinarizer(sparse_output=False)\nY = mlb.fit_transform(grouped['term'].values)  # shape: (n_samples, n_classes)\n\nprint(\"Y shape:\", Y.shape, \"Num classes:\", len(mlb.classes_))\n# NOTE: you must ensure train_red rows align with grouped order; if they don't, you need to align them.\n# If your embeddings correspond to the same order as grouped['seq_id'], OK.\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:27:34.247652Z","iopub.execute_input":"2025-10-19T15:27:34.24801Z","iopub.status.idle":"2025-10-19T15:27:35.285752Z","shell.execute_reply.started":"2025-10-19T15:27:34.247975Z","shell.execute_reply":"2025-10-19T15:27:35.284786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell C: Adaptive PCA compression (safe)\nfrom sklearn.decomposition import PCA\nimport numpy as np, gc\n\nprint(\"Running adaptive PCA...\")\n\nn_samples, n_features = train_red.shape\n# choose target max components (256 preferred), but bound by data\ntarget_components = 256\nn_components = min(target_components, n_samples - 1, n_features)\nif n_components <= 0:\n    n_components = min(1, n_features)   # fallback safe option\nprint(f\"Using PCA n_components = {n_components} (samples={n_samples}, features={n_features})\")\n\nif n_components < n_features:\n    pca = PCA(n_components=n_components, svd_solver='randomized', random_state=42)\n    train_red = pca.fit_transform(train_red)\n    test_red  = pca.transform(test_red)\n    print(\"PCA done. Shapes:\", train_red.shape, test_red.shape)\nelse:\n    print(\"Skipping PCA because n_components >= n_features\")\n\n# Ensure float32\ntrain_red = train_red.astype('float32', copy=False)\ntest_red  = test_red.astype('float32', copy=False)\ngc.collect()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:27:40.181696Z","iopub.execute_input":"2025-10-19T15:27:40.18236Z","iopub.status.idle":"2025-10-19T15:27:40.715874Z","shell.execute_reply.started":"2025-10-19T15:27:40.182332Z","shell.execute_reply":"2025-10-19T15:27:40.715123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # --- ALIGN EMBEDDINGS TO LABELLED SEQUENCES ---\n\n# # 1. Load the sequence IDs in the same order as your embeddings were generated.\n# #    (Replace this with the correct path to your fasta file.)\n# from Bio import SeqIO\n\n# fasta_path = \"/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta\"\n# all_seq_ids = [rec.id for rec in SeqIO.parse(fasta_path, \"fasta\")]\n# print(\"Total embeddings/sequences:\", len(all_seq_ids))\n\n# # 2. Create a mapping from seq_id → row index in train_red\n# id_to_idx = {sid: i for i, sid in enumerate(all_seq_ids)}\n\n# # 3. Select only the embeddings whose seq_id appears in grouped['seq_id']\n# label_indices = [id_to_idx[sid] for sid in grouped['seq_id'] if sid in id_to_idx]\n# train_red = train_red[label_indices]\n\n# print(\"Filtered train_red shape (aligned):\", train_red.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T14:55:59.256765Z","iopub.execute_input":"2025-10-19T14:55:59.257157Z","iopub.status.idle":"2025-10-19T14:55:59.676562Z","shell.execute_reply.started":"2025-10-19T14:55:59.257135Z","shell.execute_reply":"2025-10-19T14:55:59.675605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- REPLACEMENT for Cell 27: ALIGN EMBEDDINGS WITH LABELLED SEQUENCES (robust) ---\nfrom Bio import SeqIO\nimport re, numpy as np, gc\n\nfasta_path = \"/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta\"\n\ndef extract_uniprot_id(header):\n    \"\"\"\n    Extract a UniProt-like ID from FASTA header.\n    Handles common formats like:\n      >sp|Q9V3I5|PROT_DROME description...\n      >Q9V3I5 description...\n    Returns the best candidate ID string.\n    \"\"\"\n    # try pattern with pipes (sp|ID|...)\n    m = re.search(r\"\\|([A-Z0-9]+)\\|\", header)\n    if m:\n        return m.group(1)\n    # fallback: take first token and strip common prefixes\n    first_tok = header.split()[0]\n    # remove leading '>' if present\n    return first_tok.lstrip('>').split('|')[-1]\n\n# Parse FASTA IDs (in the exact order they appear in the FASTA)\nall_seq_ids = [extract_uniprot_id(rec.id) for rec in SeqIO.parse(fasta_path, \"fasta\")]\nprint(\"Total FASTA IDs (parsed):\", len(all_seq_ids))\n\n# Ensure grouped exists (from Cell 25) and contains labelled seq ids\ntry:\n    terms_ids = grouped['seq_id'].astype(str).str.strip().tolist()\nexcept NameError:\n    raise NameError(\"Variable `grouped` not found. Run the cell that creates `grouped`/`Y` from train_terms.tsv first (Cell 25).\")\n\nprint(\"Total labelled IDs (from train_terms):\", len(terms_ids))\n\n# Build mapping and find matches\nid_to_idx = {sid: i for i, sid in enumerate(all_seq_ids)}\n\nlabel_indices = [id_to_idx[sid] for sid in terms_ids if sid in id_to_idx]\nprint(\"Matched labelled embeddings:\", len(label_indices), \" / \", len(terms_ids))\n\nif len(label_indices) == 0:\n    # helpful debug output\n    print(\"Example fasta ids:\", all_seq_ids[:5])\n    print(\"Example label ids:\", terms_ids[:5])\n    raise ValueError(\"No matches found between train_terms IDs and FASTA IDs. Check formats; may need custom parsing.\")\n\n# Subset embeddings to only labelled rows (order follows grouped['seq_id'])\ntrain_red = train_red[label_indices]\nprint(\"Filtered train_red shape:\", train_red.shape)\ngc.collect()\n\n# Final assertion: shapes must match Y\nprint(\"Y shape:\", getattr(globals().get('Y', None), 'shape', None))\nif 'Y' in globals():\n    assert train_red.shape[0] == Y.shape[0], \"train_red and Y row counts still mismatched!\"\nelse:\n    print(\"Warning: Y not found in namespace; ensure you built Y earlier.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T15:31:12.356929Z","iopub.execute_input":"2025-10-19T15:31:12.357314Z","iopub.status.idle":"2025-10-19T15:31:12.869495Z","shell.execute_reply.started":"2025-10-19T15:31:12.357292Z","shell.execute_reply":"2025-10-19T15:31:12.868441Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. LogisticRegression baseline\n","metadata":{}},{"cell_type":"code","source":"# Cell D: Memory-safe LogisticRegression baseline (REPLACE 3rd-last cell)\nimport numpy as np, gc, time\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.multiclass import OneVsRestClassifier\n\n# If train_red has extremely low dims (e.g., shape[1] <= 2), try to load richer embeddings if available\n# Uncomment and edit paths if you have original embeddings stored as .npy\n# richer_train_path = '/kaggle/input/embeddings/orig_train_embeddings.npy'\n# richer_test_path  = '/kaggle/input/embeddings/orig_test_embeddings.npy'\n# if train_red.shape[1] <= 2 and os.path.exists(richer_train_path):\n#     print(\"Switching to richer embeddings for LR baseline.\")\n#     train_lr = np.load(richer_train_path).astype('float32')\n#     test_lr  = np.load(richer_test_path).astype('float32')\n# else:\ntrain_lr = train_red\ntest_lr  = test_red\n\nprint(\"train_lr shape:\", train_lr.shape, \"test_lr shape:\", test_lr.shape)\n\n# Setup\nn_folds = 3\nkf = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n\n# If Y not created yet, ensure you created it earlier (Cell B)\n# We limit to top_k labels by frequency to keep memory small\nlabel_freq = Y.sum(axis=0)\ntop_k = min(50, Y.shape[1])   # change to 20 or 10 if still heavy\ntop_label_idx = np.argsort(-label_freq)[:top_k]\n\noof = np.zeros((len(train_lr), top_k), dtype=np.float32)\npreds = np.zeros((len(test_lr), top_k), dtype=np.float32)\n\nstart = time.time()\nauc_scores = []\nfor idx_pos, label_i in enumerate(top_label_idx):\n    y_label = Y[:, label_i]\n    print(f\"\\nTraining label {idx_pos+1}/{top_k}  (global label index {label_i})\")\n    fold_scores = []\n    for fold, (tr, val) in enumerate(kf.split(train_lr)):\n        X_tr, X_val = train_lr[tr], train_lr[val]\n        y_tr, y_val = y_label[tr], y_label[val]\n\n        # lightweight solver, smaller max_iter\n        clf = LogisticRegression(max_iter=500, solver='liblinear')\n        clf.fit(X_tr, y_tr)\n\n        oof[val, idx_pos] = clf.predict_proba(X_val)[:,1]\n        preds[:, idx_pos] += clf.predict_proba(test_lr)[:,1] / n_folds\n\n        # evaluate fold\n        try:\n            fold_auc = roc_auc_score(y_val, oof[val, idx_pos])\n            fold_scores.append(fold_auc)\n        except Exception:\n            pass\n\n        # cleanup memory\n        del X_tr, X_val, y_tr, y_val, clf\n        gc.collect()\n\n    if fold_scores:\n        mean_auc = np.mean(fold_scores)\n        print(f\"Label {idx_pos+1} mean AUC: {mean_auc:.4f}\")\n        auc_scores.append(mean_auc)\n\nelapsed = time.time() - start\nprint(f\"\\nFinished LR baseline for top {top_k} labels. Time: {elapsed/60:.2f} min\")\nif auc_scores:\n    print(\"Mean AUC across trained labels:\", np.mean(auc_scores))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T14:55:59.677405Z","iopub.execute_input":"2025-10-19T14:55:59.677636Z","iopub.status.idle":"2025-10-19T14:55:59.741396Z","shell.execute_reply.started":"2025-10-19T14:55:59.677617Z","shell.execute_reply":"2025-10-19T14:55:59.740267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- REPLACEMENT for Cell 30: Memory-safe LogisticRegression baseline ---\nimport numpy as np, gc, time\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\ntrain_lr = train_red\ntest_lr  = test_red\n\nprint(\"train_lr shape:\", train_lr.shape, \" test_lr shape:\", test_lr.shape)\nn_folds = 3\nkf = KFold(n_splits=n_folds, shuffle=True, random_state=42)\n\nlabel_freq = Y.sum(axis=0)\ntop_k = min(50, Y.shape[1])   # reduce to 20 if kernel still dies\ntop_label_idx = np.argsort(-label_freq)[:top_k]\n\noof = np.zeros((len(train_lr), top_k), dtype=np.float32)\npreds = np.zeros((len(test_lr), top_k), dtype=np.float32)\n\nstart = time.time()\nauc_scores = []\nfor idx_pos, label_i in enumerate(top_label_idx):\n    y_label = Y[:, label_i]\n    print(f\"\\nTraining label {idx_pos+1}/{top_k}  (global label index {label_i})\")\n    fold_scores = []\n    for fold, (tr, val) in enumerate(kf.split(train_lr)):\n        X_tr, X_val = train_lr[tr], train_lr[val]\n        y_tr, y_val = y_label[tr], y_label[val]\n\n        clf = LogisticRegression(max_iter=500, solver='liblinear')\n        clf.fit(X_tr, y_tr)\n\n        oof[val, idx_pos] = clf.predict_proba(X_val)[:,1]\n        preds[:, idx_pos] += clf.predict_proba(test_lr)[:,1] / n_folds\n\n        try:\n            fold_auc = roc_auc_score(y_val, oof[val, idx_pos])\n            fold_scores.append(fold_auc)\n        except Exception:\n            pass\n\n        del X_tr, X_val, y_tr, y_val, clf\n        gc.collect()\n\n    if fold_scores:\n        mean_auc = np.mean(fold_scores)\n        print(f\"Label {idx_pos+1} mean AUC: {mean_auc:.4f}\")\n        auc_scores.append(mean_auc)\n\nelapsed = time.time() - start\nprint(f\"\\nFinished LR baseline for top {top_k} labels. Time: {elapsed/60:.2f} min\")\nif auc_scores:\n    print(\"Mean AUC across trained labels:\", np.mean(auc_scores))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # 7. LogisticRegression baseline\n# kf = KFold(n_splits=5, shuffle=True, random_state=42)\n# oof = np.zeros((len(train_red), Y.shape[1]))\n# preds = np.zeros((len(test_red), Y.shape[1]))\n\n# # Generate splits based on the number of samples in the label matrix Y\n# for fold, (tr, val) in enumerate(kf.split(Y)):  # Changed from kf.split(train_red) to kf.split(Y)\n#     print('Fold', fold)\n#     X_tr, X_val = train_red[tr], train_red[val]\n#     y_tr, y_val = Y[tr], Y[val]\n#     model_lr = OneVsRestClassifier(LogisticRegression(max_iter=1000))\n#     model_lr.fit(X_tr, y_tr)\n#     oof[val] = model_lr.predict_proba(X_val)\n#     preds += model_lr.predict_proba(test_red) / kf.n_splits\n\n# auc_scores = []\n# for i in range(Y.shape[1]):\n#     try: auc_scores.append(roc_auc_score(Y[:,i], oof[:,i]))\n#     except: pass\n# print('Mean AUC:', np.mean(auc_scores))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T14:55:59.742188Z","iopub.status.idle":"2025-10-19T14:55:59.742556Z","shell.execute_reply.started":"2025-10-19T14:55:59.742413Z","shell.execute_reply":"2025-10-19T14:55:59.742428Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. LightGBM","metadata":{}},{"cell_type":"code","source":"# 8. LightGBM stacking (Optimized Memory-Safe Version)\nimport numpy as np\nimport lightgbm as lgb\nimport gc, time\n\n# ✅ Reduce number of classes\nclass_counts = train_df['labels_list'].explode().value_counts()\ntop_classes = class_counts.head(10).index.tolist()  # 🔸 Top 10 only\ntop_idx = [np.where(mlb.classes_ == c)[0][0] for c in top_classes]\n\n# Parameters\nNUM_BOOST_ROUND = 50\nEARLY_STOP = 10\n\nstart_time = time.time()\nfor j, ci in enumerate(top_idx):\n    y = Y[:, ci]\n    print(f\"\\n🧠 Training class {j+1}/{len(top_idx)} (index {ci})\")\n\n    # Single fold instead of all folds → lower memory\n    tr, val = next(iter(kf.split(train_red)))\n\n    try:\n        X_tr = train_red[tr].astype(np.float32)\n        X_val = train_red[val].astype(np.float32)\n        y_tr = y[tr]\n        y_val = y[val]\n\n        dtrain = lgb.Dataset(X_tr, label=y_tr, free_raw_data=True)\n        dval = lgb.Dataset(X_val, label=y_val, reference=dtrain, free_raw_data=True)\n\n        params = {\n            'objective': 'binary',\n            'metric': 'auc',\n            'verbosity': -1,\n            'num_threads': 2,\n            'learning_rate': 0.05,\n            'num_leaves': 31,\n            'feature_fraction': 0.8,\n            'bagging_fraction': 0.8,\n            'force_col_wise': True\n        }\n\n        bst = lgb.train(\n            params,\n            dtrain,\n            num_boost_round=NUM_BOOST_ROUND,\n            valid_sets=[dval],\n            early_stopping_rounds=EARLY_STOP,\n            verbose_eval=False\n        )\n\n        preds[:, ci] += bst.predict(test_red, num_iteration=bst.best_iteration)\n\n        # Memory cleanup\n        del X_tr, X_val, y_tr, y_val, dtrain, dval, bst\n        gc.collect()\n\n    except Exception as e:\n        print(f\"⚠️ Skipped class {ci} due to error: {e}\")\n        gc.collect()\n        continue\n\nelapsed = time.time() - start_time\nprint(f\"\\n✅ LightGBM stacking done in {elapsed/60:.2f} minutes (safe mode)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T14:55:59.743853Z","iopub.status.idle":"2025-10-19T14:55:59.744181Z","shell.execute_reply.started":"2025-10-19T14:55:59.744Z","shell.execute_reply":"2025-10-19T14:55:59.744012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# there would be having problem in runing this cell becuase the kernal would have die\n# # 8. LightGBM stacking on top 50 classes\n# class_counts = train_df['labels_list'].explode().value_counts()\n# top_classes = class_counts.head(50).index.tolist()\n# top_idx = [np.where(mlb.classes_ == c)[0][0] for c in top_classes]\n\n# for j, ci in enumerate(top_idx):\n#     y = Y[:, ci]\n#     for fold, (tr, val) in enumerate(kf.split(train_red)):\n#         X_tr, X_val = train_red[tr], train_red[val]\n#         y_tr, y_val = y[tr], y[val]\n#         dtrain = lgb.Dataset(X_tr, label=y_tr)\n#         dval = lgb.Dataset(X_val, label=y_val)\n#         params = {'objective': 'binary', 'metric': 'auc', 'verbosity': -1}\n#         bst = lgb.train(params, dtrain, valid_sets=[dtrain, dval], num_boost_round=200, early_stopping_rounds=20, verbose_eval=False)\n#         preds[:, ci] += bst.predict(test_red, num_iteration=bst.best_iteration) / kf.n_splits\n# print('LightGBM stacking done')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T14:55:59.745336Z","iopub.status.idle":"2025-10-19T14:55:59.74573Z","shell.execute_reply.started":"2025-10-19T14:55:59.745524Z","shell.execute_reply":"2025-10-19T14:55:59.745543Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. submission","metadata":{}},{"cell_type":"code","source":"# 9. Create submission\nsub = sample_sub.copy()\nfor i, cname in enumerate(mlb.classes_):\n    if cname in sub.columns:\n        sub[cname] = preds[:, i]\nsub.to_csv('submission.tsv', index=False)\nprint('submission.tsv saved')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T14:55:59.746489Z","iopub.status.idle":"2025-10-19T14:55:59.746851Z","shell.execute_reply.started":"2025-10-19T14:55:59.746671Z","shell.execute_reply":"2025-10-19T14:55:59.746689Z"}},"outputs":[],"execution_count":null}]}