{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"readme_text = \"\"\"🧾 CAFA-6 Baseline: Protein Function Prediction using k-mers + TF-IDF + Logistic Regression\n\n(Predicción de la función proteica mediante k-mers + TF-IDF + Regresión Logística)\n\n⸻\n\n👤 Author / Autor\n\nFrancisco José de la Corte López\nCompetition: CAFA-6 Protein Function Prediction\nVersion: Baseline v1 – October 2025\n\n⸻\n\n🧠 Objective / Objetivo\n\nEN:\nThis notebook develops a baseline model for the CAFA-6 challenge. The goal is to predict Gene Ontology (GO) terms — Molecular Function (MF), Biological Process (BP), and Cellular Component (CC) — for each protein, based solely on its amino-acid sequence.\n\nES:\nEste notebook desarrolla un modelo base (baseline) para la competición CAFA-6. El objetivo es predecir los términos de la Ontología Génica (GO) — Función Molecular (MF), Proceso Biológico (BP) y Componente Celular (CC) — de cada proteína a partir únicamente de su secuencia de aminoácidos.\n\n⸻\n\n⚙️ Methodology / Metodología\n\nEN:\n\t1.\tData preparation: unified FASTA sequences and GO annotations into a clean train_merged.parquet.\n\t2.\tFeature extraction: split sequences into k-mers (k=3) and applied TF-IDF vectorization.\n\t3.\tModeling: trained One-Vs-Rest Logistic Regression separately for MF, BP, and CC.\n\t4.\tValidation: 80/20 protein split, evaluated with F1 (micro/macro).\n\t5.\tSubmission: top-200 GO terms per ontology combined into submission_baseline.tsv.\n\nES:\n\t1.\tPreparación de datos: unificación de secuencias FASTA y anotaciones GO en un único archivo limpio (train_merged.parquet).\n\t2.\tExtracción de características: división de las secuencias en k-mers (k=3) y aplicación de vectorización TF-IDF.\n\t3.\tModelado: entrenamiento de un modelo de Regresión Logística One-Vs-Rest para cada subontología (MF, BP y CC).\n\t4.\tValidación: división 80/20 por proteína y evaluación con F1 (micro y macro).\n\t5.\tGeneración de resultados: selección de los 200 términos GO con mayor probabilidad por ontología y combinación en submission_baseline.tsv.\n\n⸻\n\n🤖 Use of AI Assistance / Uso de Inteligencia Artificial\n\nEN:\nParts of the code and documentation were created with help from ChatGPT (GPT-5) as a coding assistant.\nAll data processing, execution, validation, and interpretation were carried out by the author.\nThis use of AI complies with Kaggle’s and CAFA’s ethical rules.\n\nES:\nParte del código y de la documentación se elaboró con el apoyo de un asistente de IA (ChatGPT – GPT-5).\nTodas las tareas de diseño, ejecución, validación e interpretación fueron realizadas por el autor.\nEl uso de IA cumple con las normas éticas de Kaggle y de la organización CAFA.\n\n⸻\n\n🔬 Next Steps / Próximos Pasos\n\nEN:\n\t•\tThreshold tuning per ontology (F-max).\n\t•\tHierarchical GO propagation (respecting DAG structure).\n\t•\tUse of pretrained embeddings (ESM-2 / ProtT5).\n\t•\tEnsemble combination of MF + BP + CC models for final submission.\n\nES:\n\t•\tAjuste de umbrales por ontología (F-max).\n\t•\tPropagación jerárquica de términos GO (respeto de la estructura DAG).\n\t•\tUso de embeddings proteicos pre-entrenados (ESM-2 / ProtT5).\n\t•\tEnsemble final combinando los modelos MF + BP + CC.\n\n⸻\n\n📜 Notes / Notas\n\nEN:\nNo post-2025 experimental data or future annotations were used. All computations rely solely on publicly available data at the competition start.\n\nES:\nNo se ha utilizado información experimental publicada después del 15 de octubre de 2025.\nTodos los cálculos se basan exclusivamente en datos públicos disponibles en la fecha de inicio de la competición.\n\n⸻\n\nwith open(\"/kaggle/working/outputs/baseline_readme.txt\", \"w\") as f:\n    f.write(readme_text)\nprint(\"✅ README guardado en /kaggle/working/outputs/baseline_readme.txt\")# ==========================================================\n\n\n# CONFIGURACIÓN INICIAL — CAFA 6 Protein Function Prediction\n# ==========================================================\n# Esta celda prepara las rutas y copia los datos de la competición\n# a una carpeta de trabajo accesible en /kaggle/working/data\n\nimport os, shutil\nfrom pathlib import Path\n\n# Ruta base de la competición montada automáticamente\nSRC = \"/kaggle/input/cafa-6-protein-function-prediction\"\nDEST = \"/kaggle/working/data\"\nos.makedirs(DEST, exist_ok=True)\n\n# Copiamos y renombramos los archivos clave\nmapping = {\n    f\"{SRC}/Train/train_sequences.fasta\": f\"{DEST}/train.fasta\",\n    f\"{SRC}/Train/train_terms.tsv\": f\"{DEST}/train_annotations.tsv\",\n    f\"{SRC}/Test/testsuperset.fasta\": f\"{DEST}/test_superset.fasta\",\n    f\"{SRC}/IA.tsv\": f\"{DEST}/IA.tsv\",\n}\nfor src, dst in mapping.items():\n    if os.path.exists(src):\n        shutil.copy(src, dst)\n\n# Mostramos lo que hay en la carpeta de trabajo\nprint(\"✅ Datos preparados en:\", DEST)\n!ls -lh $DEST","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================\n# EXPLORACIÓN INICIAL DE train_annotations.tsv\n# ===============================================\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nDATA_DIR = \"/kaggle/working/data\"\n\n# Cargar las anotaciones\nann = pd.read_csv(f\"{DATA_DIR}/train_annotations.tsv\", sep='\\t', header=None,\n                  names=['protein_id', 'go_id', 'ontology'])\n\nprint(\"✅ Anotaciones cargadas:\", ann.shape)\nprint(ann.head())\n\n# Resumen general\nn_proteins = ann['protein_id'].nunique()\nn_terms = ann['go_id'].nunique()\nprint(f\"\\nNúmero de proteínas únicas: {n_proteins:,}\")\nprint(f\"Número de términos GO únicos: {n_terms:,}\")\n\n# Conteo por subontología\nont_count = ann['ontology'].value_counts()\nprint(\"\\nAnotaciones por subontología:\")\nprint(ont_count)\n\n# Visualización rápida\nont_count.plot(kind='bar', color=['#5DADE2', '#58D68D', '#F5B041'])\nplt.title(\"Distribución de anotaciones por subontología (MF / BP / CC)\")\nplt.ylabel(\"Número de anotaciones\")\nplt.show()\n\n# Conteo de términos por frecuencia (cola larga)\nterm_freq = ann['go_id'].value_counts()\nprint(\"\\nEjemplo de términos más frecuentes:\")\nprint(term_freq.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:20:58.634985Z","iopub.execute_input":"2025-10-20T09:20:58.635372Z","iopub.status.idle":"2025-10-20T09:20:59.886474Z","shell.execute_reply.started":"2025-10-20T09:20:58.635342Z","shell.execute_reply":"2025-10-20T09:20:59.885509Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Métrica\nValor\n✅ Anotaciones cargadas\n537.028 filas (protein–GO–ontology)\n🧬 Proteínas únicas\n82.405\n🧠 Términos GO únicos\n26.126\n📊 Subontologías\nP (Biological Process) → 250.805  C (Cellular Component) → 157.770  F (Molecular Function) → 128.452\nY la gráfica de barras muestra justo eso:\nBP domina (P), seguida por CC (C) y MF (F).\nAdemás, los términos GO más frecuentes (GO:0005515, GO:005634, etc.) coinciden con lo esperado:\n\t•\tGO:0005515 = protein binding → uno de los términos más comunes de la ontología MF.\nAsí que el análisis es coherente 💯.","metadata":{}},{"cell_type":"code","source":"# ===============================================\n# EXPLORACIÓN DE SECUENCIAS (train.fasta)\n# ===============================================\nimport gzip\nimport numpy as np\n\ndef read_fasta(fp):\n    \"\"\"Generador simple de FASTA (soporta .gz)\"\"\"\n    opener = gzip.open if str(fp).endswith('.gz') else open\n    header, seq = None, []\n    with opener(fp, 'rt') as f:\n        for line in f:\n            line = line.strip()\n            if not line:\n                continue\n            if line.startswith('>'):\n                if header:\n                    yield header, ''.join(seq)\n                header = line[1:].split()[0]\n                seq = []\n            else:\n                seq.append(line)\n        if header:\n            yield header, ''.join(seq)\n\nlengths = []\nfor _, seq in read_fasta(f\"{DATA_DIR}/train.fasta\"):\n    lengths.append(len(seq))\n\nprint(f\"✅ Secuencias leídas: {len(lengths):,}\")\nprint(f\"Longitud mínima: {np.min(lengths)}\")\nprint(f\"Longitud máxima: {np.max(lengths)}\")\nprint(f\"Longitud media: {np.mean(lengths):.1f}\")\nprint(f\"Mediana: {np.median(lengths):.1f}\")\n\nplt.hist(lengths, bins=50, color=\"#5DADE2\", edgecolor=\"black\")\nplt.title(\"Distribución de longitudes de secuencia (train.fasta)\")\nplt.xlabel(\"Longitud (aminoácidos)\")\nplt.ylabel(\"Número de proteínas\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:24:53.067425Z","iopub.execute_input":"2025-10-20T09:24:53.067793Z","iopub.status.idle":"2025-10-20T09:24:53.944598Z","shell.execute_reply.started":"2025-10-20T09:24:53.067758Z","shell.execute_reply":"2025-10-20T09:24:53.943624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================================\n# UNIÓN DE SECUENCIAS Y ANOTACIONES\n# ==========================================================\nfrom tqdm import tqdm\n\n# Convertimos el FASTA a DataFrame\nrecords = []\nfor pid, seq in tqdm(read_fasta(f\"{DATA_DIR}/train.fasta\")):\n    records.append((pid, seq))\ndf_seq = pd.DataFrame(records, columns=[\"protein_id\", \"sequence\"])\nprint(\"Proteínas con secuencia:\", len(df_seq))\n\n# Comprobamos que todas las proteínas de las anotaciones están en el FASTA\nmissing = set(ann['protein_id']) - set(df_seq['protein_id'])\nprint(f\"Proteínas anotadas sin secuencia: {len(missing)}\")\n\n# Unimos las anotaciones (uno-a-muchos)\ndf_merged = ann.merge(df_seq, on=\"protein_id\", how=\"left\")\n\nprint(\"\\nDimensiones del dataframe final:\", df_merged.shape)\nprint(df_merged.head())\n\n# Guardamos para futuros pasos\ndf_merged.to_parquet(f\"{DATA_DIR}/train_merged.parquet\", index=False)\nprint(\"✅ Guardado train_merged.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:26:29.232028Z","iopub.execute_input":"2025-10-20T09:26:29.232344Z","iopub.status.idle":"2025-10-20T09:26:30.394182Z","shell.execute_reply.started":"2025-10-20T09:26:29.23232Z","shell.execute_reply":"2025-10-20T09:26:30.393133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Releer anotaciones indicando que SÍ tienen cabecera\nimport pandas as pd\n\nDATA_DIR = \"/kaggle/working/data\"\n\nann = pd.read_csv(f\"{DATA_DIR}/train_annotations.tsv\",\n                  sep=\"\\t\", header=0,  # <- hay cabecera\n                  names=[\"protein_id\",\"go_id\",\"ontology\"])  # nombres unificados\n\n# Limpieza básica\nann = ann.dropna(subset=[\"protein_id\",\"go_id\",\"ontology\"])\nann = ann.astype({\"protein_id\": str, \"go_id\": str, \"ontology\": str})\n\nprint(\"✅ Anotaciones recargadas:\", ann.shape)\nprint(ann.head())\n\nprint(\"\\nRecuento por ontología (ya sin 'aspect'):\")\nprint(ann[\"ontology\"].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:28:32.561907Z","iopub.execute_input":"2025-10-20T09:28:32.562293Z","iopub.status.idle":"2025-10-20T09:28:33.029329Z","shell.execute_reply.started":"2025-10-20T09:28:32.562268Z","shell.execute_reply":"2025-10-20T09:28:33.028204Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gzip, re\nfrom tqdm import tqdm\n\ndef read_fasta_with_header(fp):\n    \"\"\"Devuelve (header_completo_sin_'>', secuencia)\"\"\"\n    opener = gzip.open if str(fp).endswith(\".gz\") else open\n    header, seq = None, []\n    with opener(fp, \"rt\") as f:\n        for line in f:\n            line = line.strip()\n            if not line:\n                continue\n            if line.startswith(\">\"):\n                if header:\n                    yield header, \"\".join(seq)\n                header = line[1:]  # sin '>'\n                seq = []\n            else:\n                seq.append(line)\n        if header:\n            yield header, \"\".join(seq)\n\ndef extract_uniprot_accession(header: str) -> str:\n    \"\"\"\n    Intenta extraer el accesión UniProt del header.\n    Casos típicos: 'sp|Q5W0B1|...', 'tr|A0A123|...', o encabezados simples.\n    \"\"\"\n    first_token = header.split()[0]\n    # Si hay tuberías, busca un token que parezca accesión\n    if \"|\" in first_token:\n        for part in first_token.split(\"|\"):\n            if re.fullmatch(r\"[A-NR-Z0-9]{6,10}\", part):  # patrón accesión\n                return part\n        # si no encontró, devuelve el último trozo\n        return first_token.split(\"|\")[-1]\n    # Sin tuberías: intenta extraer un patrón de accesión\n    m = re.search(r\"[A-NR-Z0-9]{6,10}\", first_token)\n    return m.group(0) if m else first_token\n\n# Convertimos el FASTA a DataFrame usando el accesión\nrecords = []\nfor header, seq in tqdm(read_fasta_with_header(f\"{DATA_DIR}/train.fasta\")):\n    acc = extract_uniprot_accession(header)\n    records.append((acc, seq))\n\ndf_seq = pd.DataFrame(records, columns=[\"protein_id\",\"sequence\"])\nprint(\"✅ Proteínas con secuencia (IDs parseados):\", len(df_seq))\nprint(df_seq.head())\n\n# Comprobación de cobertura\nmissing = set(ann[\"protein_id\"]) - set(df_seq[\"protein_id\"])\nprint(\"Proteínas anotadas sin secuencia (debería ser ~0):\", len(missing))\n\n# Unión correcta\ndf_merged = ann.merge(df_seq, on=\"protein_id\", how=\"left\")\nprint(\"\\nDimensiones del dataframe final:\", df_merged.shape)\nprint(df_merged.head())\n\n# Guardar limpio\ndf_merged.to_parquet(f\"{DATA_DIR}/train_merged.parquet\", index=False)\nprint(\"✅ Guardado train_merged.parquet (IDs alineados)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:28:50.144633Z","iopub.execute_input":"2025-10-20T09:28:50.144984Z","iopub.status.idle":"2025-10-20T09:28:51.489798Z","shell.execute_reply.started":"2025-10-20T09:28:50.144951Z","shell.execute_reply":"2025-10-20T09:28:51.488792Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re, gzip\nfrom tqdm import tqdm\nimport pandas as pd\n\nDATA_DIR = \"/kaggle/working/data\"\n\n# Patrones de accesión UniProt: 6 caracteres (clásico) o 10 (A0A…)\nACC_REGEX = re.compile(r\"(?:[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9][A-Z0-9]{3}[0-9]|[A-Z0-9]{10})\")\n\ndef read_fasta_headers(fp):\n    opener = gzip.open if str(fp).endswith(\".gz\") else open\n    header, seq = None, []\n    with opener(fp, \"rt\") as f:\n        for line in f:\n            line = line.strip()\n            if not line:\n                continue\n            if line.startswith(\">\"):\n                if header:\n                    yield header, \"\".join(seq)\n                header = line[1:]\n                seq = []\n            else:\n                seq.append(line)\n        if header:\n            yield header, \"\".join(seq)\n\ndef extract_accession(header: str) -> str:\n    \"\"\"\n    Reglas:\n    1) Si hay tuberías (sp|ACC|ENTRY / tr|ACC|ENTRY), toma el campo que cumpla ACC_REGEX.\n    2) Si no, busca el primer match ACC_REGEX en TODO el header.\n    3) Si falla, devuelve el primer token (caso residual).\n    \"\"\"\n    # Caso con tuberías\n    if \"|\" in header:\n        for part in header.split(\"|\"):\n            m = ACC_REGEX.fullmatch(part)\n            if m:\n                return m.group(0)\n    # Búsqueda en todo el header\n    m = ACC_REGEX.search(header)\n    if m:\n        return m.group(0)\n    # Fallback: primer token\n    return header.split()[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:30:55.747584Z","iopub.execute_input":"2025-10-20T09:30:55.748125Z","iopub.status.idle":"2025-10-20T09:30:55.762289Z","shell.execute_reply.started":"2025-10-20T09:30:55.748095Z","shell.execute_reply":"2025-10-20T09:30:55.761216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# FASTA -> DataFrame con accesión correcto\nrecords = []\nfor header, seq in tqdm(read_fasta_headers(f\"{DATA_DIR}/train.fasta\")):\n    acc = extract_accession(header)\n    records.append((acc, seq))\ndf_seq = pd.DataFrame(records, columns=[\"protein_id\",\"sequence\"])\nprint(\"✅ Proteínas con secuencia (IDs parseados):\", len(df_seq))\nprint(df_seq.head())\n\n# Recargar anotaciones con cabecera real (por si no lo hiciste antes)\nann = pd.read_csv(f\"{DATA_DIR}/train_annotations.tsv\",\n                  sep=\"\\t\", header=0,\n                  names=[\"protein_id\",\"go_id\",\"ontology\"])\nann = ann.dropna(subset=[\"protein_id\",\"go_id\",\"ontology\"]).astype(str)\n\n# Comprobación de cobertura\nmissing = set(ann[\"protein_id\"]) - set(df_seq[\"protein_id\"])\nprint(\"Proteínas anotadas sin secuencia (objetivo ~0):\", len(missing))\n\n# Unir\ndf_merged = ann.merge(df_seq, on=\"protein_id\", how=\"left\")\nprint(\"\\nDimensiones del dataframe final:\", df_merged.shape)\nprint(df_merged.head())\n\n# Guardar limpio\ndf_merged.to_parquet(f\"{DATA_DIR}/train_merged.parquet\", index=False)\nprint(\"✅ Guardado train_merged.parquet (IDs alineados de verdad)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:31:16.656476Z","iopub.execute_input":"2025-10-20T09:31:16.656832Z","iopub.status.idle":"2025-10-20T09:31:19.386793Z","shell.execute_reply.started":"2025-10-20T09:31:16.656807Z","shell.execute_reply":"2025-10-20T09:31:19.385756Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"HASTA AHORA TENEMOS:Fase\nEstado\nArchivo resultante\nDescarga y organización de datos\n✅ completada\n/kaggle/working/data\nEDA de anotaciones (MF/BP/CC)\n✅ completada\ntrain_annotations.tsv\nEDA de secuencias (longitudes, distribución)\n✅ completada\ntrain.fasta\nUnión limpia (proteína + GO + ontología + secuencia)\n✅ completada\ntrain_merged.parquet\n","metadata":{}},{"cell_type":"code","source":"# ===============================================\n# HELPERS: k-mers, corpus y utilidades\n# ===============================================\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nfrom collections import defaultdict, Counter\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score, precision_score, recall_score\n\nDATA_DIR = Path(\"/kaggle/working/data\")\nOUT_DIR  = Path(\"/kaggle/working/outputs\"); OUT_DIR.mkdir(exist_ok=True, parents=True)\n\ndef seq_to_kmers(seq: str, k=3):\n    return [seq[i:i+k] for i in range(0, max(0, len(seq)-k+1))]\n\ndef make_docs(df_seq: pd.DataFrame, k=3):\n    # df_seq: columnas ['protein_id','sequence']\n    docs = []\n    ids  = []\n    for pid, seq in zip(df_seq['protein_id'].values, df_seq['sequence'].values):\n        ids.append(pid)\n        docs.append(\" \".join(seq_to_kmers(seq, k=k)))\n    return ids, docs\n\ndef df_labels_from_ann(df_ann: pd.DataFrame, ontology: str):\n    sub = df_ann[df_ann['ontology'] == ontology].copy()\n    labels = sub.groupby('protein_id')['go_id'].apply(set).reset_index()\n    return labels  # columnas: protein_id | set(go_ids)\n\ndef split_by_protein(df_seq: pd.DataFrame, random_state=42, test_size=0.2):\n    # split simple a nivel de proteína\n    train_ids, valid_ids = train_test_split(\n        df_seq['protein_id'].unique(), test_size=test_size, random_state=random_state, shuffle=True\n    )\n    return set(train_ids), set(valid_ids)\n\ndef show_scores(y_true, y_prob, thr=0.5, name=\"\"):\n    y_pred = (y_prob >= thr).astype(int)\n    micro = f1_score(y_true, y_pred, average=\"micro\", zero_division=0)\n    macro = f1_score(y_true, y_pred, average=\"macro\", zero_division=0)\n    prec  = precision_score(y_true, y_pred, average=\"micro\", zero_division=0)\n    rec   = recall_score(y_true, y_pred, average=\"micro\", zero_division=0)\n    print(f\"[{name}] F1 micro={micro:.4f} | F1 macro={macro:.4f} | P={prec:.4f} | R={rec:.4f}\")\n    return {\"f1_micro\": micro, \"f1_macro\": macro, \"prec\": prec, \"rec\": rec}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:41:31.742242Z","iopub.execute_input":"2025-10-20T09:41:31.742571Z","iopub.status.idle":"2025-10-20T09:41:32.68455Z","shell.execute_reply.started":"2025-10-20T09:41:31.742551Z","shell.execute_reply":"2025-10-20T09:41:32.683613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================\n# CARGA DE DATOS UNIFICADOS (hecho en pasos previos)\n# ===============================================\ndf = pd.read_parquet(DATA_DIR / \"train_merged.parquet\")\nprint(df.shape, df.head())\n\n# Creamos dataframe de secuencias únicas\ndf_seq = df[['protein_id','sequence']].drop_duplicates().reset_index(drop=True)\nprint(\"Proteínas (únicas) con secuencia:\", df_seq.shape[0])\n\n# Probaremos k=3 (rápido y robusto). Puedes cambiar a k=4 más adelante.\nKMER = 3\n\n# Split por proteína\ntrain_ids, valid_ids = split_by_protein(df_seq, test_size=0.2, random_state=42)\nprint(\"Train proteins:\", len(train_ids), \" | Valid proteins:\", len(valid_ids))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:41:53.983095Z","iopub.execute_input":"2025-10-20T09:41:53.983604Z","iopub.status.idle":"2025-10-20T09:41:55.977682Z","shell.execute_reply.started":"2025-10-20T09:41:53.983572Z","shell.execute_reply":"2025-10-20T09:41:55.976821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================\n# ENTRENAMIENTO + VALIDACIÓN POR SUBONTOLOGÍA\n# ===============================================\nfrom collections import namedtuple\nModelPack = namedtuple(\"ModelPack\", [\"vectorizer\",\"clf\",\"mlb\",\"terms\"])\n\nmodels = {}\nmetrics = {}\n\nfor ont, ont_name in [(\"F\",\"MF\"), (\"P\",\"BP\"), (\"C\",\"CC\")]:\n    print(\"\\n\",\"=\"*20, f\"{ont_name}\", \"=\"*20)\n    # Etiquetas por proteína en esta ontología\n    lab = df_labels_from_ann(df, ontology=ont)\n    # Merge con secuencias\n    sub = df_seq.merge(lab, on=\"protein_id\", how=\"left\")\n    sub['go_id'] = sub['go_id'].apply(lambda x: x if isinstance(x, set) else set())\n    \n    # Split\n    train_df = sub[sub['protein_id'].isin(train_ids)].reset_index(drop=True)\n    valid_df = sub[sub['protein_id'].isin(valid_ids)].reset_index(drop=True)\n    print(\"train:\", train_df.shape, \"valid:\", valid_df.shape)\n    \n    # Documentos k-mer\n    tr_ids, tr_docs = make_docs(train_df[['protein_id','sequence']], k=KMER)\n    va_ids, va_docs = make_docs(valid_df[['protein_id','sequence']], k=KMER)\n    \n    # Vectorizador TF-IDF (control de tamaño)\n    tfidf = TfidfVectorizer(min_df=2, max_df=0.9, max_features=200_000)\n    X_tr = tfidf.fit_transform(tr_docs)\n    X_va = tfidf.transform(va_docs)\n    \n    # Binarizador multilabel\n    mlb = MultiLabelBinarizer(sparse_output=False)\n    Y_tr = mlb.fit_transform(train_df['go_id'].tolist())\n    Y_va = mlb.transform(valid_df['go_id'].tolist())\n    print(\"n_labels:\", Y_tr.shape[1])\n    \n    # Modelo (One-vs-Rest LR con solver liblinear/saga según tamaño)\n    clf = OneVsRestClassifier(\n        LogisticRegression(max_iter=200, n_jobs=1, solver=\"liblinear\")\n    )\n    clf.fit(X_tr, Y_tr)\n    \n    # Validación simple (umbral 0.5; luego haremos tuning)\n    Yp_va = clf.predict_proba(X_va)\n    metrics[ont] = show_scores(Y_va, Yp_va, thr=0.5, name=f\"{ont_name}\")\n    \n    models[ont] = ModelPack(vectorizer=tfidf, clf=clf, mlb=mlb, terms=list(mlb.classes_))\n\nmetrics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-20T09:42:13.72347Z","iopub.execute_input":"2025-10-20T09:42:13.723812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================\n# PREDICCIÓN EN TEST + ARCHIVO DE ENVÍO\n# ===============================================\nimport gzip, re\n\nTEST_FASTA = DATA_DIR / \"test_superset.fasta\"\nSUBMIT_FP  = OUT_DIR / \"submission_baseline.tsv\"\n\n# Reutilizamos el lector de FASTA con extracción de accesión (del paso previo)\nACC_REGEX = re.compile(r\"(?:[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9][A-Z0-9]{3}[0-9]|[A-Z0-9]{10})\")\n\ndef read_fasta_headers(fp):\n    opener = gzip.open if str(fp).endswith(\".gz\") else open\n    header, seq = None, []\n    with opener(fp, \"rt\") as f:\n        for line in f:\n            line = line.strip()\n            if not line: \n                continue\n            if line.startswith(\">\"):\n                if header:\n                    yield header, \"\".join(seq)\n                header = line[1:]\n                seq = []\n            else:\n                seq.append(line)\n        if header:\n            yield header, \"\".join(seq)\n\ndef extract_accession(header: str) -> str:\n    if \"|\" in header:\n        for part in header.split(\"|\"):\n            if ACC_REGEX.fullmatch(part):\n                return part\n    m = ACC_REGEX.search(header)\n    if m:\n        return m.group(0)\n    return header.split()[0]\n\n# Construimos documentos test (una pasada)\ntest_records = []\nfor h, s in tqdm(read_fasta_headers(TEST_FASTA)):\n    acc = extract_accession(h)\n    test_records.append((acc, s))\ndf_test = pd.DataFrame(test_records, columns=[\"protein_id\",\"sequence\"])\nprint(\"Test proteins:\", df_test.shape)\n\n# Predicción por ontología\nper_protein_scores = defaultdict(list)  # protein_id -> list[(go_id, score)]\n\nTOP_K_PER_ONTO = 200  # puedes ajustar (el límite global de Kaggle es 1500 por proteína sumando MF/BP/CC)\n\nfor ont, pack in models.items():\n    print(f\"Predicting {ont} ...\")\n    ids, docs = make_docs(df_test, k=KMER)\n    X = pack.vectorizer.transform(docs)\n    Yp = pack.clf.predict_proba(X)  # shape: [n_prot, n_terms_ont]\n    \n    # Para cada proteína, quedarnos con top-k de esta ontología\n    for i, pid in enumerate(ids):\n        row = Yp[i]\n        if row.ndim == 0:  # corner case\n            continue\n        top_idx = np.argsort(-row)[:TOP_K_PER_ONTO]\n        for j in top_idx:\n            score = float(row[j])\n            if score <= 0.0: \n                continue\n            go_id = pack.terms[j]\n            per_protein_scores[pid].append((go_id, score))\n\n# Escribir submission (tab-separated, score con hasta 3 cifras significativas, sin header)\nn_lines = 0\nwith open(SUBMIT_FP, \"w\", encoding=\"utf-8\") as f:\n    for pid, items in per_protein_scores.items():\n        # ordenar globalmente (MF+BP+CC juntos) y podar duro si quisieras (ej. 600)\n        items.sort(key=lambda x: x[1], reverse=True)\n        for go_id, score in items:\n            s = f\"{score:.3g}\"\n            if float(s) <= 0.0:\n                s = \"1e-6\"\n            f.write(f\"{pid}\\t{go_id}\\t{s}\\n\")\n            n_lines += 1\n\nprint(f\"✅ Submission escrito: {SUBMIT_FP} ({n_lines:,} líneas)\")\n!head -n 10 {SUBMIT_FP}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Guardar métricas y parámetros del baseline\npd.DataFrame(metrics).to_csv(OUT_DIR / \"baseline_metrics.csv\")\nwith open(OUT_DIR / \"baseline_readme.txt\",\"w\") as w:\n    w.write(\n        f\"Baseline k-mer={KMER}, TF-IDF(max_features=200k), OvR-LogReg(liblinear)\\n\"\n        f\"TOP_K_PER_ONTO={TOP_K_PER_ONTO}\\n\"\n        f\"Valid split: 80/20 by protein\\n\"\n    )\nprint(\"Artefactos guardados en:\", OUT_DIR)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================\n# TUNING DE UMBRAL (F-max) POR SUBONTOLOGÍA\n# ===============================================\nimport numpy as np\nfrom sklearn.metrics import f1_score\n\nbest_thresholds = {}\n\nfor ont, pack in models.items():\n    print(f\"\\nTuning threshold for {ont} ...\")\n    \n    # Obtener validación\n    lab = df_labels_from_ann(df, ontology=ont)\n    sub = df_seq.merge(lab, on=\"protein_id\", how=\"left\")\n    sub['go_id'] = sub['go_id'].apply(lambda x: x if isinstance(x, set) else set())\n    valid_df = sub[sub['protein_id'].isin(valid_ids)].reset_index(drop=True)\n    ids, docs = make_docs(valid_df[['protein_id','sequence']], k=KMER)\n    \n    X_val = pack.vectorizer.transform(docs)\n    Y_true = pack.mlb.transform(valid_df['go_id'].tolist())\n    Y_prob = pack.clf.predict_proba(X_val)\n    \n    best_f1 = 0\n    best_thr = 0.5\n    for thr in np.linspace(0.1, 0.9, 17):  # prueba cada 0.05\n        Y_pred = (Y_prob >= thr).astype(int)\n        f1 = f1_score(Y_true, Y_pred, average=\"micro\", zero_division=0)\n        if f1 > best_f1:\n            best_f1, best_thr = f1, thr\n    best_thresholds[ont] = {\"thr\": best_thr, \"f1_val\": best_f1}\n    print(f\"Best threshold for {ont}: {best_thr:.2f} → F1={best_f1:.4f}\")\n\nbest_thresholds","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============================================\n# PROPAGACIÓN JERÁRQUICA SIMPLIFICADA\n# ===============================================\n# NOTA: esta es una versión aproximada sin DAG externo (útil como baseline jerárquico)\n# Más adelante podemos usar obonet para cargar el GO completo.\n\nfrom collections import defaultdict\n\n# Simulación de un \"mapa padre\" basado en IA.tsv (prepararemos el real en la siguiente fase)\nparent_map = defaultdict(list)\n# parent_map['GO:0005524'] = ['GO:0000166']  # ejemplo manual\n\ndef propagate_predictions(pred_dict, parent_map):\n    \"\"\"propaga puntuaciones desde los hijos a los padres\"\"\"\n    for pid, items in pred_dict.items():\n        scores = dict(items)\n        for go_id, score in list(scores.items()):\n            for parent in parent_map.get(go_id, []):\n                scores[parent] = max(scores.get(parent, 0), score)\n        pred_dict[pid] = list(scores.items())\n    return pred_dict","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}