{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"},{"sourceId":13965540,"sourceType":"datasetVersion","datasetId":8886753}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:21:20.075115Z","iopub.execute_input":"2025-12-14T14:21:20.075849Z","iopub.status.idle":"2025-12-14T14:21:21.708903Z","shell.execute_reply.started":"2025-12-14T14:21:20.075818Z","shell.execute_reply":"2025-12-14T14:21:21.708108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nprint(sys.version)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:21:21.710411Z","iopub.execute_input":"2025-12-14T14:21:21.710851Z","iopub.status.idle":"2025-12-14T14:21:21.714920Z","shell.execute_reply.started":"2025-12-14T14:21:21.710830Z","shell.execute_reply":"2025-12-14T14:21:21.714241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install obonet biopython matplotlib seaborn torch pandas numpy tqdm scikit-learn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:21:21.715852Z","iopub.execute_input":"2025-12-14T14:21:21.716123Z","iopub.status.idle":"2025-12-14T14:22:37.589427Z","shell.execute_reply.started":"2025-12-14T14:21:21.716099Z","shell.execute_reply":"2025-12-14T14:22:37.588469Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport warnings\nimport random\nfrom tqdm import tqdm \nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom Bio import SeqIO  # Đọc file FASTA\nimport networkx as nx   # Xử lý đồ thị (file .obo)\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom sklearn.metrics import f1_score, accuracy_score, classification_report\nwarnings.filterwarnings('ignore') \n\n# Hiển thị tối đa cột của Pandas để dễ nhìn\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', 50)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:37.590627Z","iopub.execute_input":"2025-12-14T14:22:37.590929Z","iopub.status.idle":"2025-12-14T14:22:48.549823Z","shell.execute_reply.started":"2025-12-14T14:22:37.590897Z","shell.execute_reply":"2025-12-14T14:22:48.549212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Protein id n Axitamin sequences\nprotein_ids = []\naxitamins = []\naxitamins_length = []\n\ntrain_sequences_filepath = \"/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta\"\nfor record in SeqIO.parse(train_sequences_filepath, \"fasta\"):\n    protein_id = record.id.split('|')[1] if len(record.id.split('|')) > 1 else record.id\n    protein_ids.append(protein_id)\n    \n    axitamins.append(str(record.seq))\n    \n    axitamins_length.append(len(record.seq))\n\ndf_sequences = pd.DataFrame({'id': protein_ids, 'axit amin': axitamins, 'aa length': axitamins_length})\nprint(\"Tổng số protein:\", len(df_sequences))\nprint(df_sequences.head(12))\nprint('\\n')\nprint(\"Thống kê theo aa_length:\")\nprint(df_sequences['aa length'].describe().apply(lambda x: format(x, 'f')))\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:48.551823Z","iopub.execute_input":"2025-12-14T14:22:48.552222Z","iopub.status.idle":"2025-12-14T14:22:49.679226Z","shell.execute_reply.started":"2025-12-14T14:22:48.552205Z","shell.execute_reply":"2025-12-14T14:22:49.678419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"coverage_1024 = (df_sequences['aa length'] <= 1024).mean() * 100\nprint(f\"Tỷ lệ protein có độ dài <= 1024 là: {coverage_1024:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:49.680031Z","iopub.execute_input":"2025-12-14T14:22:49.680248Z","iopub.status.idle":"2025-12-14T14:22:49.685083Z","shell.execute_reply.started":"2025-12-14T14:22:49.680223Z","shell.execute_reply":"2025-12-14T14:22:49.684529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#TRỰC QUAN HÓA\nlimit_len = 1024\ncoverage = (df_sequences['aa length'] <= limit_len).mean() * 100\nfig, (ax_box, ax_hist) = plt.subplots(2, 1, figsize=(12, 8), \n                                      sharex=True,\n                                      gridspec_kw={\"height_ratios\": (.15, .85)})\nsns.boxplot(data=df_sequences, x='aa length', ax=ax_box, color='lightblue', fliersize=2)\nax_box.set(xlabel='')\nax_box.set_title('Phân phối độ dài chuỗi axit amin (Boxplot & Histogram)', fontsize=14, fontweight='bold')\nsns.histplot(data=df_sequences, x='aa length', ax=ax_hist, \n             bins=100, kde=True,\n             log_scale=True,     \n             color='teal', edgecolor='black')\nmean_val = df_sequences['aa length'].mean()\nmedian_val = df_sequences['aa length'].median()\nax_hist.axvline(limit_len, color='orange', linestyle='-.', linewidth=2, \n                label=f'Cutoff {limit_len} (Cover: {coverage:.2f}%)')\ny_min, y_max = ax_hist.get_ylim()\nax_hist.fill_betweenx([y_min, y_max], 0, limit_len, color='orange', alpha=0.1)\n\nax_hist.axvline(mean_val, color='red', linestyle='--', label=f'Mean: {mean_val:.0f}')\nax_hist.axvline(median_val, color='green', linestyle='-', label=f'Median: {median_val:.0f}')\nplt.xlabel('Độ dài chuỗi (Log Scale)', fontsize=12)\nplt.ylabel('Số lượng Protein', fontsize=12)\nplt.legend(loc='upper right', frameon=True, shadow=True)\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:49.685867Z","iopub.execute_input":"2025-12-14T14:22:49.686062Z","iopub.status.idle":"2025-12-14T14:22:51.089815Z","shell.execute_reply.started":"2025-12-14T14:22:49.686047Z","shell.execute_reply":"2025-12-14T14:22:51.089168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Protein id & taxon id\ntrain_taxonomy_filepath = '/kaggle/input/cafa-6-protein-function-prediction/Train/train_taxonomy.tsv'\ndf_taxonomy = pd.read_csv(train_taxonomy_filepath, sep='\\t',\n                         header=None, names=['id', 'taxon'])\nprint(df_taxonomy.head(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:51.090548Z","iopub.execute_input":"2025-12-14T14:22:51.090829Z","iopub.status.idle":"2025-12-14T14:22:51.145442Z","shell.execute_reply.started":"2025-12-14T14:22:51.090810Z","shell.execute_reply":"2025-12-14T14:22:51.144863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_species = df_taxonomy['taxon'].value_counts().head(10)\n\nprint(\"Tổng số protein trong file taxonomy:\", len(df_taxonomy))\nprint(\"\\nTop 10 loài phổ biến nhất:\")\nprint(top_species)\n\n# 3. Vẽ biểu đồ\nplt.figure(figsize=(12, 6))\nsns.barplot(x=top_species.index.astype(str), y=top_species.values, palette='viridis')\nplt.title('Top 10 Loài sinh vật phổ biến nhất (Toàn bộ tập Train)')\nplt.xlabel('Mã loài (Taxon ID)')\nplt.ylabel('Số lượng Protein')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:51.146307Z","iopub.execute_input":"2025-12-14T14:22:51.146889Z","iopub.status.idle":"2025-12-14T14:22:51.338220Z","shell.execute_reply.started":"2025-12-14T14:22:51.146860Z","shell.execute_reply":"2025-12-14T14:22:51.337650Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Merge taxon & sequence\ndf_full = pd.merge(df_sequences, df_taxonomy, on = 'id', how='left')\ndf_full","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:51.339037Z","iopub.execute_input":"2025-12-14T14:22:51.339296Z","iopub.status.idle":"2025-12-14T14:22:51.402930Z","shell.execute_reply.started":"2025-12-14T14:22:51.339277Z","shell.execute_reply":"2025-12-14T14:22:51.402126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_terms = pd.read_csv(\"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\", sep=\"\\t\")\nprint(df_train_terms.head())\n# cột sẽ là: EntryID, term, aspect\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:51.404372Z","iopub.execute_input":"2025-12-14T14:22:51.404627Z","iopub.status.idle":"2025-12-14T14:22:51.686181Z","shell.execute_reply.started":"2025-12-14T14:22:51.404608Z","shell.execute_reply":"2025-12-14T14:22:51.685539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train_terms = df_train_terms.rename(columns={\n    \"EntryID\": \"id\",\n    \"term\": \"go_term\",\n    \"aspect\": \"aspect\"\n})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:51.686976Z","iopub.execute_input":"2025-12-14T14:22:51.687243Z","iopub.status.idle":"2025-12-14T14:22:51.708611Z","shell.execute_reply.started":"2025-12-14T14:22:51.687215Z","shell.execute_reply":"2025-12-14T14:22:51.707813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# MF (F = Function)\ndf_mf = (\n    df_train_terms[df_train_terms[\"aspect\"] == \"F\"]\n    .groupby(\"id\")[\"go_term\"]\n    .apply(list)\n    .reset_index()\n    .rename(columns={\"go_term\": \"MF_raw\"})\n)\n\n# BP (P = Process)\ndf_bp = (\n    df_train_terms[df_train_terms[\"aspect\"] == \"P\"]\n    .groupby(\"id\")[\"go_term\"]\n    .apply(list)\n    .reset_index()\n    .rename(columns={\"go_term\": \"BP_raw\"})\n)\n\n# CC (C = Component)\ndf_cc = (\n    df_train_terms[df_train_terms[\"aspect\"] == \"C\"]\n    .groupby(\"id\")[\"go_term\"]\n    .apply(list)\n    .reset_index()\n    .rename(columns={\"go_term\": \"CC_raw\"})\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:51.709265Z","iopub.execute_input":"2025-12-14T14:22:51.709448Z","iopub.status.idle":"2025-12-14T14:22:54.620861Z","shell.execute_reply.started":"2025-12-14T14:22:51.709426Z","shell.execute_reply":"2025-12-14T14:22:54.620203Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# df_full: lấy từ train_sequences.fasta + train_taxonomy.tsv\n# cột: id | axit amin | aa length | taxon\n\ndf_merged = df_full.copy()\ndf_merged = df_merged.merge(df_mf, on=\"id\", how=\"left\")\ndf_merged = df_merged.merge(df_bp, on=\"id\", how=\"left\")\ndf_merged = df_merged.merge(df_cc, on=\"id\", how=\"left\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:54.623767Z","iopub.execute_input":"2025-12-14T14:22:54.624026Z","iopub.status.idle":"2025-12-14T14:22:54.775822Z","shell.execute_reply.started":"2025-12-14T14:22:54.624006Z","shell.execute_reply":"2025-12-14T14:22:54.775205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_merged","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:54.776689Z","iopub.execute_input":"2025-12-14T14:22:54.777711Z","iopub.status.idle":"2025-12-14T14:22:56.851558Z","shell.execute_reply.started":"2025-12-14T14:22:54.777684Z","shell.execute_reply":"2025-12-14T14:22:56.850753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df_merged.copy()\n\ndef clean_raw(x):\n    if isinstance(x, list):\n        return x\n    return []    # biến NaN thành []\ndf[\"MF_raw\"] = df[\"MF_raw\"].apply(clean_raw)\ndf[\"BP_raw\"] = df[\"BP_raw\"].apply(clean_raw)\ndf[\"CC_raw\"] = df[\"CC_raw\"].apply(clean_raw)\n\nimport obonet\ngo_graph = obonet.read_obo(\"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\")\n\nparents = {}\nfor term, data in go_graph.nodes(data=True):\n    if \"is_a\" in data:\n        parents[term] = [p.split(\" ! \")[0] for p in data[\"is_a\"]]\n    else:\n        parents[term] = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:22:56.852366Z","iopub.execute_input":"2025-12-14T14:22:56.852631Z","iopub.status.idle":"2025-12-14T14:23:02.363800Z","shell.execute_reply.started":"2025-12-14T14:22:56.852597Z","shell.execute_reply":"2025-12-14T14:23:02.363235Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from functools import lru_cache\n\n@lru_cache(None)\ndef get_ancestors(term):\n    result = set()\n    for p in parents.get(term, []):\n        result.add(p)\n        result.update(get_ancestors(p))\n    return result\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:02.364566Z","iopub.execute_input":"2025-12-14T14:23:02.364789Z","iopub.status.idle":"2025-12-14T14:23:02.369728Z","shell.execute_reply.started":"2025-12-14T14:23:02.364772Z","shell.execute_reply":"2025-12-14T14:23:02.369140Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def propagate(term_list):\n    expanded = set(term_list)\n    for t in term_list:\n        expanded.update(get_ancestors(t))\n    return sorted(expanded)\n\n\ndf[\"MF_labels\"] = df[\"MF_raw\"].apply(propagate)\ndf[\"BP_labels\"] = df[\"BP_raw\"].apply(propagate)\ndf[\"CC_labels\"] = df[\"CC_raw\"].apply(propagate)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:02.370385Z","iopub.execute_input":"2025-12-14T14:23:02.370646Z","iopub.status.idle":"2025-12-14T14:23:04.534568Z","shell.execute_reply.started":"2025-12-14T14:23:02.370611Z","shell.execute_reply":"2025-12-14T14:23:04.533961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df\ndf_final = df[[\n    \"id\",\n    \"axit amin\",\n    \"aa length\",\n    \"taxon\",\n    \"MF_labels\",\n    \"BP_labels\",\n    \"CC_labels\"\n]]\n\ndf_final.to_csv(\"train_protein_propagated.tsv\", sep=\"\\t\", index=False)\n\ndf_final","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:04.535308Z","iopub.execute_input":"2025-12-14T14:23:04.535545Z","iopub.status.idle":"2025-12-14T14:23:07.417849Z","shell.execute_reply.started":"2025-12-14T14:23:04.535525Z","shell.execute_reply":"2025-12-14T14:23:07.416975Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#ESM2\nimport torch\nfrom transformers import AutoTokenizer, AutoModel\nfrom tqdm import tqdm\n\nMODEL_NAME = \"facebook/esm2_t33_650M_UR50D\"\nBATCH_SIZE = 2\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nMAX_LEN = 1024\n\n#Load model & tokenizer\ntokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\nmodel = AutoModel.from_pretrained(MODEL_NAME).to(DEVICE)\nmodel.eval()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:07.418537Z","iopub.execute_input":"2025-12-14T14:23:07.418836Z","iopub.status.idle":"2025-12-14T14:23:40.874380Z","shell.execute_reply.started":"2025-12-14T14:23:07.418812Z","shell.execute_reply":"2025-12-14T14:23:40.873548Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"aa = df_final[\"axit amin\"].tolist()\nid = df_final[\"id\"].tolist()\n\n#embedding\ndef get_embeddings(sequences, batch_size):\n    all_embeddings = []\n\n    for i in tqdm(range(0, len(sequences), batch_size), desc=\"Workin\"):\n        batch_seqs = sequences[i : i + batch_size]\n\n        #Tokenize\n        inputs = tokenizer(\n            batch_seqs,\n            return_tensors = \"pt\",\n            padding = True,\n            truncation = True,\n            max_length = MAX_LEN\n        ).to(DEVICE)\n\n        #model\n        with torch.no_grad():\n            outputs = model(**inputs)\n\n        #sig vector (mean pooling)\n        attention_mask = inputs[\"attention_mask\"]\n        embeddings = outputs.last_hidden_state\n\n        mask = attention_mask.unsqueeze(-1).expand(embeddings.size()).float()\n        masked_embeddings = embeddings * mask\n        summed = torch.sum(masked_embeddings, 1)\n        summed_mask = torch.clamp(mask.sum(1), min=1e-9)\n        mean_pooled = summed / summed_mask\n\n        all_embeddings.append(mean_pooled.cpu())\n\n    return torch.cat(all_embeddings)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:40.875373Z","iopub.execute_input":"2025-12-14T14:23:40.876147Z","iopub.status.idle":"2025-12-14T14:23:40.887789Z","shell.execute_reply.started":"2025-12-14T14:23:40.876127Z","shell.execute_reply":"2025-12-14T14:23:40.887122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# embeddings_tensor = get_embeddings(aa, BATCH_SIZE)\n# print(embeddings_tensor.shape)\n# print(embeddings_tensor[0])\n# if torch.isnan(embeddings_tensor).any():\n#     print(\"❌ CẢNH BÁO: Có giá trị NaN (Lỗi) trong vector!\")\n# else:\n#     print(\"✅ Dữ liệu sạch, không có NaN.\")\n# result = {\n#     \"ids\": id,\n#     \"embeddings\": embeddings_tensor\n# }\n# torch.save(result, \"embeddings_train.pt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:40.888567Z","iopub.execute_input":"2025-12-14T14:23:40.888834Z","iopub.status.idle":"2025-12-14T14:23:41.118825Z","shell.execute_reply.started":"2025-12-14T14:23:40.888812Z","shell.execute_reply":"2025-12-14T14:23:41.118099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if os.path.exists(\"/kaggle/input/embeddings/embeddings_train.pt\"):\n    print(\"Đã có embeddings_train!\")\nelse:\n    print(\"SOS\")\n\nembeddings_train_filepath = \"/kaggle/input/embeddings/embeddings_train.pt\"\nembeddings_test_filepath = \"/kaggle/input/embeddings/embeddings_test.pt\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:41.119630Z","iopub.execute_input":"2025-12-14T14:23:41.119954Z","iopub.status.idle":"2025-12-14T14:23:41.138607Z","shell.execute_reply.started":"2025-12-14T14:23:41.119927Z","shell.execute_reply":"2025-12-14T14:23:41.137991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_super_set_filepath = \"/kaggle/input/cafa-6-protein-function-prediction/Test/testsuperset.fasta\"\ntest_ids = []\ntest_aa = []\ntest_aa_length = []\nfor record in SeqIO.parse(test_super_set_filepath, \"fasta\"):\n    test_id = record.id.split(\"|\")[1] if len(record.id.split(\"|\")) > 1 else record.id\n    test_ids.append(test_id)\n    test_aa.append(str(record.seq))\n    test_aa_length.append(len(record.seq))\n\ndf_test = pd.DataFrame({\n    \"test_ids\": test_ids,\n    \"test_aa\": test_aa,\n    \"test_aa_length\": test_aa_length,\n})\nprint(len(df_test))\nprint(df_test.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:41.139292Z","iopub.execute_input":"2025-12-14T14:23:41.139566Z","iopub.status.idle":"2025-12-14T14:23:43.602067Z","shell.execute_reply.started":"2025-12-14T14:23:41.139549Z","shell.execute_reply":"2025-12-14T14:23:43.601213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_emb_id = df_test[\"test_ids\"].tolist()\ntest_embeddings_aa = df_test[\"test_aa\"].tolist()\nprint(len(test_emb_id))\nprint(len(test_embeddings_aa))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:43.602982Z","iopub.execute_input":"2025-12-14T14:23:43.603276Z","iopub.status.idle":"2025-12-14T14:23:43.618288Z","shell.execute_reply.started":"2025-12-14T14:23:43.603257Z","shell.execute_reply":"2025-12-14T14:23:43.617571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test_emb_id = df_test[\"test_ids\"].tolist()\n# test_embeddings_aa = df_test[\"test_aa\"].tolist()\n# embeddings_test = get_embeddings(test_embeddings_aa, BATCH_SIZE)\n# print(embeddings_test.shape)\n# print(embeddings_test[0])\n# if torch.isnan(embeddings_test).any():\n#     print(\"❌ CẢNH BÁO: Có giá trị NaN (Lỗi) trong vector!\")\n# else:\n#     print(\"✅ Dữ liệu sạch, không có NaN.\")\n# test_result = {\n#     \"ids\": test_emb_id,\n    \n#     \"embeddings\": embeddings_test\n# }\n# torch.save(test_result, \"embeddings_test.pt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:43.619227Z","iopub.execute_input":"2025-12-14T14:23:43.619550Z","iopub.status.idle":"2025-12-14T14:23:53.718096Z","shell.execute_reply.started":"2025-12-14T14:23:43.619520Z","shell.execute_reply":"2025-12-14T14:23:53.717210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Vẽ phân phối Top-N GO Label\n\nfrom collections import Counter\n\n# Config: Chọn Top-N cho mỗi Label\nTOPS = {\n    'MF_labels': 1000,\n    'BP_labels': 2000, \n    'CC_labels': 500,\n}\n\nif 'df_final' not in globals():\n    raise RuntimeError(\"df_final not found. Load your DataFrame (train labels) before running this cell.\")\n\ndef flatten_label_lists(series):\n    flat = []\n    for x in series:\n        if isinstance(x, (list, tuple)):\n            flat.extend(x)\n    return flat\n\nplt.figure(figsize=(10, 5))\ncolors = {'MF_labels': 'C0', 'BP_labels': 'C1', 'CC_labels': 'C2'}\n\nfor col, top_n in TOPS.items():\n    if col not in df_final.columns:\n        print(f\"⚠️ Column '{col}' not in df_final — skipping\")\n        continue\n\n    ser = df_final[col].fillna(\"\")\n    all_labels = flatten_label_lists(ser)\n    if len(all_labels) == 0:\n        print(f\"⚠️ No annotations found in {col}; skipping\")\n        continue\n\n    counter = Counter(all_labels)\n    sorted_counts = np.array(sorted(counter.values(), reverse=True))\n\n    ranks = np.arange(1, len(sorted_counts) + 1)\n    plt.plot(ranks, sorted_counts, label=f\"{col.split('_')[0]} (unique={len(sorted_counts)})\", color=colors.get(col, None))\n\n    top_n_count = sorted_counts[:top_n].sum() if len(sorted_counts) >= 1 else 0\n    total_count = sorted_counts.sum()\n    coverage = top_n_count / total_count if total_count > 0 else 0.0\n\n    print(f\"{col.split('_')[0]}: Top {top_n} covers {coverage:.2%} of annotations ({top_n_count}/{total_count})\")\n\n# Vẽ\nplt.xlabel('Label rank (1 = most frequent)')\nplt.ylabel('Annotation count')\nplt.title('Top-N GO label distributions (MF / BP / CC)')\nplt.xscale('log')\nplt.yscale('log')\nplt.legend()\nplt.grid(which='both', linestyle='--', alpha=0.3)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:53.719063Z","iopub.execute_input":"2025-12-14T14:23:53.719368Z","iopub.status.idle":"2025-12-14T14:23:54.884086Z","shell.execute_reply.started":"2025-12-14T14:23:53.719340Z","shell.execute_reply":"2025-12-14T14:23:54.883178Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Bắt đầu làm CAFA6","metadata":{}},{"cell_type":"code","source":"# Chuẩn bị X_tensor, test_tensor, df_aligned\n\nimport torch\nimport pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom tqdm import tqdm\n\nprint(\"=\"*60)\nprint(\"📂 CHUẨN BỊ DỮ LIỆU\")\nprint(\"=\"*60)\n\n# Load Embeddings\nprint(\"\\n1️⃣ Load Embeddings...\")\n\n# Sửa đường dẫn nếu cần\nembeddings_train_filepath = \"/kaggle/input/embeddings/embeddings_train.pt\"  # Sửa đây\nembeddings_test_filepath = \"/kaggle/input/embeddings/embeddings_test.pt\"    # Sửa đây\n\ndata_emb = torch.load(embeddings_train_filepath)\nemb_ids = data_emb['ids']\nX_tensor = data_emb['embeddings']\n\ntest_data = torch.load(embeddings_test_filepath)\ntest_ids = test_data['ids']\ntest_tensor = test_data['embeddings']\n\nprint(f\"   ✅ Train embeddings: {X_tensor.shape}\")\nprint(f\"   ✅ Test embeddings: {test_tensor.shape}\")\n\n# Khớp dữ liệu\nprint(\"\\n2️⃣ Khớp dữ liệu...\")\n\ndf_aligned = df_final.set_index('id').reindex(emb_ids).reset_index()\n\nprint(f\"   ✅ DataFrame align: {df_aligned.shape}\")\n\n# Kiểm tra dữ liệu\nprint(\"\\n3️⃣ Kiểm tra dữ liệu...\")\n\nprint(f\"   Cột có sẵn: {list(df_aligned.columns)}\")\n\nlabel_cols = ['MF_labels', 'BP_labels', 'CC_labels']\n\ndef _count_annotations(series):\n   # series chứa list/string/NaN; đếm tổng số annotation\n   total = 0\n   for x in series:\n      # list/tuple: số annotation = độ dài\n      if isinstance(x, (list, tuple)):\n         total += len(x)\n      # string: nếu không rỗng, đếm là 1 \n      elif isinstance(x, str):\n         if x.strip():\n            total += 1\n      # NaN/None: bỏ qua\n      else:\n         try:\n            if pd.isna(x):\n               continue\n         except Exception:\n            pass\n         # các kiểu khác (số, dict...) - bỏ qua hoặc đếm 0\n   return total\n\nfor col in tqdm(label_cols, desc='Checking label columns', unit='col'):\n   if col not in df_aligned.columns:\n      print(f\"   {col}: (not found in df_aligned)\")\n      continue\n   n_nan = int(df_aligned[col].isnull().sum())\n   # Truyền trực tiếp series vào hàm đếm \n   total_ann = _count_annotations(df_aligned[col])\n   print(f\"   {col}: NaN={n_nan}, total annotations={total_ann}\")\n\n# Tóm tắt thông tin\nprint(\"\\n\" + \"=\"*60)\nprint(\"✅ CHUẨN BỊ XONG!\")\nprint(\"=\"*60)\nprint(f\"\"\"\n📊 Thông tin:\n   • Train proteins: {len(df_aligned)}\n   • Test proteins: {len(test_ids)}\n   • Embedding dimension: {X_tensor.shape[1]}\n   • Device: {'cuda' if torch.cuda.is_available() else 'cpu'}\n\n🎯 Các biến chuẩn bị sẵn:\n   • X_tensor       : Embeddings train {X_tensor.shape}\n   • test_tensor    : Embeddings test {test_tensor.shape}\n   • test_ids       : IDs của protein test (length={len(test_ids)})\n   • df_aligned     : DataFrame train {df_aligned.shape}\n   • emb_ids        : IDs của protein train (length={len(emb_ids)})\n\"\"\")\n\nprint(\"✨ Sẵn sàng chạy cell tiếp theo!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:23:54.885040Z","iopub.execute_input":"2025-12-14T14:23:54.885412Z","iopub.status.idle":"2025-12-14T14:24:05.027563Z","shell.execute_reply.started":"2025-12-14T14:23:54.885387Z","shell.execute_reply":"2025-12-14T14:24:05.026795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import f1_score, roc_auc_score\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport numpy as np\nimport pandas as pd\nimport gc\nfrom tqdm import tqdm\nimport os\nimport pickle\n\n# Cấu hình chung\nBATCH_SIZE = 64\nLEARNING_RATE = 1e-3\nNUM_EPOCHS = 10\nHIDDEN_DIM = 512\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nTHRESHOLD_FILTER = 0.01\n\n# Cấu hình Top-N cho mỗi namespace\nTOP_N_CONFIG = {\n    'MF': 500,\n    'BP': 1500,\n    'CC': 500,\n}\n\nprint(f\"🚀 Thiết bị: {DEVICE}\")\n\n# Định nghĩa mô hình\nclass ProteinPredictor(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super().__init__()\n        self.network = nn.Sequential(\n            # Layer 1\n            nn.Linear(input_dim, hidden_dim),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n\n            # Layer 2\n            # nn.Linear(hidden_dim, hidden_dim // 2),\n            # nn.BatchNorm1d(hidden_dim // 2),\n            # nn.ReLU(),\n            # nn.Dropout(0.3),\n            # Đang bỏ layer 2 vì cho ra kết quả kém hơn\n\n            # Layer 3\n            nn.Linear(hidden_dim, output_dim)\n        )\n    \n    def forward(self, x):\n        return self.network(x)\n\n\n# Focal loss (BCE + FOCAL)\nclass FocalLossBCE(nn.Module):\n    def __init__(self, alpha: float = 1.0, gamma: float = 2.0, reduction: str = 'mean'):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n\n    def forward(self, logits, targets):\n        # logits: (batch, n_classes), targets: {0,1}\n        bce_loss = nn.functional.binary_cross_entropy_with_logits(logits, targets, reduction='none')\n        probs = torch.sigmoid(logits)\n        p_t = probs * targets + (1 - probs) * (1 - targets)\n        focal_factor = (1 - p_t) ** self.gamma\n        loss = self.alpha * focal_factor * bce_loss\n        if self.reduction == 'mean':\n            return loss.mean()\n        elif self.reduction == 'sum':\n            return loss.sum()\n        else:\n            return loss\n\n# Training\nTRAINING_HISTORY = {}\n\ndef train(namespace, n_splits: int = 3):\n    \"\"\"\n    Train model cho một namespace (MF/BP/CC) và dự đoán\n    \n    Args:\n        namespace: 'MF', 'BP', hoặc 'CC'\n        n_splits: số fold (mặc định 3 để tiết kiệm RAM)\n    \n    Returns:\n        None (output file pred_{namespace}.tsv)\n    \"\"\"\n    \n    # Xác định cấu hình\n    if namespace not in TOP_N_CONFIG:\n        raise ValueError(f\"namespace phải là MF, BP, hoặc CC. Nhận: {namespace}\")\n    \n    TOP_N = TOP_N_CONFIG[namespace]\n    TARGET_COL = f'{namespace}_labels'\n    \n    print(f\"\\n{'='*60}\")\n    print(f\"📌 TRAIN: {namespace} (Top {TOP_N})\")\n    print(f\"{'='*60}\")\n    \n    # Chuẩn bị dữ liệu\n    print(\"🎯 Chuẩn bị dữ liệu...\")\n    \n    # Kiểm tra NaN\n    if df_aligned[TARGET_COL].isnull().any():\n        print(\"⚠️ Có NaN, thay thế bằng list rỗng\")\n        df_aligned[TARGET_COL] = df_aligned[TARGET_COL].apply(\n            lambda x: x if isinstance(x, list) else []\n        )\n    \n    # Lấy nhãn\n    y_lists = df_aligned[TARGET_COL]\n    \n    # Lọc Top N nhãn\n    all_labels = [label for sublist in y_lists for label in sublist]\n    top_labels = pd.Series(all_labels).value_counts().head(TOP_N).index.tolist()\n    coverage = sum(pd.Series(all_labels).value_counts().head(TOP_N).values) / len(all_labels) if len(all_labels) > 0 else 0.0\n    \n    print(f\"   Tổng nhãn độc nhất: {len(set(all_labels))}\")\n    print(f\"   Top N: {len(top_labels)} (coverage: {coverage:.2%})\")\n    print(f\"   X shape: {X_tensor.shape}\")\n    \n    # Lọc và One-hot encoding\n    y_lists_filtered = y_lists.apply(lambda x: [i for i in x if i in top_labels])\n    mlb = MultiLabelBinarizer(classes=top_labels)\n    Y_matrix = mlb.fit_transform(y_lists_filtered)\n    \n    print(f\"   Y shape: {Y_matrix.shape}\")\n    \n    # Chia dữ liệu theo K-Fold\n    print(f\"🔁 Bắt đầu {n_splits}-fold cross-validation...\")\n\n    X = X_tensor.numpy()\n    Y = Y_matrix\n\n    kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n    fold_scores = []\n    saved_model_paths = []\n\n    # Lưu lại history để vẽ biểu đồ\n    avg_loss_history = [0.0] * NUM_EPOCHS\n    avg_auc_history = [0.0] * NUM_EPOCHS\n\n    # Thu thập validation probs/targets ở các folds để tìm global threshold\n    aggregated_val_probs = []\n    aggregated_val_targets = []\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n        print(f\"\\n---- Fold {fold+1}/{n_splits} ----\")\n\n        X_train, X_val = X[train_idx], X[val_idx]\n        y_train, y_val = Y[train_idx], Y[val_idx]\n\n        train_dataset = TensorDataset(torch.tensor(X_train).float(), torch.tensor(y_train).float())\n        val_dataset = TensorDataset(torch.tensor(X_val).float(), torch.tensor(y_val).float())\n\n        train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n        # Xây dựng model\n        model = ProteinPredictor(input_dim=1280, hidden_dim=HIDDEN_DIM, output_dim=Y.shape[1]).to(DEVICE)\n        criterion = FocalLossBCE(alpha=1.0, gamma=2.0)\n        optimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE)\n        scheduler = CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS)\n\n        best_val_auc = -np.inf\n        best_model_path = f\"best_model_{namespace}_fold{fold+1}.pth\"\n\n        for epoch in tqdm(range(NUM_EPOCHS), desc=f'Training {namespace} fold{fold+1}', unit='epoch', leave=True):\n            model.train()\n            train_loss = 0.0\n            for inputs, labels in train_loader:\n                inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)\n                optimizer.zero_grad()\n                outputs = model(inputs)\n                loss = criterion(outputs, labels)\n                loss.backward()\n                optimizer.step()\n                train_loss += loss.item()\n\n            train_loss /= len(train_loader)\n            scheduler.step()\n\n            # Validation: Tính val AUC (macro)\n            model.eval()\n            val_probs = []\n            val_targets_list = []\n            with torch.no_grad():\n                for inputs, labels in val_loader:\n                    inputs = inputs.to(DEVICE)\n                    outputs = model(inputs)\n                    probs = torch.sigmoid(outputs)\n                    val_probs.append(probs.cpu().numpy())\n                    val_targets_list.append(labels.numpy())\n\n            val_probs = np.vstack(val_probs)\n            val_targets_arr = np.vstack(val_targets_list)\n\n            # Tính ROC-AUC macro\n            try:\n                auc = roc_auc_score(val_targets_arr, val_probs, average='macro')\n            except Exception:\n                auc = np.nan\n\n            if (epoch + 1) % 5 == 0 or epoch == 0:\n                tqdm.write(f\"  Epoch {epoch+1:2d} | Train loss: {train_loss:.4f} | Val AUC: {auc if not np.isnan(auc) else 'nan'}\")\n\n            if fold == 0:\n                avg_loss_history[epoch] = train_loss\n                avg_auc_history[epoch] = auc\n            \n            # Save best by AUC (fallback to loss if AUC nan)\n            if not np.isnan(auc):\n                if auc > best_val_auc:\n                    best_val_auc = auc\n                    torch.save(model.state_dict(), best_model_path)\n            else:\n                # fallback: save by train loss improvement\n                if train_loss < 1e6 and (best_val_auc == -np.inf):\n                    torch.save(model.state_dict(), best_model_path)\n\n        # Kết thúc epochs cho fold này\n        print(f\"--> Fold {fold+1} best val AUC: {best_val_auc}\")\n        fold_scores.append(best_val_auc)\n        saved_model_paths.append(best_model_path)\n\n        # Load best model cho fold này và thu thập val predictions \n        try:\n            model.load_state_dict(torch.load(best_model_path))\n        except Exception:\n            pass\n\n        model.eval()\n        val_probs = []\n        val_targets_list = []\n        with torch.no_grad():\n            for inputs, labels in val_loader:\n                inputs = inputs.to(DEVICE)\n                outputs = model(inputs)\n                probs = torch.sigmoid(outputs)\n                val_probs.append(probs.cpu().numpy())\n                val_targets_list.append(labels.numpy())\n\n        if len(val_probs) > 0:\n            val_probs = np.vstack(val_probs)\n            val_targets_arr = np.vstack(val_targets_list)\n            aggregated_val_probs.append(val_probs)\n            aggregated_val_targets.append(val_targets_arr)\n\n        # Cleanup fold\n        del model, optimizer, scheduler, train_loader, val_loader, train_dataset, val_dataset\n        gc.collect()\n\n    TRAINING_HISTORY[namespace] = {\n        'loss': avg_loss_history,\n        'auc': avg_auc_history\n    }\n\n    # End kfold\n    mean_auc = np.nanmean(fold_scores)\n    print(f\"\\n⭐ Mean validation AUC across folds: {mean_auc:.6f}\")\n\n    # Aggregate val probs/targets \n    if len(aggregated_val_probs) > 0:\n        all_val_probs = np.vstack(aggregated_val_probs)\n        all_val_targets = np.vstack(aggregated_val_targets)\n    else:\n        all_val_probs = np.zeros((0, Y.shape[1]))\n        all_val_targets = np.zeros((0, Y.shape[1]))\n\n    # Tìm threshold\n    print(\"🔍 Tìm ngưỡng tối ưu (dùng tập val gộp)...\")\n    best_f1 = 0\n    best_thresh = 0.5\n    if all_val_probs.shape[0] > 0:\n        thresholds_to_try = np.arange(0.1, 0.6, 0.05)\n        for t in tqdm(thresholds_to_try, desc=f'Threshold search {namespace}', leave=False):\n            preds = (all_val_probs > t).astype(int)\n            score = f1_score(all_val_targets, preds, average='micro')\n            if score > best_f1:\n                best_f1 = score\n                best_thresh = t\n\n    print(f\"   Ngưỡng tốt nhất: {best_thresh:.2f}\")\n    print(f\"   F1-Score (agg val): {best_f1:.4f}\")\n\n    # cleanup aggregated\n    del aggregated_val_probs, aggregated_val_targets, all_val_probs, all_val_targets\n    gc.collect()\n    \n    # Dự đoán test\n    print(\"🎯 Dự đoán trên tập test bằng ensemble các fold...\")\n\n    test_dataset = TensorDataset(test_tensor.float())\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n    # Kiểm tra các model đã lưu từ các fold\n    valid_model_paths = [p for p in saved_model_paths if os.path.exists(p)]\n    if len(valid_model_paths) == 0:\n        # fallback: try the generic best_model file name\n        generic = f\"best_model_{namespace}.pth\"\n        if os.path.exists(generic):\n            valid_model_paths = [generic]\n\n    if len(valid_model_paths) == 0:\n        raise RuntimeError(\"Không tìm thấy model đã lưu để dự đoán. Hãy đảm bảo quá trình training đã lưu ít nhất một model.\")\n\n    all_model_probs = []\n    for mp in valid_model_paths:\n        print(f\" - Load model: {mp}\")\n        m = ProteinPredictor(input_dim=1280, hidden_dim=HIDDEN_DIM, output_dim=Y_matrix.shape[1]).to(DEVICE)\n        try:\n            m.load_state_dict(torch.load(mp, map_location=DEVICE))\n        except Exception as e:\n            print(f\"   ⚠️ Không thể load {mp}: {e}\")\n            continue\n        m.eval()\n\n        probs_acc = []\n        with torch.no_grad():\n            for inputs in tqdm(test_loader, desc=f\"Predicting {namespace} ({os.path.basename(mp)})\", leave=False):\n                inputs = inputs[0].to(DEVICE)\n                outputs = m(inputs)\n                probs = torch.sigmoid(outputs)\n                probs_acc.append(probs.cpu().numpy())\n\n        if len(probs_acc) > 0:\n            probs_model = np.vstack(probs_acc)\n            all_model_probs.append(probs_model)\n\n        del m\n        gc.collect()\n\n    if len(all_model_probs) == 0:\n        raise RuntimeError(\"Không có dự đoán từ bất kỳ model nào trong ensemble.\")\n\n    # Trung bình xác suất từ các model\n    all_test_probs = np.mean(np.stack(all_model_probs, axis=0), axis=0)\n    \n    # Lưu predictión & MLB\n    print(\"💾 Lưu predictions và MLBinarizer...\")\n    \n    # Lưu predictions\n    pred_file = f\"pred_probs_{namespace}.pkl\"\n    with open(pred_file, \"wb\") as f:\n        pickle.dump({\n            'test_ids': test_ids,\n            'all_test_probs': all_test_probs,\n            'go_classes': mlb.classes_,\n            'namespace': namespace\n        }, f)\n    \n    # Lưu MLBinarizer\n    with open(f\"mlb_{namespace}_labels.pkl\", \"wb\") as f:\n        pickle.dump(mlb, f)\n    \n    print(f\"   ✅ Predictions lưu: {pred_file}\")\n    print(f\"   ✅ MLBinarizer lưu: mlb_{namespace}_labels.pkl\")\n    \n    # Xóa RAM \n    if 'model' in locals():\n        try:\n            del model\n        except Exception:\n            pass\n    if 'm' in locals():\n        try:\n            del m\n        except Exception:\n            pass\n    if 'test_loader' in locals():\n        try:\n            del test_loader\n        except Exception:\n            pass\n    if 'all_test_probs' in locals():\n        try:\n            del all_test_probs\n        except Exception:\n            pass\n    if 'all_model_probs' in locals():\n        try:\n            del all_model_probs\n        except Exception:\n            pass\n    gc.collect()\n    \n    print(\"\\n\" + \"=\"*60)\n    print(f\"🎉 HOÀN TẤT {namespace}!\")\n    print(\"=\"*60)\n    print(f\"✅ Predictions file: pred_probs_{namespace}.pkl\")\n    print(f\"✅ Best threshold: {best_thresh:.2f}\")\n    print(f\"✅ MLBinarizer: mlb_{namespace}_labels.pkl\")\n    print(f\"✅ (Ghi file TSV trong cell_inference_output.py)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:53:07.325539Z","iopub.execute_input":"2025-12-14T14:53:07.326165Z","iopub.status.idle":"2025-12-14T14:53:07.360338Z","shell.execute_reply.started":"2025-12-14T14:53:07.326139Z","shell.execute_reply":"2025-12-14T14:53:07.359778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train('MF', 5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:53:17.120188Z","iopub.execute_input":"2025-12-14T14:53:17.120459Z","iopub.status.idle":"2025-12-14T15:00:31.823249Z","shell.execute_reply.started":"2025-12-14T14:53:17.120439Z","shell.execute_reply":"2025-12-14T15:00:31.822642Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train('CC', 5)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T15:00:31.824750Z","iopub.execute_input":"2025-12-14T15:00:31.824966Z","iopub.status.idle":"2025-12-14T15:07:48.601314Z","shell.execute_reply.started":"2025-12-14T15:00:31.824950Z","shell.execute_reply":"2025-12-14T15:07:48.600661Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train(\"BP\", 2)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T15:07:48.602335Z","iopub.execute_input":"2025-12-14T15:07:48.602541Z","iopub.status.idle":"2025-12-14T15:22:17.566996Z","shell.execute_reply.started":"2025-12-14T15:07:48.602525Z","shell.execute_reply":"2025-12-14T15:22:17.566167Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Kiểm tra xem có dữ liệu chưa\nif not TRAINING_HISTORY:\n    print(\"❌ Chưa có dữ liệu! Mày đã chạy hàm train() chưa?\")\nelse:\n    epochs = range(1, NUM_EPOCHS + 1)\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))\n    plt.style.use('ggplot')\n\n    # Vẽ Loss\n    for ns in TRAINING_HISTORY:\n        ax1.plot(epochs, TRAINING_HISTORY[ns]['loss'], 'o-', label=f'{ns} Loss', linewidth=2)\n    \n    ax1.set_title('Training Loss (Fold 1)', fontsize=14, fontweight='bold')\n    ax1.set_xlabel('Epochs')\n    ax1.set_ylabel('Focal Loss')\n    ax1.legend()\n    ax1.grid(True, linestyle='--', alpha=0.7)\n\n    # Vẽ AUC\n    for ns in TRAINING_HISTORY:\n        ax2.plot(epochs, TRAINING_HISTORY[ns]['auc'], 's-', label=f'{ns} AUC', linewidth=2)\n    \n    ax2.set_title('Validation AUC (Fold 1)', fontsize=14, fontweight='bold')\n    ax2.set_xlabel('Epochs')\n    ax2.set_ylabel('Macro AUC')\n    ax2.legend()\n    ax2.grid(True, linestyle='--', alpha=0.7)\n\n    plt.tight_layout()\n    plt.savefig('training_curves_real.png', dpi=300)\n    plt.show()\n    # print(\"✅ Đã vẽ xong ảnh training_curves_real.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T15:25:09.501843Z","iopub.execute_input":"2025-12-14T15:25:09.502123Z","iopub.status.idle":"2025-12-14T15:25:10.612795Z","shell.execute_reply.started":"2025-12-14T15:25:09.502103Z","shell.execute_reply":"2025-12-14T15:25:10.612236Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport pickle\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import f1_score, roc_auc_score\nfrom torch.utils.data import TensorDataset, DataLoader\nimport gc\nimport obonet\n\n# Config\nTHRESHOLD_CONFIG = {'MF': 0.02, 'CC': 0.02, 'BP': 0.05}\nTOP_K_CONFIG = {'MF': 30, 'CC': 30, 'BP': 60}\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nBATCH_SIZE = 64\n\n# Chuẩn bị lại model\nimport torch.nn as nn\nclass ProteinPredictor(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super().__init__()\n        self.network = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(hidden_dim, output_dim)\n        )\n    def forward(self, x):\n        return self.network(x)\n\n# Load dữ liệu gốc\nprint(\"⏳ Đang load dữ liệu gốc...\")\n# Load Embeddings Train\ndata_emb = torch.load(\"/kaggle/input/embeddings/embeddings_train.pt\") # Check lại path\nX_tensor_all = data_emb['embeddings']\nemb_ids = data_emb['ids']\n\n# Load DataFrame nhãn (df_final)\nif 'df_final' not in globals():\n    print(\"⚠️ CẢNH BÁO: Mất df_final!\")\nelse:\n    # Khớp nối lại dữ liệu\n    df_aligned = df_final.set_index('id').reindex(emb_ids).reset_index()\n\n# Load Parents Map (Hierarchy)\nprint(\"⏳ Đang load OBO...\")\ntry:\n    go_graph = obonet.read_obo(\"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\")\n    parents_map = {term: [p.split(\" ! \")[0] for p in data.get(\"is_a\", [])] for term, data in go_graph.nodes(data=True)}\nexcept:\n    print(\"❌ Lỗi load OBO. Kiểm tra path.\")\n    parents_map = {}\n\n# Hàm Enforce Hierarchy\ndef enforce_hierarchy_batch(probs, classes, parent_map):\n    term_to_idx = {term: i for i, term in enumerate(classes)}\n    for i, term in enumerate(classes):\n        if term not in parent_map: continue\n        for p in parent_map[term]:\n            if p in term_to_idx:\n                p_idx = term_to_idx[p]\n                probs[:, p_idx] = np.maximum(probs[:, p_idx], probs[:, i])\n    return probs\n\n# Tính điểm\ndef calc_metrics_for_report(namespace):\n    print(f\"\\n{'='*60}\")\n    print(f\"🚀 TÍNH ĐIỂM BÁO CÁO: {namespace}\")\n    \n    # Load MLB đã lưu \n    try:\n        with open(f\"mlb_{namespace}_labels.pkl\", \"rb\") as f:\n            mlb = pickle.load(f)\n        classes = mlb.classes_\n    except:\n        print(f\"❌ Không tìm thấy file mlb_{namespace}_labels.pkl.\")\n        return\n\n    # Lấy Target (Y)\n    TARGET_COL = f'{namespace}_labels'\n    y_lists = df_aligned[TARGET_COL].fillna(\"\").apply(lambda x: x if isinstance(x, list) else [])\n    # Filter nhãn theo MLB đã lưu\n    y_lists_filtered = y_lists.apply(lambda x: [i for i in x if i in classes])\n    Y_matrix = mlb.transform(y_lists_filtered)\n    \n    # Chỉ chạy Fold 1\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    train_idx, val_idx = next(kf.split(X_tensor_all)) # Lấy fold đầu tiên\n    \n    X_val = X_tensor_all[val_idx]\n    y_val = Y_matrix[val_idx]\n    \n    # Load Model đã lưu (Fold 1)\n    model_path = f\"best_model_{namespace}_fold1.pth\"\n    model = ProteinPredictor(1280, 512, len(classes)).to(DEVICE)\n    try:\n        model.load_state_dict(torch.load(model_path, map_location=DEVICE))\n    except:\n        print(f\"⚠️ Không load được {model_path}. Thử file best_model_{namespace}.pth...\")\n        try:\n            model.load_state_dict(torch.load(f\"best_model_{namespace}.pth\", map_location=DEVICE))\n        except:\n            print(\"❌ Không tìm thấy model nào.\")\n            return\n\n    # Dự đoán (Inference)\n    print(\"⚡ Đang chạy model dự đoán lại trên tập Val...\")\n    model.eval()\n    val_dataset = TensorDataset(X_val.float())\n    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n    \n    probs_list = []\n    with torch.no_grad():\n        for inputs in val_loader:\n            outputs = model(inputs[0].to(DEVICE))\n            probs_list.append(torch.sigmoid(outputs).cpu().numpy())\n    \n    val_probs_raw = np.vstack(probs_list)\n    \n    # === BẮT ĐẦU TÍNH ĐIỂM ===\n    \n    # 1. STAGE 1: RAW \n    print(\"🔎 Đang tìm Best Threshold cho Raw...\")\n    best_f1_raw = 0\n    best_thresh_raw = 0\n    thresholds = np.arange(0.1, 0.6, 0.05) # Quét từ 0.1 đến 0.6\n    \n    for t in thresholds:\n        preds = (val_probs_raw >= t).astype(int)\n        score = f1_score(y_val, preds, average='micro')\n        if score > best_f1_raw:\n            best_f1_raw = score\n            best_thresh_raw = t\n            \n    # Tính AUC cho Raw\n    try: auc_raw = roc_auc_score(y_val, val_probs_raw, average='macro')\n    except: auc_raw = 0.0\n    \n    print(f\"👉 Stage 1 (Raw @ BestThresh {best_thresh_raw:.2f}): F1={best_f1_raw:.4f} | AUC={auc_raw:.4f}\")\n\n    # 2. STAGE 2: HIERARCHY \n    print(\"⚡ Đang chạy Hierarchy...\")\n    val_probs_hie = val_probs_raw.copy()\n    for _ in range(2):\n        val_probs_hie = enforce_hierarchy_batch(val_probs_hie, classes, parents_map)\n        \n    # Tính điểm Hie tại ngưỡng tối ưu của Raw (để so sánh công bằng hiệu quả của thuật toán)\n    val_preds_hie = (val_probs_hie >= best_thresh_raw).astype(int)\n    f1_hie = f1_score(y_val, val_preds_hie, average='micro')\n    try: auc_hie = roc_auc_score(y_val, val_probs_hie, average='macro')\n    except: auc_hie = 0.0\n    \n    print(f\"👉 Stage 2 (Hierarchy @ {best_thresh_raw:.2f}): F1={f1_hie:.4f} | AUC={auc_hie:.4f}\")\n    print(f\"   (Improvement: {f1_hie - best_f1_raw:+.4f})\")\n\n    # 3. STAGE 3: TOP-K \n    current_thresh = best_thresh_raw \n    k_val = TOP_K_CONFIG.get(namespace, 60)\n    \n    print(f\"⚡ Đang chạy Top-{k_val} Filter (với ngưỡng tối ưu {current_thresh:.2f})...\")\n    \n    val_preds_final = np.zeros_like(val_probs_hie, dtype=int)\n    for i in range(len(val_probs_hie)):\n        row_probs = val_probs_hie[i]\n        \n        # Lấy candidate với ngưỡng TỐI ƯU\n        indices = np.where(row_probs >= current_thresh)[0]\n        \n        if len(indices) > k_val:\n            top_k_idx = np.argsort(row_probs[indices])[::-1][:k_val]\n            final_idx = indices[top_k_idx]\n        else:\n            final_idx = indices\n        val_preds_final[i, final_idx] = 1\n        \n    f1_final = f1_score(y_val, val_preds_final, average='micro')\n    \n    # Tính AUC cho Top-K \n    try: auc_final = roc_auc_score(y_val, val_preds_final, average='macro')\n    except: auc_final = 0.0\n\n    print(f\"👉 Stage 3 (Top-K Final): F1={f1_final:.4f} | AUC={auc_final:.4f}\")\n    print(f\"   (So với Raw Optimal: {f1_final - best_f1_raw:+.4f})\")\n    \n    # Clean\n    del val_probs_raw, val_probs_hie\n    gc.collect()\n\n# Chạy hàm tính điểm\ncalc_metrics_for_report('MF')\ncalc_metrics_for_report('BP')\ncalc_metrics_for_report('CC')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T16:13:09.936661Z","iopub.execute_input":"2025-12-14T16:13:09.936961Z","iopub.status.idle":"2025-12-14T16:17:35.878417Z","shell.execute_reply.started":"2025-12-14T16:13:09.936940Z","shell.execute_reply":"2025-12-14T16:17:35.877657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import f1_score\nfrom torch.utils.data import TensorDataset, DataLoader\nimport gc\nfrom tqdm import tqdm\n\n# Config\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nBATCH_SIZE = 64\n\n# Định nghĩa model\nimport torch.nn as nn\nclass ProteinPredictor(nn.Module):\n    def __init__(self, input_dim, hidden_dim, output_dim):\n        super().__init__()\n        self.network = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.BatchNorm1d(hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(hidden_dim, output_dim)\n        )\n    def forward(self, x):\n        return self.network(x)\n\n# Load dữ liệu\nprint(\"⏳ Đang load dữ liệu nền...\")\n# Load Embeddings\nif 'X_tensor_all' not in globals():\n    data_emb = torch.load(\"/kaggle/input/embeddings/embeddings_train.pt\") \n    X_tensor_all = data_emb['embeddings']\n    emb_ids = data_emb['ids']\n\n# Load DataFrame nhãn\nif 'df_final' not in globals():\n    print(\"⚠️ Cảnh báo: Biến df_final không tồn tại.\")\nelse:\n    df_aligned = df_final.set_index('id').reindex(emb_ids).reset_index()\n\n# Hàm tính\ndef compute_threshold_curve(namespace):\n    print(f\"\\n📊 Đang tính curve cho {namespace}...\")\n    \n    with open(f\"mlb_{namespace}_labels.pkl\", \"rb\") as f:\n        mlb = pickle.load(f)\n    classes = mlb.classes_\n\n    TARGET_COL = f'{namespace}_labels'\n    y_lists = df_aligned[TARGET_COL].fillna(\"\").apply(lambda x: x if isinstance(x, list) else [])\n    y_lists_filtered = y_lists.apply(lambda x: [i for i in x if i in classes])\n    Y_matrix = mlb.transform(y_lists_filtered)\n\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    _, val_idx = next(kf.split(X_tensor_all))\n    X_val = X_tensor_all[val_idx]\n    y_val = Y_matrix[val_idx]\n\n    model = ProteinPredictor(1280, 512, len(classes)).to(DEVICE)\n    model_path = f\"best_model_{namespace}_fold1.pth\"\n    try:\n        model.load_state_dict(torch.load(model_path, map_location=DEVICE))\n    except:\n        # Fallback nếu không thấy fold1\n        try:\n            model.load_state_dict(torch.load(f\"best_model_{namespace}.pth\", map_location=DEVICE))\n        except:\n            print(\"❌ Không tìm thấy model.\")\n            return\n    model.eval()\n\n    val_loader = DataLoader(TensorDataset(X_val.float()), batch_size=BATCH_SIZE, shuffle=False)\n\n    probs = []\n    with torch.no_grad():\n        for (x,) in val_loader:\n            probs.append(torch.sigmoid(model(x.to(DEVICE))).cpu().numpy())\n    val_probs = np.vstack(probs)\n\n    thresholds = np.arange(0.05, 0.65, 0.05)\n    f1_scores = []\n\n    best_f1, best_thresh = 0, 0\n    for t in thresholds:\n        score = f1_score(y_val, (val_probs > t).astype(int), average='micro')\n        f1_scores.append(score)\n        if score > best_f1:\n            best_f1, best_thresh = score, t\n\n    print(f\"✅ {namespace}: best F1={best_f1:.4f} @ t={best_thresh:.2f}\")\n    return thresholds, f1_scores, best_thresh, best_f1\n\n# Vẽ\n\nplt.figure(figsize=(9, 6))\n\ncolors = {\n    \"MF\": \"tab:blue\",\n    \"BP\": \"tab:orange\",\n    \"CC\": \"tab:green\"\n}\n\nfor ns in [\"MF\", \"BP\", \"CC\"]:\n    thresholds, f1s, best_t, best_f1 = compute_threshold_curve(ns)\n    plt.plot(\n        thresholds, f1s,\n        marker='o',\n        linewidth=2,\n        color=colors[ns],\n        label=f\"{ns} (F1 best={best_f1:.3f})\"\n    )\n    plt.axvline(best_t, color=colors[ns], linestyle=\"--\", alpha=0.4)\n\nplt.title(\"Threshold Sensitivity across GO Subontologies\", fontsize=13, fontweight=\"bold\")\nplt.xlabel(\"Decision Threshold\")\nplt.ylabel(\"Micro F1-score\")\nplt.legend()\nplt.grid(True, linestyle=\"--\", alpha=0.4)\n\nplt.savefig(\"threshold_sensitivity_all.png\", dpi=300)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T15:52:17.630543Z","iopub.execute_input":"2025-12-14T15:52:17.631287Z","iopub.status.idle":"2025-12-14T15:55:55.498852Z","shell.execute_reply.started":"2025-12-14T15:52:17.631259Z","shell.execute_reply":"2025-12-14T15:55:55.498136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\ndomains = ['MF (Molecular Function)', 'BP (Biological Process)', 'CC (Cellular Component)']\n\nraw_scores = [0.6053, 0.3839, 0.5639] \nhie_scores = [0.6126, 0.3949, 0.5753] \n\n# Config\nx = np.arange(len(domains))  # Vị trí các nhóm trên trục X\nwidth = 0.35  # Độ rộng của mỗi cột\n\nfig, ax = plt.subplots(figsize=(10, 6))\n\nrects1 = ax.bar(x - width/2, raw_scores, width, label='Stage 1: Raw Output', color='#95a5a6', edgecolor='black', alpha=0.8)\nrects2 = ax.bar(x + width/2, hie_scores, width, label='Stage 2: + Hierarchy', color='#1abc9c', edgecolor='black')\n\nax.set_ylabel('Micro F1-Score', fontweight='bold', fontsize=11)\nax.set_title('Impact of Hierarchical Consistency Enforcement on F1-Score', fontweight='bold', fontsize=12, pad=20)\nax.set_xticks(x)\nax.set_xticklabels(domains, fontweight='bold')\n\nax.set_ylim(0, 0.8) \n\nax.legend(loc='lower right', fontsize=10, frameon=True, shadow=True)\nax.grid(axis='y', linestyle='--', alpha=0.5)\n\ndef autolabel(rects):\n    for rect in rects:\n        height = rect.get_height()\n        ax.annotate(f'{height:.4f}',\n                    xy=(rect.get_x() + rect.get_width() / 2, height),\n                    xytext=(0, 3),\n                    textcoords=\"offset points\",\n                    ha='center', va='bottom', fontsize=9, fontweight='bold', color='black')\n\nautolabel(rects1)\nautolabel(rects2)\n\nfig.tight_layout()\n\n# Lưu ảnh\nimg_name = 'hierarchy_impact_barchart.png'\nplt.savefig(img_name, dpi=300, bbox_inches='tight')\nplt.show()\nprint(f\"✅ Đã vẽ xong! Lưu ảnh tại: {img_name}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T16:30:38.045056Z","iopub.execute_input":"2025-12-14T16:30:38.045683Z","iopub.status.idle":"2025-12-14T16:30:38.696778Z","shell.execute_reply.started":"2025-12-14T16:30:38.045658Z","shell.execute_reply":"2025-12-14T16:30:38.696015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load prediction và lưu file\n\nimport pickle\nimport numpy as np\nfrom tqdm import tqdm\nimport gc\nimport obonet\nimport os\n\n# Config\nTHRESHOLD_CONFIG = {\n    'MF': 0.02, \n    'CC': 0.02, \n    'BP': 0.05  # BP cần lọc kỹ hơn vì nó rất nhiều nhãn\n}\n\nTOP_K_CONFIG = {\n    'MF': 30,\n    'CC': 30,\n    'BP': 60\n}\n\n# Chuẩn bị cây GO\nprint(\"🌳 Đang load cây GO (Gene Ontology)...\")\nobo_path = \"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\"\ngo_graph = obonet.read_obo(obo_path)\n\nparents_map = {term: [p.split(\" ! \")[0] for p in data.get(\"is_a\", [])] for term, data in go_graph.nodes(data=True)}\nprint(\"✅ Đã load xong quan hệ cha-con.\")\n\n# Hàm Enforce Hierarchy\ndef enforce_hierarchy_batch(probs, classes, parent_map):\n    term_to_idx = {term: i for i, term in enumerate(classes)}\n    \n    # Duyệt qua từng nhãn\n    for i, term in enumerate(classes):\n        if term not in parent_map: continue\n        \n        for p in parent_map[term]:\n            if p in term_to_idx:\n                p_idx = term_to_idx[p]\n                probs[:, p_idx] = np.maximum(probs[:, p_idx], probs[:, i])\n    return probs\n\n# Hàm ghi file\ndef write_output_final(namespace):\n    print(f\"\\n{'='*60}\")\n    print(f\"🚀 XỬ LÝ & GHI OUTPUT: {namespace}\")\n    print(f\"{'='*60}\")\n    \n    thresh = THRESHOLD_CONFIG.get(namespace, 0.01)\n    k_limit = TOP_K_CONFIG.get(namespace, 60)\n    print(f\"⚙️ Config: Threshold >= {thresh} | Top-K <= {k_limit}\")\n\n    print(\"Mw Đang load file pickle...\")\n    pred_file = f\"pred_probs_{namespace}.pkl\"\n    \n    try:\n        with open(pred_file, \"rb\") as f:\n            data = pickle.load(f)\n        \n        test_ids = data['test_ids']\n        all_test_probs = data['all_test_probs'] \n        go_classes = data['go_classes']\n        \n        del data\n        gc.collect()\n        \n    except Exception as e:\n        print(f\"❌ Lỗi đọc file {pred_file}: {e}\")\n        return\n\n    output_file = f\"pred_{namespace}.tsv\"\n    BATCH_SIZE = 5000  \n    total_samples = len(test_ids)\n    \n    print(f\"🔨 Đang xử lý {total_samples} protein (Gia phả + Lọc)...\")\n    \n    count_lines = 0\n    with open(output_file, \"w\") as f:\n        for start_idx in tqdm(range(0, total_samples, BATCH_SIZE), desc=f\"Processing {namespace}\"):\n            end_idx = min(start_idx + BATCH_SIZE, total_samples)\n            \n            batch_ids = test_ids[start_idx : end_idx]\n            batch_probs = all_test_probs[start_idx : end_idx].copy()\n \n            for _ in range(2):\n                batch_probs = enforce_hierarchy_batch(batch_probs, go_classes, parents_map)\n            \n            lines_buffer = []\n            \n            for i, pid in enumerate(batch_ids):\n                row_probs = batch_probs[i]\n                \n                indices = np.where(row_probs >= thresh)[0]\n                \n                if len(indices) > k_limit:\n                    filtered_scores = row_probs[indices]\n                    top_k_idx_local = np.argsort(filtered_scores)[::-1][:k_limit]\n                    indices = indices[top_k_idx_local]\n                \n                for idx in indices:\n                    go_id = go_classes[idx]\n                    score = row_probs[idx]\n                    # lines_buffer.append(f\"{pid}\\t{go_id}\\t{score:.3f}\")\n                    lines_buffer.append(f\"{pid}\\t{go_id}\\t{score:.4f}\")\n            \n            if lines_buffer:\n                f.write(\"\\n\".join(lines_buffer) + \"\\n\")\n                count_lines += len(lines_buffer)\n            \n            del batch_probs, lines_buffer\n            \n    print(f\"✅ Xong! Đã tạo file: {output_file}\")\n    print(f\"📊 Tổng số dòng: {count_lines}\")\n    \n    del test_ids, all_test_probs, go_classes\n    gc.collect()\n\n# Chạy\nwrite_output_final('MF')\nwrite_output_final('BP')\nwrite_output_final('CC')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:27:34.836244Z","iopub.status.idle":"2025-12-14T14:27:34.836573Z","shell.execute_reply.started":"2025-12-14T14:27:34.836398Z","shell.execute_reply":"2025-12-14T14:27:34.836411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Gom file submission.tsv\nimport os\nimport pandas as pd\nfrom tqdm import tqdm\nimport gc\n\nINPUT_FILES = [\"pred_MF.tsv\", \"pred_BP.tsv\", \"pred_CC.tsv\"]\nOUTPUT_FILE = \"submission.tsv\"\nCHUNK_SIZE = 500_000 \n\nif os.path.exists(OUTPUT_FILE):\n    os.remove(OUTPUT_FILE)\n\nprint(f\"🚀 Bắt đầu tạo file: {OUTPUT_FILE} (Chế độ Turbo: Không Sort)\")\ntotal_rows_written = 0\n\nfor f_path in INPUT_FILES:\n    if not os.path.exists(f_path):\n        continue\n    \n    print(f\"{'='*40}\")\n    print(f\"📂 Đang xử lý: {os.path.basename(f_path)}\")\n    \n    try:\n        # Đếm dòng để hiện thanh % \n        num_lines = sum(1 for _ in open(f_path, 'rb'))\n        chunks_total = (num_lines // CHUNK_SIZE) + 1\n\n        reader = pd.read_csv(\n            f_path, sep='\\t', header=None, \n            names=['id','go','score'], \n            dtype={'id': str, 'go': str, 'score': float},\n            chunksize=CHUNK_SIZE\n        )\n        \n        for chunk in tqdm(reader, total=chunks_total, desc=f\"Writing\", unit=\"chunk\"):         \n            chunk.to_csv(OUTPUT_FILE, sep='\\t', header=False, index=False, float_format='%.3f', mode='a')\n            total_rows_written += len(chunk)\n\n        del reader\n        gc.collect()\n        \n    except Exception as e:\n        print(f\"❌ Lỗi: {e}\")\n\nprint(f\"\\n✅ XONG! Tổng số dòng: {total_rows_written:,}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-14T14:27:34.837519Z","iopub.status.idle":"2025-12-14T14:27:34.837742Z","shell.execute_reply.started":"2025-12-14T14:27:34.837639Z","shell.execute_reply":"2025-12-14T14:27:34.837649Z"}},"outputs":[],"execution_count":null}]}