{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"},{"sourceId":13482672,"sourceType":"datasetVersion","datasetId":8559888},{"sourceId":13914361,"sourceType":"datasetVersion","datasetId":8865984},{"sourceId":14073960,"sourceType":"datasetVersion","datasetId":8954895},{"sourceId":14135009,"sourceType":"datasetVersion","datasetId":9007213},{"sourceId":14147181,"sourceType":"datasetVersion","datasetId":8866598}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Tải file Gene Ontology Annotation (UniProt)\n!wget -c https://ftp.ebi.ac.uk/pub/databases/GO/goa/UNIPROT/goa_uniprot_all.gaf.gz","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T12:34:33.784387Z","iopub.execute_input":"2025-12-09T12:34:33.784630Z","iopub.status.idle":"2025-12-09T12:36:39.513331Z","shell.execute_reply.started":"2025-12-09T12:34:33.784610Z","shell.execute_reply":"2025-12-09T12:36:39.512279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport gzip\nimport os\nimport csv\n\n# --- CẤU HÌNH ---\nSWISSPROT_FASTA = '/kaggle/input/cafa6-benchmark-data/cafa6-benchmark-data/swissprot_2025.fasta'\nGOA_FILE_GZ = 'goa_uniprot_all.gaf.gz'\nCAFA_TRAIN_TERMS = '/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv'\nOUTPUT_MERGED_FILE = 'train_terms_augmented.tsv'\n\n# --- ĐỊNH NGHĨA EVIDENCE CODES ---\nEXPERIMENTAL_CODES = {\n    'EXP', 'IDA', 'IPI', 'IMP', 'IGI', 'IEP', \n    'TAS', 'IC'\n}\n\ndef extract_clean_id(header_line):\n\n    clean = header_line.strip().lstrip('>')\n\n    if '|' in clean:\n        parts = clean.split('|')\n        if len(parts) >= 2:\n            return parts[1]\n\n    return clean.split()[0]\n\ndef get_swissprot_ids(fasta_path):\n    print(f\"[1/3] Đang đọc whitelist từ {fasta_path}...\")\n    valid_ids = set()\n    try:\n        with open(fasta_path, 'r') as f:\n            for line in f:\n                if line.startswith('>'):\n                    clean_id = extract_clean_id(line)\n                    valid_ids.add(clean_id)\n                    \n    except Exception as e:\n        print(f\"❌ Lỗi đọc FASTA: {e}\")\n        return set()\n    \n    print(f\"   => Whitelist: {len(valid_ids):,} Swiss-Prot IDs (Đã chuẩn hóa).\")\n    return valid_ids\n\ndef parse_and_filter_goa_gz(goa_gz_path, valid_ids):\n    print(f\"[2/3] Đang quét file nén {goa_gz_path}...\")\n    \n    new_data = []\n    count = 0\n    match_count = 0\n    \n    try:\n        with gzip.open(goa_gz_path, 'rt') as f:\n            for line in f:\n                if line.startswith('!'): continue \n                \n                parts = line.split('\\t')\n                if len(parts) < 7: continue\n\n                prot_id = parts[1]\n                go_id = parts[4]\n                evidence = parts[6]\n                \n                if evidence in EXPERIMENTAL_CODES:\n                    # So khớp ID sạch với ID sạch trong whitelist\n                    if prot_id in valid_ids:\n                        new_data.append([prot_id, go_id])\n                        match_count += 1\n                \n                count += 1\n                if count % 10000000 == 0: \n                    print(f\"   ... Đã quét {count:,} dòng (Tìm thấy {match_count:,} matches)...\")\n                    \n    except Exception as e:\n        print(f\"❌ Lỗi khi đọc file GZ: {e}\")\n        return pd.DataFrame()\n\n    print(f\"   => TỔNG KẾT: Thu thập được {len(new_data):,} annotations chất lượng cao.\")\n    return pd.DataFrame(new_data, columns=['EntryID', 'term'])\n\ndef merge_and_save(new_df, cafa_train_path, output_path):\n    print(f\"[3/3] Đang hợp nhất và lưu file...\")\n\n    if os.path.exists(cafa_train_path):\n        old_df = pd.read_csv(cafa_train_path, sep='\\t', dtype={'EntryID': str})[['EntryID', 'term']]\n        print(f\"   - Dữ liệu gốc: {len(old_df):,} dòng.\")\n        \n        # Gộp\n        merged_df = pd.concat([old_df, new_df])\n    else:\n        print(\"   - Không thấy file gốc, chỉ dùng dữ liệu mới.\")\n        merged_df = new_df\n\n    before_dedup = len(merged_df)\n    merged_df = merged_df.drop_duplicates(subset=['EntryID', 'term'], keep='first')\n    \n    print(f\"   - Tổng trước khi xóa trùng: {before_dedup:,}\")\n    print(f\"   - Tổng sau khi xóa trùng: {len(merged_df):,} dòng.\")\n    print(f\"   => Đã loại bỏ được {before_dedup - len(merged_df):,} dòng trùng lặp.\")\n    \n    # Lưu file\n    merged_df.to_csv(output_path, sep='\\t', index=False)\n    print(f\"✅ HOÀN TẤT. File đã lưu tại: {output_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T12:36:51.550502Z","iopub.execute_input":"2025-12-09T12:36:51.550827Z","iopub.status.idle":"2025-12-09T12:36:52.033279Z","shell.execute_reply.started":"2025-12-09T12:36:51.550791Z","shell.execute_reply":"2025-12-09T12:36:52.032270Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if os.path.exists(SWISSPROT_FASTA) and os.path.exists(GOA_FILE_GZ):\n    valid_swiss_ids = get_swissprot_ids(SWISSPROT_FASTA)\n    \n    if len(valid_swiss_ids) > 0:\n        df_augmented = parse_and_filter_goa_gz(GOA_FILE_GZ, valid_swiss_ids)\n        if not df_augmented.empty:\n            merge_and_save(df_augmented, CAFA_TRAIN_TERMS, OUTPUT_MERGED_FILE)\n        else:\n            print(\"Không tìm thấy dữ liệu phù hợp nào.\")\n    else:\n        print(\"Whitelist rỗng.\")\nelse:\n    print(f\"❌ Thiếu file đầu vào.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T12:36:55.179784Z","iopub.execute_input":"2025-12-09T12:36:55.180806Z","iopub.status.idle":"2025-12-09T13:13:03.897115Z","shell.execute_reply.started":"2025-12-09T12:36:55.180773Z","shell.execute_reply":"2025-12-09T13:13:03.896176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# --- CẤU HÌNH ---\nAUGMENTED_TERMS_FILE = 'train_terms_augmented.tsv'\nOLD_FASTA = '/kaggle/input/cafa-6-protein-function-prediction/Train/train_sequences.fasta'\nSWISSPROT_FASTA = '/kaggle/input/cafa6-benchmark-data/cafa6-benchmark-data/swissprot_2025.fasta'\nOUTPUT_FASTA = 'train_sequences_augmented.fasta'\n\ndef create_final_train_fasta():\n    print(\"--- TẠO FILE TRAIN FASTA (FINAL) ---\")\n\n    print(\"1. Đọc danh sách ID có nhãn...\")\n    df = pd.read_csv(AUGMENTED_TERMS_FILE, sep='\\t', usecols=['EntryID'])\n    valid_ids = set(df['EntryID'].unique())\n    print(f\"   => Cần sequence cho {len(valid_ids):,} protein.\")\n\n    print(\"2. Đang ghi file...\")\n    seen_ids = set()\n    count = 0\n    \n    with open(OUTPUT_FASTA, 'w') as f_out:\n\n        with open(OLD_FASTA, 'r') as f_in:\n            for line in f_in:\n                if line.startswith('>'):\n                    pid = extract_clean_id(line)\n                    seen_ids.add(pid)\n                    count += 1\n                f_out.write(line)\n                \n        print(f\"   - Đã chép {count:,} sequences từ CAFA6 gốc.\")\n\n        count_added = 0\n        with open(SWISSPROT_FASTA, 'r') as f_in:\n            write_mode = False\n            for line in f_in:\n                if line.startswith('>'):\n                    pid = extract_clean_id(line)\n\n                    if (pid in valid_ids) and (pid not in seen_ids):\n                        write_mode = True\n                        seen_ids.add(pid)\n                        f_out.write(line)\n                        count_added += 1\n                    else:\n                        write_mode = False\n                else:\n                    if write_mode:\n                        f_out.write(line)\n                        \n    print(f\"✅ HOÀN TẤT.\")\n    print(f\"   - Bổ sung thêm: {count_added:,} sequences mới.\")\n    print(f\"   - Tổng cộng: {len(seen_ids):,} sequences.\")\n    print(f\"   - File: {OUTPUT_FASTA}\")\n\ncreate_final_train_fasta()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T13:13:47.441861Z","iopub.execute_input":"2025-12-09T13:13:47.442223Z","iopub.status.idle":"2025-12-09T13:13:54.592682Z","shell.execute_reply.started":"2025-12-09T13:13:47.442195Z","shell.execute_reply":"2025-12-09T13:13:54.591689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport subprocess\nimport pandas as pd\nimport numpy as np\nimport math\nfrom collections import defaultdict, Counter\nimport sys\n\nTRAIN_FASTA = '/kaggle/input/train-augmented/train_sequences_augmented.fasta'\nTRAIN_TERMS = '/kaggle/input/train-augmented/train_terms_augmented.tsv'\nTEST_FASTA  = '/kaggle/input/cafa-6-protein-function-prediction/Test/testsuperset.fasta'\n\nWORK_DIR = './work_diamond'\nOUTPUT_FILE = 'submission.tsv'\nDIAMOND_BIN = './diamond'\n\n# Cài đặt Diamond nếu chưa có\nif not os.path.exists(DIAMOND_BIN):\n    print(\"[SETUP] Đang tải và cài đặt DIAMOND...\")\n    !wget -q http://github.com/bbuchfink/diamond/releases/download/v2.1.8/diamond-linux64.tar.gz\n    !tar xzf diamond-linux64.tar.gz\n    print(\"[SETUP] Hoàn tất.\")\nelse:\n    print(\"[SETUP] DIAMOND đã sẵn sàng.\")\n\nos.makedirs(WORK_DIR, exist_ok=True)\n\ndef build_diamond_database(fasta_file, db_path):\n    if os.path.exists(f\"{db_path}.dmnd\"):\n        print(f\"[INFO] DB đã tồn tại: {db_path}\")\n        return\n    print(f\"[INFO] Đang tạo DB từ {fasta_file}...\")\n    subprocess.run([DIAMOND_BIN, 'makedb', '--in', fasta_file, '--db', db_path, '--quiet'], check=True)\n\ndef run_diamond_alignment(query_fasta, db_path, output_file):\n    print(f\"[INFO] Đang chạy Alignment cho {query_fasta}...\")\n    cmd = [\n        DIAMOND_BIN, 'blastp',\n        '--query', query_fasta,\n        '--db', db_path,\n        '--out', output_file,\n        '--outfmt', '6', 'qseqid', 'sseqid', 'bitscore',\n        '--evalue', '0.001',\n        '--max-target-seqs', '500', \n        '--more-sensitive'\n    ]\n    subprocess.run(cmd, check=True)\n    print(f\"[INFO] Kết quả lưu tại {output_file}\")\n\ndef load_train_annotations(terms_file):\n    print(f\"[INFO] Đang tải dữ liệu nhãn từ {terms_file}...\")\n    try:\n        df = pd.read_csv(terms_file, sep='\\t', dtype=str)\n    except:\n        sys.exit(\"[ERROR] Không đọc được file terms.\")\n\n    id_col, term_col = None, None\n    for col in df.columns:\n        if col in ['EntryID', 'ProteinID', 'UniprotID', 'id']: id_col = col\n        if col in ['term', 'GO_ID', 'go_id']: term_col = col\n    \n    if not id_col or not term_col:\n        print(\"[WARN] Fallback: Đọc không header.\")\n        df = pd.read_csv(terms_file, sep='\\t', header=None, dtype=str)\n        id_col, term_col = df.columns[0], df.columns[1]\n\n    return df.groupby(id_col)[term_col].apply(set).to_dict()\n\ndef calculate_blast_knn_scores(diamond_output, train_annotations, output_submission, \n                               k_neighbors=50, max_output_terms=1000, \n                               train_self_scores=None):\n    print(f\"[INFO] Cấu hình: Top-{k_neighbors} KNN | IC Weighting Global Normalization\")\n    \n    # TÍNH VÀ CHUẨN HÓA IC \n    print(\"[INFO] Đang tính toán Information Content (IC)...\")\n    term_counts = Counter()\n    total_proteins = len(train_annotations)\n    \n    for terms in train_annotations.values():\n        term_counts.update(terms)\n        \n    ic_map = {}\n    max_ic = 0.0 \n    \n    for term, count in term_counts.items():\n        prob = count / total_proteins\n        if prob > 0:\n            val = -math.log2(prob)\n            ic_map[term] = val\n            if val > max_ic: max_ic = val\n        else:\n            ic_map[term] = 0.0\n            \n    print(f\"       => Max IC Global = {max_ic:.4f}. Sẽ dùng giá trị này để chuẩn hóa.\")\n\n    if max_ic > 0:\n        for term in ic_map:\n            ic_map[term] = ic_map[term] / max_ic\n    else:\n        for term in ic_map: ic_map[term] = 1.0\n\n    try:\n        hits_df = pd.read_csv(diamond_output, sep='\\t', names=['q', 's', 'bit'])\n    except pd.errors.EmptyDataError:\n        print(\"[ERROR] File hits rỗng!\")\n        return\n\n    hits_df = hits_df.sort_values(['q', 'bit'], ascending=[True, False])\n\n    count_written = 0\n    with open(output_submission, 'w') as f:\n        for q_id, group in hits_df.groupby('q', sort=False):\n            \n            neighbors = group.head(k_neighbors)\n            total_w = 0.0\n            term_scores = defaultdict(float)\n            \n            for _, row in neighbors.iterrows():\n                s_id_raw = str(row['s'])\n                raw_score = row['bit']\n\n                real_s_id = None\n                if s_id_raw in train_annotations: real_s_id = s_id_raw\n                elif '|' in s_id_raw:\n                    parts = s_id_raw.split('|')\n                    if len(parts) > 1 and parts[1] in train_annotations: real_s_id = parts[1]\n                elif ' ' in s_id_raw:\n                     simple_id = s_id_raw.split()[0]\n                     if simple_id in train_annotations: real_s_id = simple_id\n\n                # Tính trọng số\n                weight = raw_score\n                if real_s_id and train_self_scores and real_s_id in train_self_scores:\n                    self_score = train_self_scores[real_s_id]\n                    if self_score > 0: weight = raw_score / self_score\n\n                if real_s_id:\n                    total_w += weight\n                    found_terms = train_annotations[real_s_id]\n                    for term in found_terms:\n\n                        term_scores[term] += weight * ic_map.get(term, 0.0)\n\n            if total_w == 0: continue\n\n            prediction_list = []\n            for term, weighted_score in term_scores.items():\n\n                final_score = weighted_score / total_w\n                \n                if final_score > 0.001: # Ngưỡng lọc\n                    prediction_list.append((term, final_score))\n            \n            prediction_list.sort(key=lambda x: x[1], reverse=True)\n            final_predictions = prediction_list[:max_output_terms]\n            \n            for term, score in final_predictions:\n\n                score = min(score, 1.0)\n                f.write(f\"{q_id}\\t{term}\\t{score:.4f}\\n\")\n                count_written += 1\n\n    print(f\"[SUCCESS] Đã xong. Output chuẩn xác suất: {output_submission}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T04:54:03.307159Z","iopub.execute_input":"2025-12-15T04:54:03.307659Z","iopub.status.idle":"2025-12-15T04:54:07.468582Z","shell.execute_reply.started":"2025-12-15T04:54:03.307445Z","shell.execute_reply":"2025-12-15T04:54:07.467356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Tạo DB\ndb_file = os.path.join(WORK_DIR, \"train_db\")\nbuild_diamond_database(TRAIN_FASTA, db_file)\n\n# 2. Chạy Alignment\nhits_file = os.path.join(WORK_DIR, \"hits.tsv\")\nrun_diamond_alignment(TEST_FASTA, db_file, hits_file)\n\n# 3. Load Annotations\ntrain_annots = load_train_annotations(TRAIN_TERMS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-15T04:54:19.829331Z","iopub.execute_input":"2025-12-15T04:54:19.830267Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 4. Tính điểm\ncalculate_blast_knn_scores(\n    diamond_output=hits_file, \n    train_annotations=train_annots, \n    output_submission=OUTPUT_FILE, \n    k_neighbors=50,       \n    max_output_terms=1000,\n    train_self_scores=None\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T13:45:26.655379Z","iopub.execute_input":"2025-12-09T13:45:26.655824Z","iopub.status.idle":"2025-12-09T13:48:43.838434Z","shell.execute_reply.started":"2025-12-09T13:45:26.655799Z","shell.execute_reply":"2025-12-09T13:48:43.837663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nfile_to_delete = \"/kaggle/working/goa_uniprot_all.gaf.gz\" \n\nif os.path.exists(file_to_delete):\n    os.remove(file_to_delete)\n    print(f\"✅ Đã xóa thành công: {file_to_delete}\")\nelse:\n    print(f\"⚠️ File không tồn tại: {file_to_delete}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport os\nfrom collections import defaultdict\n\nOBO_FILE = '/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo'\n\nclass GODag:\n    def __init__(self):\n        self.graph = defaultdict(set)\n        self.ancestors_cache = {}\n        if os.path.exists(OBO_FILE):\n            self.load_obo(OBO_FILE)\n        else:\n            print(f\"❌ Lỗi: Không tìm thấy file OBO tại {OBO_FILE}\")\n\n    def load_obo(self, obo_path):\n        print(f\"[OBO] Đang đọc cấu trúc Gene Ontology từ {obo_path}...\")\n        current_term = None\n        \n        with open(obo_path, 'r') as f:\n            for line in f:\n                line = line.strip()\n                if line == '[Term]':\n                    current_term = None\n                elif line.startswith('id: GO:'):\n                    current_term = line[4:]\n                elif line.startswith('is_a: GO:') and current_term:\n                    parent = line[6:].split('!')[0].strip()\n                    self.graph[current_term].add(parent)\n                elif line.startswith('relationship: part_of GO:') and current_term:\n                    parent = line.split()[2]\n                    self.graph[current_term].add(parent)\n                elif line.startswith('is_obsolete: true'):\n                    if current_term in self.graph:\n                        del self.graph[current_term]\n                    current_term = None\n\n        print(f\"[OBO] Đã load {len(self.graph):,} mối quan hệ cha-con.\")\n\n    def get_ancestors(self, term):\n        if term in self.ancestors_cache:\n            return self.ancestors_cache[term]\n        \n        ancestors = set()\n        if term in self.graph:\n            parents = self.graph[term]\n            ancestors.update(parents)\n            for p in parents:\n                ancestors.update(self.get_ancestors(p))\n        \n        self.ancestors_cache[term] = ancestors\n        return ancestors\n\ndef propagate_scores_inplace(io_file):\n    print(\"\\n=== BẮT ĐẦU LAN TRUYỀN ĐIỂM SỐ (IN-PLACE) ===\")\n\n    if not os.path.exists(OBO_FILE):\n        print(\"❌ Lỗi: File OBO mặc định không tồn tại.\")\n        return\n\n    go_dag = GODag()\n\n    print(f\"[DATA] Đang load dự đoán từ {io_file}...\")\n    data = defaultdict(dict)\n    count_lines = 0\n    \n    if not os.path.exists(io_file):\n        print(f\"❌ Lỗi: Không tìm thấy file {io_file}\")\n        return\n\n    with open(io_file, 'r') as f:\n        for line in f:\n            parts = line.strip().split('\\t')\n            if len(parts) < 3: continue\n            \n            p_id, term_id, score = parts[0], parts[1], float(parts[2])\n            data[p_id][term_id] = score\n            count_lines += 1\n            \n    print(f\"[DATA] Đã load {count_lines:,} dòng. Đang xử lý lan truyền...\")\n\n    temp_file = io_file + \".tmp\"\n    \n    try:\n        with open(temp_file, 'w') as f_out:\n            for p_id, terms_dict in data.items():\n                final_scores = terms_dict.copy()\n\n                for term, score in terms_dict.items():\n                    ancestors = go_dag.get_ancestors(term)\n                    for ancestor in ancestors:\n                        current_anc_score = final_scores.get(ancestor, 0.0)\n                        if score > current_anc_score:\n                            final_scores[ancestor] = score\n\n                sorted_terms = sorted(final_scores.items(), key=lambda x: x[1], reverse=True)\n                \n                for term, score in sorted_terms:\n                    if score >= 0.001: \n                        f_out.write(f\"{p_id}\\t{term}\\t{score:.4f}\\n\")\n        \n        os.replace(temp_file, io_file)\n        print(f\"✅ HOÀN TẤT. Đã ghi đè (update) file: {io_file}\")\n        \n    except Exception as e:\n        print(f\"❌ Có lỗi xảy ra: {e}\")\n        if os.path.exists(temp_file):\n            os.remove(temp_file)\n        print(\"File gốc vẫn được giữ nguyên.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-09T13:49:22.095878Z","iopub.execute_input":"2025-12-09T13:49:22.096283Z","iopub.status.idle":"2025-12-09T13:50:28.713942Z","shell.execute_reply.started":"2025-12-09T13:49:22.096254Z","shell.execute_reply":"2025-12-09T13:50:28.712840Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if os.path.exists('submission.tsv'):\n    propagate_scores_inplace('submission.tsv')\nelse:\n    print(\"❌ Thiếu file Submission.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}