{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"},{"sourceId":284979869,"sourceType":"kernelVersion"}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q tensorflow","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:16.037533Z","iopub.execute_input":"2025-12-10T15:00:16.037773Z","iopub.status.idle":"2025-12-10T15:00:22.576515Z","shell.execute_reply.started":"2025-12-10T15:00:16.03775Z","shell.execute_reply":"2025-12-10T15:00:22.575772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport os\nimport gc\nfrom sklearn.model_selection import train_test_split\n\n# ==========================================\n# 1. CẤU HÌNH (CONFIG)\n# ==========================================\nclass CFG:\n    RAW_DIR = \"/kaggle/input/cafa-6-protein-function-prediction\"\n    TRAIN_TERMS = os.path.join(RAW_DIR, \"Train/train_terms.tsv\")\n    IA_PATH = os.path.join(RAW_DIR, \"IA.tsv\")\n    OBO_PATH = os.path.join(RAW_DIR, \"Train/go-basic.obo\")\n    \n    # ĐƯỜNG DẪN EMBEDDING (Check kỹ lại đường dẫn input của bạn)\n    EMBED_DIR = \"/kaggle/input/notebook1cf1cfb684\" \n    TRAIN_EMBEDS = os.path.join(EMBED_DIR, \"train_embeddings_esm2.npy\")\n    TRAIN_IDS = os.path.join(EMBED_DIR, \"train_ids_esm2.npy\")\n    TEST_EMBEDS = os.path.join(EMBED_DIR, \"test_embeddings_esm2.npy\")\n    TEST_IDS = os.path.join(EMBED_DIR, \"test_ids_esm2.npy\")\n    \n    INPUT_DIM = 1280\n    BATCH_SIZE = 256\n    EPOCHS = 20\n    LEARNING_RATE = 1e-3\n    \n    # Giữ nguyên key là BPO/CCO/MFO để dễ quản lý\n    NUM_LABELS = {'BPO': 1500, 'CCO': 800, 'MFO': 800}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:22.577552Z","iopub.execute_input":"2025-12-10T15:00:22.577847Z","iopub.status.idle":"2025-12-10T15:00:43.294892Z","shell.execute_reply.started":"2025-12-10T15:00:22.577813Z","shell.execute_reply":"2025-12-10T15:00:43.293946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# 2. HÀM LOAD & PREPARE DỮ LIỆU\n# ==========================================\ndef load_dataset_fixed():\n    print(\"--- 1. LOADING & CLEANING IDs ---\")\n    \n    # A. Load Embeddings\n    X = np.load(CFG.TRAIN_EMBEDS)\n    train_ids_raw = np.load(CFG.TRAIN_IDS)\n    \n    # B. Clean Embeddings IDs (Cắt bỏ 'sp|...|')\n    train_ids_clean = []\n    for uid in train_ids_raw:\n        uid_str = str(uid).strip()\n        if '|' in uid_str:\n            parts = uid_str.split('|')\n            if len(parts) >= 2:\n                train_ids_clean.append(parts[1])\n            else:\n                train_ids_clean.append(uid_str)\n        else:\n            train_ids_clean.append(uid_str)\n            \n    # Ép kiểu string\n    train_ids_clean = [str(x) for x in train_ids_clean]\n    \n    # Tạo Dictionary Map\n    id_to_idx = {uid: i for i, uid in enumerate(train_ids_clean)}\n    \n    # C. Load Labels\n    terms_df = pd.read_csv(CFG.TRAIN_TERMS, sep='\\t')\n    terms_df['EntryID'] = terms_df['EntryID'].astype(str).str.strip()\n    \n    # Check khớp lệnh\n    common = set(train_ids_clean).intersection(set(terms_df['EntryID']))\n    print(f\"   > IDs khớp nhau: {len(common)}\")\n    \n    return X, id_to_idx, terms_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:43.296485Z","iopub.execute_input":"2025-12-10T15:00:43.296885Z","iopub.status.idle":"2025-12-10T15:00:43.303444Z","shell.execute_reply.started":"2025-12-10T15:00:43.296866Z","shell.execute_reply":"2025-12-10T15:00:43.302633Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_ia_weights(terms_list, ia_path):\n    \"\"\"\n    Tạo vector trọng số (weights) khớp với danh sách nhãn đang train.\n    \"\"\"\n    print(f\"Loading IA weights from {ia_path}...\")\n    ia_df = pd.read_csv(ia_path, sep='\\t', header=None, names=['term', 'score'])\n    \n    # Tạo dict để tra cứu nhanh\n    ia_dict = dict(zip(ia_df['term'], ia_df['score']))\n    \n    # Map weights theo thứ tự của terms_list (các cột output của model)\n    weights = []\n    for term in terms_list:\n        # Nếu term không có trong file IA, mặc định weight = 1.0\n        weights.append(ia_dict.get(term, 1.0))\n        \n    return np.array(weights, dtype=np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:43.30436Z","iopub.execute_input":"2025-12-10T15:00:43.304942Z","iopub.status.idle":"2025-12-10T15:00:43.350917Z","shell.execute_reply.started":"2025-12-10T15:00:43.304911Z","shell.execute_reply":"2025-12-10T15:00:43.350222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_data_for_aspect(aspect_name, X, id_to_idx, terms_df):\n    print(f\"\\n--- PREPARING {aspect_name} ---\")\n    \n    # [SỬA LỖI QUAN TRỌNG] Mapping từ tên Aspect sang ký hiệu trong file (P, C, F)\n    # BPO -> P, CCO -> C, MFO -> F\n    aspect_map = {\n        'BPO': 'P',\n        'CCO': 'C',\n        'MFO': 'F'\n    }\n    target_code = aspect_map[aspect_name]\n    print(f\"   > Mapping {aspect_name} -> '{target_code}' in dataset\")\n    \n    # 1. Filter aspect\n    df_aspect = terms_df[terms_df['aspect'] == target_code].copy()\n    \n    if len(df_aspect) == 0:\n        raise ValueError(f\"Không tìm thấy dữ liệu cho code '{target_code}'. Kiểm tra lại file terms!\")\n\n    # 2. Top-K Terms\n    top_k = CFG.NUM_LABELS[aspect_name]\n    top_terms = df_aspect['term'].value_counts().head(top_k).index.tolist()\n    df_aspect = df_aspect[df_aspect['term'].isin(top_terms)]\n    \n    # 3. Pivot Table (Tạo One-Hot)\n    print(\"   > Creating pivot table (this may take a moment)...\")\n    df_aspect['val'] = 1\n    label_matrix = df_aspect.pivot_table(index='EntryID', columns='term', values='val', fill_value=0)\n    \n    # Ép kiểu index\n    label_ids = [str(x) for x in label_matrix.index]\n    \n    # 4. Tìm giao thoa\n    valid_ids = list(set(label_ids).intersection(set(id_to_idx.keys())))\n    \n    print(f\"   > Label Matrix Rows: {len(label_matrix)}\")\n    print(f\"   > Valid IDs (Intersection): {len(valid_ids)}\")\n    \n    if len(valid_ids) == 0:\n        raise ValueError(f\"Không tìm thấy protein nào cho aspect {aspect_name}!\")\n\n    # 5. Tạo dữ liệu train\n    # Lấy vector embedding tương ứng\n    indices = [id_to_idx[uid] for uid in valid_ids]\n    X_sub = X[indices]\n    \n    # Lấy nhãn tương ứng (cần reindex label_matrix theo valid_ids để đảm bảo thứ tự)\n    # Lưu ý: label_matrix.loc[valid_ids] sẽ tự sắp xếp theo thứ tự valid_ids\n    Y_sub = label_matrix.loc[valid_ids].values\n    \n    print(f\"   > Final Train Data: X={X_sub.shape}, Y={Y_sub.shape}\")\n    return X_sub, Y_sub, label_matrix.columns.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:43.351739Z","iopub.execute_input":"2025-12-10T15:00:43.352027Z","iopub.status.idle":"2025-12-10T15:00:43.370047Z","shell.execute_reply.started":"2025-12-10T15:00:43.352002Z","shell.execute_reply":"2025-12-10T15:00:43.369511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def build_go_graph(obo_path):\n    print(f\"Parsing Ontology Graph form {obo_path}...\")\n    parents = {} # Dict lưu: child -> [list of parents]\n    \n    current_term = None\n    \n    with open(obo_path, 'r') as f:\n        for line in f:\n            line = line.strip()\n            if line == '[Term]':\n                current_term = None\n            elif line.startswith('id: GO:'):\n                current_term = line.replace('id: ', '')\n            elif line.startswith('is_a: GO:') and current_term:\n                # is_a: GO:0048308 ! ...\n                parent = line.split(' ! ')[0].replace('is_a: ', '')\n                if current_term not in parents:\n                    parents[current_term] = []\n                parents[current_term].append(parent)\n                \n    return parents\n\ndef propagate_scores(preds, terms_list, parents_dict):\n    \"\"\"\n    Lan truyền điểm số: Điểm của cha = Max(Điểm của cha, Điểm của con)\n    Lưu ý: Để tối ưu tốc độ, ta chỉ làm đơn giản 1 vòng hoặc dùng logic ma trận.\n    Ở đây dùng logic đơn giản: Duyệt ngược từ nhãn con cập nhật lên nhãn cha.\n    \"\"\"\n    print(\"Propagating scores (Consistency enforcing)...\")\n    \n    # Tạo map từ Term ID -> Index cột trong preds\n    term_to_idx = {term: i for i, term in enumerate(terms_list)}\n    \n    # Convert preds sang DataFrame để dễ xử lý (hoặc xử lý trực tiếp trên numpy)\n    # Xử lý trực tiếp trên numpy cho nhanh\n    # preds shape: (n_samples, n_classes)\n    \n    # Để lan truyền chuẩn xác, ta cần topological sort, nhưng đơn giản hóa:\n    # Lặp vài lần để điểm lan truyền hết từ lá lên gốc\n    \n    for _ in range(2): # Chạy 2 pass là đủ phủ hầu hết graph nông\n        for child, parent_list in parents_dict.items():\n            # Nếu child không có trong tập dự đoán -> bỏ qua\n            if child not in term_to_idx:\n                continue\n            \n            child_idx = term_to_idx[child]\n            \n            for parent in parent_list:\n                # Nếu parent có trong tập dự đoán\n                if parent in term_to_idx:\n                    parent_idx = term_to_idx[parent]\n                    \n                    # Logic: Score Parent ít nhất phải bằng Score Child\n                    # Dùng np.maximum để tính toán song song cho toàn bộ batch\n                    preds[:, parent_idx] = np.maximum(preds[:, parent_idx], preds[:, child_idx])\n                    \n    return preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:43.370731Z","iopub.execute_input":"2025-12-10T15:00:43.370972Z","iopub.status.idle":"2025-12-10T15:00:43.393474Z","shell.execute_reply.started":"2025-12-10T15:00:43.37095Z","shell.execute_reply":"2025-12-10T15:00:43.39273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_weighted_loss(weights):\n    \"\"\"\n    Closure function để tạo hàm loss với weights cố định.\n    Chuyển weights sang Tensor ngay từ đầu để tránh overhead khi train.\n    \"\"\"\n    # Chuyển numpy array sang constant tensor để TF tối ưu đồ thị tính toán\n    weight_tensor = tf.constant(weights, dtype=tf.float32)\n    \n    def weighted_binary_crossentropy(y_true, y_pred):\n        # Tính BCE chuẩn\n        bce = tf.keras.backend.binary_crossentropy(y_true, y_pred)\n        # Nhân trọng số\n        weighted_bce = bce * weight_tensor\n        return tf.reduce_mean(weighted_bce)\n        \n    return weighted_binary_crossentropy","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:43.394117Z","iopub.execute_input":"2025-12-10T15:00:43.394323Z","iopub.status.idle":"2025-12-10T15:00:43.41402Z","shell.execute_reply.started":"2025-12-10T15:00:43.394306Z","shell.execute_reply":"2025-12-10T15:00:43.413325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_model(num_classes, class_weights):\n    \n    model = tf.keras.Sequential([\n        tf.keras.layers.Input(shape=(CFG.INPUT_DIM,)),\n        tf.keras.layers.Dense(1024, activation='relu'),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.2),\n        tf.keras.layers.Dense(512, activation='relu'),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.2),\n        tf.keras.layers.Dense(512, activation='relu'),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.2),\n        tf.keras.layers.Dense(num_classes, activation='sigmoid')\n    ])\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=CFG.LEARNING_RATE), \n        loss=get_weighted_loss(class_weights),\n        metrics=['binary_accuracy', tf.keras.metrics.AUC(name='auc')]\n    )\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:43.414706Z","iopub.execute_input":"2025-12-10T15:00:43.414875Z","iopub.status.idle":"2025-12-10T15:00:43.433351Z","shell.execute_reply.started":"2025-12-10T15:00:43.414861Z","shell.execute_reply":"2025-12-10T15:00:43.432629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# A. Load Global Data\nX_global, id_to_idx, terms_df = load_dataset_fixed()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:43.435474Z","iopub.execute_input":"2025-12-10T15:00:43.435664Z","iopub.status.idle":"2025-12-10T15:00:46.89284Z","shell.execute_reply.started":"2025-12-10T15:00:43.435649Z","shell.execute_reply":"2025-12-10T15:00:46.892036Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# B. Load Test Data\nprint(\"\\nLoading Test Data...\")\nif os.path.exists(CFG.TEST_EMBEDS):\n    X_test = np.load(CFG.TEST_EMBEDS)\n    test_ids = np.load(CFG.TEST_IDS)\n    print(f\"Test loaded: {X_test.shape}\")\nelse:\n    print(\"WARNING: Test files not found. Creating dummy test data just to run code.\")\n    # Dummy để code không crash nếu bạn chưa có file test\n    X_test = np.zeros((10, CFG.INPUT_DIM)) \n    test_ids = np.array([f\"TEST_{i}\" for i in range(10)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:46.893717Z","iopub.execute_input":"2025-12-10T15:00:46.894335Z","iopub.status.idle":"2025-12-10T15:00:53.838443Z","shell.execute_reply.started":"2025-12-10T15:00:46.894315Z","shell.execute_reply":"2025-12-10T15:00:53.837786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submissions = []\n\ngo_parents = build_go_graph(CFG.OBO_PATH)\n\n# C. Loop Training\nfor aspect in ['BPO', 'CCO', 'MFO']:\n    try:\n        # A. Chuẩn bị dữ liệu\n        # Hàm get_data_for_aspect đã sửa ở bước trước (có map BPO->P)\n        X_train, Y_train, target_terms = get_data_for_aspect(aspect, X_global, id_to_idx, terms_df)\n\n        # B. Load Weights cho Aspect hiện tại\n        class_weights_vector = load_ia_weights(target_terms, CFG.IA_PATH)\n        \n        # C. Split Data\n        x_tr, x_val, y_tr, y_val = train_test_split(X_train, Y_train, test_size=0.1, random_state=42)\n        \n        # D. Tạo Model (Truyền weights vào hàm)\n        print(f\"Training {aspect} model (Output: {len(target_terms)} classes)...\")\n        model = create_model(len(target_terms), class_weights_vector)\n        \n        # Callbacks\n        early_stop = tf.keras.callbacks.EarlyStopping(\n            monitor='val_loss', patience=3, restore_best_weights=True, verbose=1\n        )\n        reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(\n            monitor='val_loss', factor=0.5, patience=2, min_lr=1e-5, verbose=1\n        )\n        \n        # E. Fit\n        model.fit(\n            x_tr, y_tr, \n            validation_data=(x_val, y_val), \n            epochs=CFG.EPOCHS, \n            batch_size=CFG.BATCH_SIZE, \n            callbacks=[early_stop, reduce_lr], \n            verbose=1\n        )\n        \n        # F. Predict\n        print(f\"Predicting {aspect}...\")\n        preds = model.predict(X_test, batch_size=CFG.BATCH_SIZE, verbose=1)\n        \n        # G. [QUAN TRỌNG] Propagate Scores (Tối ưu kết quả)\n        preds = propagate_scores(preds, target_terms, go_parents)\n        \n        # H. Format & Save to RAM\n        df_pred = pd.DataFrame(preds, columns=target_terms)\n        df_pred['EntryID'] = test_ids\n        \n        melted = df_pred.melt(id_vars='EntryID', var_name='term', value_name='score')\n        \n        # Lọc ngưỡng: Chỉ giữ lại điểm > 0.005 để giảm kích thước file\n        # (Ngưỡng này an toàn cho tính F-max)\n        melted = melted[melted['score'] > 0.005] \n        \n        submissions.append(melted)\n        \n        # I. Dọn dẹp RAM triệt để\n        del model, x_tr, x_val, y_tr, y_val, df_pred, melted, preds, X_train, Y_train, class_weights_vector\n        tf.keras.backend.clear_session()\n        gc.collect()\n        \n    except Exception as e:\n        print(f\"ERROR processing {aspect}: {e}\")\n        import traceback\n        traceback.print_exc()\n        continue","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:00:53.83913Z","iopub.execute_input":"2025-12-10T15:00:53.839325Z","iopub.status.idle":"2025-12-10T15:04:14.857781Z","shell.execute_reply.started":"2025-12-10T15:00:53.839309Z","shell.execute_reply":"2025-12-10T15:04:14.857186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==========================================\n# D. SAVE SUBMISSION (TSV FORMAT)\n# ==========================================\nprint(\"\\nSaving final submission...\")\n\nif len(submissions) > 0:\n    # 1. Gộp tất cả các aspect lại\n    final_df = pd.concat(submissions, axis=0, ignore_index=True)\n    \n    # 2. Sắp xếp lại cho đẹp (Protein ID tăng dần, Score giảm dần) - Optional\n    # Giúp file dễ nhìn hơn nếu bạn mở ra check\n    print(\"Sorting data...\")\n    final_df.sort_values(by=['EntryID', 'score'], ascending=[True, False], inplace=True)\n    \n    # 3. Lưu file .tsv\n    output_filename = \"submission.tsv\"\n    \n    print(f\"Writing to {output_filename}...\")\n    # sep='\\t': Dùng tab làm dấu phân cách\n    # index=False: Không lưu cột số thứ tự dòng (0,1,2...)\n    # float_format='%.3f': Làm tròn 3 số thập phân để giảm dung lượng file (Optional)\n    final_df.to_csv(output_filename, sep='\\t', index=False, float_format='%.3f')\n    \n    print(f\"Done! Saved {len(final_df)} rows to {output_filename}\")\n    print(final_df.head())\n\nelse:\n    print(\"No predictions made.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T15:04:14.861481Z","iopub.execute_input":"2025-12-10T15:04:14.861714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}