{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"},{"sourceId":14115963,"sourceType":"datasetVersion","datasetId":8992374}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Cell 1: Import & Config & Seeding\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport gc\nimport random\nimport os\n\ndef seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed) \n    \n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    \n    print(f\"Set Global Seed: {seed}\")\n\nSEED = 42\nseed_everything(SEED)\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {DEVICE}\")\n\nINPUT_EMBED_DIR = \"/kaggle/input/cafa6-t5-embeddings\"\n\nCONFIG = {\n    \"num_labels\": 1500,\n    \"batch_size\": 128,\n    \"lr\": 0.0007,      \n    \"epochs\": 22,      \n    \"seed\": SEED,\n    \"paths\": {\n        \"train_embeds\": f\"{INPUT_EMBED_DIR}/train_embeddings_esm2.npy\",\n        \"train_ids\":    f\"{INPUT_EMBED_DIR}/train_ids_esm2.npy\",\n        \"test_embeds\":  f\"{INPUT_EMBED_DIR}/test_embeddings_esm2.npy\",\n        \"test_ids\":     f\"{INPUT_EMBED_DIR}/test_ids_esm2.npy\",\n        \"train_terms\":  \"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\"\n    }\n}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 2: Load Embeddings\nprint(\"Loading embeddings from disk...\")\n\n# Load Train\ntrain_embeds = np.load(CONFIG['paths']['train_embeds'])\ntrain_ids = np.load(CONFIG['paths']['train_ids'], allow_pickle=True)\n\n# Load Test\ntest_embeds = np.load(CONFIG['paths']['test_embeds'])\ntest_ids = np.load(CONFIG['paths']['test_ids'], allow_pickle=True)\n\n# Tự động cập nhật kích thước đầu vào (Input Dimension)\n# Giúp code chạy đúng dù vector là T5 (1024) hay ESM2 (1280/2560)\nEMBED_DIM = train_embeds.shape[1]\n\nprint(f\"--- Data Loaded ---\")\nprint(f\"Train shape: {train_embeds.shape}\")\nprint(f\"Test shape:  {test_embeds.shape}\")\nprint(f\"Embedding Dimension detected: {EMBED_DIM}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 3: Prepare Targets (SỬA LỖI ĐỊNH DẠNG ID: sp|ID|NAME -> ID)\nimport gc\n\nprint(\"--- Processing Targets & Fixing ID Format ---\")\n\n# 1. Load dữ liệu\ntrain_terms = pd.read_csv(CONFIG['paths']['train_terms'], sep=\"\\t\")\ntrain_ids = np.load(CONFIG['paths']['train_ids'], allow_pickle=True)\n\n# --- SỬA LỖI LOGIC TẠI ĐÂY ---\ndef clean_id(pid):\n    # Chuyển bytes sang string nếu cần\n    if isinstance(pid, bytes):\n        pid = pid.decode('utf-8')\n    pid_str = str(pid).strip()\n    \n    # Logic tách chuỗi: \"sp|A0A0C5B5G6|MOTSC_HUMAN\" -> lấy \"A0A0C5B5G6\"\n    # Tách bằng dấu gạch đứng '|' và lấy phần tử thứ 2 (index 1)\n    parts = pid_str.split('|')\n    if len(parts) > 1:\n        return parts[1] # Lấy mã ở giữa\n    return pid_str # Trả về nguyên gốc nếu không tìm thấy dấu |\n\n# Áp dụng hàm sửa lỗi\ntrain_ids_clean = [clean_id(pid) for pid in train_ids]\n\nprint(f\"[DEBUG] ID gốc: {train_ids[0]}\")\nprint(f\"[DEBUG] ID sau khi sửa: {train_ids_clean[0]}\") # Mong đợi: A0A0C5B5G6\n\n# Tạo map\nid_map = {pid: i for i, pid in enumerate(train_ids_clean)}\n\n# 2. Chọn Top Labels\ntop_terms = train_terms['term'].value_counts().index[:CONFIG['num_labels']]\nterm_to_idx = {term: i for i, term in enumerate(top_terms)}\n\n# 3. Tạo ma trận Targets\nnum_samples = len(train_ids)\nlabels_matrix = np.zeros((num_samples, CONFIG['num_labels']), dtype=np.float32)\n\n# Lọc dữ liệu\ntrain_terms['EntryID'] = train_terms['EntryID'].astype(str).str.strip()\nfiltered_terms = train_terms[\n    (train_terms['EntryID'].isin(id_map)) & \n    (train_terms['term'].isin(top_terms))\n]\n\nprint(f\"Số lượng dòng khớp được: {len(filtered_terms)}\")\n\nif len(filtered_terms) == 0:\n    print(\"Error\")\nelse:\n    print(\"Đang tạo ma trận nhãn...\")\n    for pid, term in tqdm(zip(filtered_terms['EntryID'], filtered_terms['term']), total=len(filtered_terms)):\n        row_idx = id_map[pid]\n        col_idx = term_to_idx[term]\n        labels_matrix[row_idx, col_idx] = 1.0\n        \n    print(f\"THÀNH CÔNG! Tổng số nhãn dương: {labels_matrix.sum()}\")\n\ndel train_terms, filtered_terms, train_ids_clean\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 4: Enhanced Hybrid Model\nclass ProteinDataset(Dataset):\n    def __init__(self, embeddings, targets=None, ids=None):\n        self.embeddings = embeddings\n        self.targets = targets\n        self.ids = ids\n        \n    def __len__(self):\n        return len(self.embeddings)\n    \n    def __getitem__(self, idx):\n        embed = torch.tensor(self.embeddings[idx], dtype=torch.float32)\n        if self.targets is not None:\n            target = torch.tensor(self.targets[idx], dtype=torch.float32)\n            return embed, target\n        return embed, self.ids[idx]\n\nclass ResidualCNNBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size=3, dropout=0.1):\n        super().__init__()\n        padding = kernel_size // 2\n        self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=padding)\n        self.bn1 = nn.BatchNorm1d(out_channels)\n        self.act = nn.GELU() # Đổi sang GELU mượt hơn\n        self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=padding)\n        self.bn2 = nn.BatchNorm1d(out_channels)\n        self.dropout = nn.Dropout(dropout)\n        \n        self.shortcut = nn.Sequential()\n        if in_channels != out_channels:\n            self.shortcut = nn.Conv1d(in_channels, out_channels, kernel_size=1)\n\n    def forward(self, x):\n        residual = self.shortcut(x)\n        out = self.act(self.bn1(self.conv1(x)))\n        out = self.dropout(self.bn2(self.conv2(out)))\n        out += residual\n        return self.act(out)\n\nclass SEBlock(nn.Module):\n    def __init__(self, channel, reduction=16):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(channel, channel // reduction, bias=False),\n            nn.ReLU(inplace=True),\n            nn.Linear(channel // reduction, channel, bias=False),\n            nn.Sigmoid()\n        )\n    def forward(self, x):\n        b, c = x.size()\n        y = x.view(b, c, 1)\n        y = self.avg_pool(y).view(b, c)\n        y = self.fc(y)\n        return x * y\n\nclass DenseResBlock(nn.Module):\n    def __init__(self, features, dropout_rate=0.2):\n        super().__init__()\n        self.block = nn.Sequential(\n            nn.Linear(features, features * 2), # Expand width\n            nn.BatchNorm1d(features * 2),\n            nn.GELU(),\n            nn.Dropout(dropout_rate),\n            nn.Linear(features * 2, features), # Project back\n            nn.BatchNorm1d(features),\n            nn.GELU(),\n            nn.Dropout(dropout_rate)\n        )\n        self.se = SEBlock(features)\n        \n    def forward(self, x):\n        return self.se(self.block(x)) + x # Residual connection\n\nclass HybridSystem(nn.Module):\n    def __init__(self, input_dim, num_classes):\n        super().__init__()\n        \n        # 1. Nhánh Residual CNN (Sâu hơn, giữ feature tốt hơn)\n        self.cnn_branch = nn.Sequential(\n            nn.Conv1d(1, 64, kernel_size=7, padding=3),\n            nn.BatchNorm1d(64),\n            nn.GELU(),\n            ResidualCNNBlock(64, 128, kernel_size=5),\n            nn.MaxPool1d(2),\n            ResidualCNNBlock(128, 256, kernel_size=3),\n            nn.MaxPool1d(2),\n            nn.Flatten(),\n            nn.Linear(256 * (input_dim // 4), 1024),\n            nn.BatchNorm1d(1024),\n            nn.GELU(),\n            nn.Dropout(0.3)\n        )\n        \n        # 2. Nhánh Wide ResNet (Mở rộng chiều ngang)\n        self.resnet_branch = nn.Sequential(\n            nn.Linear(input_dim, 1024),\n            nn.BatchNorm1d(1024),\n            nn.GELU(),\n            DenseResBlock(1024, dropout_rate=0.2),\n            DenseResBlock(1024, dropout_rate=0.2),\n            DenseResBlock(1024, dropout_rate=0.2), # Thêm 1 block nữa\n        )\n        \n        # 3. Fusion\n        self.fusion = nn.Sequential(\n            nn.Linear(1024 + 1024, 1536),\n            nn.BatchNorm1d(1536),\n            nn.GELU(),\n            nn.Dropout(0.3)\n        )\n        \n        # Multi-Sample Dropout\n        self.dropouts = nn.ModuleList([nn.Dropout(0.4) for _ in range(5)])\n        self.fc = nn.Linear(1536, num_classes)\n\n    def forward(self, x):\n        # CNN Branch\n        x_cnn = x.unsqueeze(1)\n        out_cnn = self.cnn_branch(x_cnn)\n        \n        # ResNet Branch\n        out_res = self.resnet_branch(x)\n        \n        # Fusion\n        combined = torch.cat([out_cnn, out_res], dim=1)\n        features = self.fusion(combined)\n        \n        # Output\n        output = torch.zeros(features.size(0), self.fc.out_features).to(features.device)\n        for dropout in self.dropouts:\n            output += self.fc(dropout(features))\n        return output / len(self.dropouts)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 5: Training with Asymmetric Loss\nfrom sklearn.model_selection import KFold\nfrom torch.optim.lr_scheduler import OneCycleLR\nimport torch.nn.utils as utils\n\n# --- ASYMMETRIC LOSS ---\nclass AsymmetricLossOptimized(nn.Module):\n    def __init__(self, gamma_neg=4, gamma_pos=1, clip=0.05, eps=1e-8, disable_torch_grad_focal_loss=False):\n        super(AsymmetricLossOptimized, self).__init__()\n        self.gamma_neg = gamma_neg\n        self.gamma_pos = gamma_pos\n        self.clip = clip\n        self.disable_torch_grad_focal_loss = disable_torch_grad_focal_loss\n        self.eps = eps\n\n    def forward(self, x, y):\n        # x: logits, y: targets (0 or 1)\n        \n        # Tính sigmoid probabilities\n        x_sigmoid = torch.sigmoid(x)\n        x_sigmoid_pos = x_sigmoid\n        x_sigmoid_neg = 1 - x_sigmoid\n\n        # Asymmetric Clipping \n        if self.clip is not None and self.clip > 0:\n            x_sigmoid_neg = (x_sigmoid_neg + self.clip).clamp(max=1)\n\n        # Basic Cross Entropy parts\n        loss_pos = y * torch.log(x_sigmoid_pos.clamp(min=self.eps))\n        loss_neg = (1 - y) * torch.log(x_sigmoid_neg.clamp(min=self.eps))\n\n        # Asymmetric Focusing\n        if self.gamma_neg > 0 or self.gamma_pos > 0:\n            if self.disable_torch_grad_focal_loss:\n                torch.set_grad_enabled(False)\n            \n            pt0 = x_sigmoid_pos * y\n            pt1 = x_sigmoid_neg * (1 - y)  # pt = p if t=1 else 1-p\n            pt = pt0 + pt1\n\n            one_sided_gamma = self.gamma_pos * y + self.gamma_neg * (1 - y)\n            one_sided_w = torch.pow(1 - pt, one_sided_gamma)\n\n            if self.disable_torch_grad_focal_loss:\n                torch.set_grad_enabled(True)\n            \n            loss_pos *= one_sided_w\n            loss_neg *= one_sided_w\n\n        return -torch.mean(loss_pos + loss_neg)\n\n# --- CONFIG TRAINING ---\nN_FOLDS = 5\nEPOCHS_PER_FOLD = 20\nLR = 0.001\nBATCH_SIZE = 128\nWEIGHT_DECAY = 1e-2 \n\nkfold = KFold(n_splits=N_FOLDS, shuffle=True, random_state=CONFIG['seed'])\nprint(f\"\\n--- START TRAINING (ASYMMETRIC LOSS + RESIDUAL MODEL) ---\")\n\nfold_metrics = []\n\nfor fold, (train_idx, val_idx) in enumerate(kfold.split(train_embeds, labels_matrix)):\n    print(f\"\\n>>> FOLD {fold+1}/{N_FOLDS}\")\n    \n    # Load Data\n    X_train, X_val = train_embeds[train_idx], train_embeds[val_idx]\n    y_train, y_val = labels_matrix[train_idx], labels_matrix[val_idx]\n    \n    train_dataset = ProteinDataset(X_train, y_train)\n    val_dataset = ProteinDataset(X_val, y_val)\n    \n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, pin_memory=True, num_workers=2)\n    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, pin_memory=True, num_workers=2)\n    \n    # Init Model\n    model = HybridSystem(input_dim=EMBED_DIM, num_classes=CONFIG['num_labels']).to(DEVICE)\n    \n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n    \n    criterion = AsymmetricLossOptimized(gamma_neg=4, gamma_pos=1, clip=0.05)\n    \n    scheduler = OneCycleLR(optimizer, max_lr=LR, steps_per_epoch=len(train_loader), \n                           epochs=EPOCHS_PER_FOLD, pct_start=0.3)\n    \n    best_loss = float('inf')\n    \n    for epoch in range(EPOCHS_PER_FOLD):\n        model.train()\n        train_loss = 0\n        for embeds, targets in tqdm(train_loader, desc=f\"Ep {epoch+1}\", leave=False):\n            embeds, targets = embeds.to(DEVICE), targets.to(DEVICE)\n            \n            optimizer.zero_grad()\n            outputs = model(embeds)\n            loss = criterion(outputs, targets)\n            loss.backward()\n            \n            # Gradient Clipping (Vẫn giữ để an toàn)\n            utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            \n            optimizer.step()\n            scheduler.step()\n            train_loss += loss.item()\n        \n        model.eval()\n        val_loss = 0\n        with torch.no_grad():\n            for embeds, targets in val_loader:\n                embeds, targets = embeds.to(DEVICE), targets.to(DEVICE)\n                outputs = model(embeds)\n                loss = criterion(outputs, targets)\n                val_loss += loss.item()\n        \n        avg_val_loss = val_loss / len(val_loader)\n        \n        if avg_val_loss < best_loss:\n            best_loss = avg_val_loss\n            torch.save(model.state_dict(), f\"model_fold_{fold}.pth\")\n            \n    print(f\"Fold {fold+1} Best ASL Loss: {best_loss:.5f}\")\n    fold_metrics.append(best_loss)\n\nprint(f\"\\n--- DONE. Avg Loss: {np.mean(fold_metrics):.5f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cell 6: Ensemble Prediction (Hybrid System)\nimport gc\nimport torch\nimport numpy as np\nimport pandas as pd\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\nprint(\"\\n--- Generating Predictions (Hybrid System) ---\")\n\n# 1. Khôi phục mapping\nprint(\"Re-creating index mapping...\")\ndf_terms = pd.read_csv(CONFIG['paths']['train_terms'], sep=\"\\t\")\ntop_terms = df_terms['term'].value_counts().index[:CONFIG['num_labels']]\nidx_to_term = {i: term for i, term in enumerate(top_terms)}\ndel df_terms, top_terms\ngc.collect()\n\n# 2. Setup Data Loader\nBATCH_SIZE = 256\ntest_dataset_final = ProteinDataset(test_embeds, ids=test_ids)\ntest_loader_final = DataLoader(test_dataset_final, batch_size=BATCH_SIZE, shuffle=False)\n\n# 3. Load Models\nmodels = []\nfor fold in range(N_FOLDS):\n    # Dùng đúng class HybridSystem\n    model = HybridSystem(input_dim=EMBED_DIM, num_classes=CONFIG['num_labels']).to(DEVICE)\n    model.load_state_dict(torch.load(f\"model_fold_{fold}.pth\"))\n    model.eval()\n    models.append(model)\nprint(f\"Loaded {len(models)} Hybrid models.\")\n\n# 4. Prediction\nTHRESHOLD = 0.02\nTOP_K = 75       \noutput_file = \"submission.tsv\"\n\nprint(f\"Inference... Threshold={THRESHOLD}, Top-K={TOP_K}\")\n\nwith open(output_file, 'w') as f:\n    with torch.no_grad():\n        for step, (embeds, batch_ids) in enumerate(tqdm(test_loader_final)):\n            embeds = embeds.to(DEVICE)\n            \n            avg_probs = None\n            for model in models:\n                logits = model(embeds)\n                probs = torch.sigmoid(logits)\n                if avg_probs is None: avg_probs = probs\n                else: avg_probs += probs\n            \n            avg_probs /= len(models)\n            avg_probs = avg_probs.cpu().numpy()\n            \n            batch_lines = []\n            for i, pid in enumerate(batch_ids):\n                row_probs = avg_probs[i]\n                idx_candidates = np.where(row_probs > THRESHOLD)[0]\n                \n                if len(idx_candidates) > TOP_K:\n                    candidate_probs = row_probs[idx_candidates]\n                    final_indices = idx_candidates[np.argsort(candidate_probs)[-TOP_K:]]\n                else:\n                    final_indices = idx_candidates\n                \n                for idx in final_indices:\n                    batch_lines.append(f\"{pid}\\t{idx_to_term[idx]}\\t{row_probs[idx]:.3f}\\n\")\n            \n            f.writelines(batch_lines)\n            del batch_lines, avg_probs, embeds\n            if step % 50 == 0: gc.collect()\n\nprint(f\"✅ Submission saved to {output_file} successfully!\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}