{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":116062,"databundleVersionId":14084779,"sourceType":"competition"},{"sourceId":5499219,"sourceType":"datasetVersion","datasetId":3167603},{"sourceId":5549164,"sourceType":"datasetVersion","datasetId":3197305},{"sourceId":5792099,"sourceType":"datasetVersion","datasetId":3327296},{"sourceId":6247561,"sourceType":"datasetVersion","datasetId":3590060}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Library Import","metadata":{}},{"cell_type":"code","source":"#library import\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport math","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:20.313664Z","iopub.execute_input":"2025-11-19T11:42:20.313951Z","iopub.status.idle":"2025-11-19T11:42:20.317935Z","shell.execute_reply.started":"2025-11-19T11:42:20.313931Z","shell.execute_reply":"2025-11-19T11:42:20.317210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#part 2\nimport time\nimport matplotlib.pyplot as plt\nimport os\nimport gc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:20.510654Z","iopub.execute_input":"2025-11-19T11:42:20.510907Z","iopub.status.idle":"2025-11-19T11:42:20.514553Z","shell.execute_reply.started":"2025-11-19T11:42:20.510888Z","shell.execute_reply":"2025-11-19T11:42:20.513955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#part 3a basic\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader, random_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:20.655381Z","iopub.execute_input":"2025-11-19T11:42:20.655643Z","iopub.status.idle":"2025-11-19T11:42:20.659260Z","shell.execute_reply.started":"2025-11-19T11:42:20.655626Z","shell.execute_reply":"2025-11-19T11:42:20.658723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#part 3b\nfrom torch.optim.lr_scheduler import ExponentialLR, CosineAnnealingLR, ReduceLROnPlateau\nfrom torchmetrics.classification import MultilabelF1Score\nimport torch.optim as optim","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:21.435554Z","iopub.execute_input":"2025-11-19T11:42:21.436261Z","iopub.status.idle":"2025-11-19T11:42:21.439828Z","shell.execute_reply.started":"2025-11-19T11:42:21.436238Z","shell.execute_reply":"2025-11-19T11:42:21.439164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#part 4 adjustment\nfrom torch.optim.lr_scheduler import _LRScheduler\nfrom torch.utils.data import Dataset #dataset\nimport torch.nn.functional as F","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:21.621422Z","iopub.execute_input":"2025-11-19T11:42:21.621699Z","iopub.status.idle":"2025-11-19T11:42:21.625689Z","shell.execute_reply.started":"2025-11-19T11:42:21.621680Z","shell.execute_reply":"2025-11-19T11:42:21.625033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#extra modification\nfrom torch.optim.lr_scheduler import LambdaLR","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:21.774181Z","iopub.execute_input":"2025-11-19T11:42:21.774437Z","iopub.status.idle":"2025-11-19T11:42:21.778355Z","shell.execute_reply.started":"2025-11-19T11:42:21.774420Z","shell.execute_reply":"2025-11-19T11:42:21.777645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Set random seeds for reproducibility\nnp.random.seed(36)\ntorch.manual_seed(36)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(36)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:22.278336Z","iopub.execute_input":"2025-11-19T11:42:22.279022Z","iopub.status.idle":"2025-11-19T11:42:22.283999Z","shell.execute_reply.started":"2025-11-19T11:42:22.278997Z","shell.execute_reply":"2025-11-19T11:42:22.283308Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Datasate maker and  Loading","metadata":{}},{"cell_type":"markdown","source":"## Configuration","metadata":{}},{"cell_type":"code","source":"#the first gc\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:24.484420Z","iopub.execute_input":"2025-11-19T11:42:24.485010Z","iopub.status.idle":"2025-11-19T11:42:24.682330Z","shell.execute_reply.started":"2025-11-19T11:42:24.484988Z","shell.execute_reply":"2025-11-19T11:42:24.681538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#main path from Kaggle\nmain_path = \"/kaggle/input\"\n#path concern in main data (competition)\nmain_df_path = f\"{main_path}/cafa-6-protein-function-prediction\" #main path for protein","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:24.752938Z","iopub.execute_input":"2025-11-19T11:42:24.753169Z","iopub.status.idle":"2025-11-19T11:42:24.756957Z","shell.execute_reply.started":"2025-11-19T11:42:24.753154Z","shell.execute_reply":"2025-11-19T11:42:24.756201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#data declaration in configuration\nclass Config:\n    \"\"\"Configuration settings for the model in main dataframe\"\"\"\n    def __init__(self):\n        self.main_dir = main_df_path\n        self.train_sequences_path = f\"{self.main_dir}/Train/train_sequences.fasta\"\n        self.train_labels_path = f\"{self.main_dir}/Train/train_terms.tsv\"\n        self.test_sequences_path = f\"{self.main_dir}/Test/testsuperset.fasta\"\n        self.ia_path = f\"{self.main_dir}/IA.tsv\"\n        \n        # Model parameters\n        self.num_labels = 500\n        self.n_epochs = 171\n        self.batch_size = 120\n        self.dropout_rate = 0.3\n        self.patience = 12\n        \n        # Base learning rate\n        self.lr = 4e-5  \n        self.min_lr = 6.8e-8\n        \n        # Dynamic learning rate configuration\n        self.use_dynamic_lr = True\n        self.lr_scheduler_type = \"exponential\"  # selections are ['exponential', 'cosine', 'plateau']\n        self.lr_params = {\n            \"decay_rate\": 0.993,\n            \"decay_steps\": 2,\n            \"min_lr\": self.min_lr,\n            \"T_max\": 10,  # cosine only\n            \"factor\": 0.5,  # plateau only\n        }\n\n        # Device\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n        # Embedding configurations\n        self.embeds_map = {\n            \"ESM2\": \"cafa-5-ems-2-embeddings-numpy\",\n            \"ProtBERT\": \"protbert-embeddings-for-cafa5\",\n            \"T5\": \"t5embeds\"\n        }\n\n        self.embeds_dim = {\n            \"ESM2\": 1280,\n            \"ProtBERT\": 1024,\n            \"T5\": 1024\n        }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:41.829001Z","iopub.execute_input":"2025-11-19T11:42:41.829274Z","iopub.status.idle":"2025-11-19T11:42:41.835491Z","shell.execute_reply.started":"2025-11-19T11:42:41.829255Z","shell.execute_reply":"2025-11-19T11:42:41.834654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class declaration\nconfig = Config()\nprint(f\"Using device: {config.device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:42.908699Z","iopub.execute_input":"2025-11-19T11:42:42.909380Z","iopub.status.idle":"2025-11-19T11:42:42.913259Z","shell.execute_reply.started":"2025-11-19T11:42:42.909359Z","shell.execute_reply":"2025-11-19T11:42:42.912581Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dataset loader","metadata":{}},{"cell_type":"code","source":"#dataset builder\nclass ProteinDataset(Dataset):\n    \"\"\"Dataset class for protein sequences and their GO annotations\"\"\"\n    \n    def __init__(self, datatype, embeddings_source, config):\n        super().__init__()\n        self.datatype = datatype\n        self.config = config\n        self.embeddings_source = embeddings_source\n        \n        # Load embeddings\n        self._load_embeddings()\n        \n        # Load labels for training data\n        if self.datatype == \"train\":\n            self._load_labels()\n        #loading embeded data\n    def _load_embeddings(self):\n        \"\"\"Load pre-computed embeddings\"\"\"\n        embed_dir = f\"{main_path}/{self.config.embeds_map[self.embeddings_source]}\"\n        \n        if self.embeddings_source == \"ESM2\":\n            embeds = np.load(f\"{embed_dir}/{self.datatype}_embeddings.npy\")\n            ids = np.load(f\"{embed_dir}/{self.datatype}_ids.npy\")\n        elif self.embeddings_source == \"ProtBERT\":\n            embeds = np.load(f\"{embed_dir}/{self.datatype}_embeddings.npy\")\n            ids = np.load(f\"{embed_dir}/{self.datatype}_ids.npy\")\n        elif self.embeddings_source == \"T5\":\n            embeds = np.load(f\"{embed_dir}/{self.datatype}_embeds.npy\")\n            ids = np.load(f\"{embed_dir}/{self.datatype}_ids.npy\")\n        \n        # Create DataFrame\n        self.df = pd.DataFrame({\n            \"EntryID\": ids,\n            \"embed\": [embeds[i] for i in range(embeds.shape[0])]\n        })\n    #loading labels\n    def _load_labels(self):\n        \"\"\"Load GO term labels for training data\"\"\"\n        # Load pre-processed top labels if available\n        label_file = f\"/kaggle/input/train-targets-top{self.config.num_labels}/train_targets_top{self.config.num_labels}.npy\"\n        \n        if os.path.exists(label_file):\n            np_labels = np.load(label_file)\n            df_labels = pd.DataFrame({\n                'EntryID': self.df['EntryID'],\n                'labels_vect': [row for row in np_labels]\n            })\n            self.df = self.df.merge(df_labels, on=\"EntryID\", how=\"inner\")\n        else:\n            # Process labels from scratch\n            self._process_labels_from_tsv()\n    def _process_labels_from_tsv(self):\n        \"\"\"Process labels from the TSV file\"\"\"\n        labels_df = pd.read_csv(self.config.train_labels_path, sep=\"\\t\", names=[\"EntryID\", \"term\", \"aspect\"])\n        \n        # Get top terms\n        top_terms = labels_df.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\n        self.top_terms = top_terms[:self.config.num_labels].index.tolist()\n        \n        # Create label vectors\n        label_vectors = []\n        for entry_id in self.df['EntryID']:\n            entry_terms = labels_df[labels_df['EntryID'] == entry_id]['term'].tolist()\n            vector = [1 if term in entry_terms else 0 for term in self.top_terms]\n            label_vectors.append(vector)\n        \n        self.df['labels_vect'] = label_vectors\n    \n    def __len__(self):\n        return len(self.df)\n    #getting item\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        embed = torch.tensor(row[\"embed\"], dtype=torch.float32)\n        \n        if self.datatype == \"train\":\n            labels = torch.tensor(row[\"labels_vect\"], dtype=torch.float32)\n            return embed, labels\n        else:\n            return embed, row[\"EntryID\"]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:44.517210Z","iopub.execute_input":"2025-11-19T11:42:44.517495Z","iopub.status.idle":"2025-11-19T11:42:44.528648Z","shell.execute_reply.started":"2025-11-19T11:42:44.517475Z","shell.execute_reply":"2025-11-19T11:42:44.527904Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Building and scheduler","metadata":{}},{"cell_type":"markdown","source":"## The LoRA schedule","metadata":{}},{"cell_type":"code","source":"# 1. Building Low-Rank Adapter Layer (LoRA)\nclass LoRALinear(nn.Module):\n    def __init__(self, in_features, out_features, r= 8, alpha= 16, dropout= 0.025, bias=True):\n        super().__init__()\n        self.r = r\n        self.alpha = alpha if alpha is not None else r\n        self.scaling = self.alpha / max(1, self.r)\n        self.linear = nn.Linear(in_features, out_features, bias=bias)\n        self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()\n\n        if r > 0:\n            # low-rank matrices\n            self.lora_A = nn.Parameter(torch.randn(r, in_features) * 0.01)\n            self.lora_B = nn.Parameter(torch.zeros(out_features, r))\n        else:\n            self.register_parameter('lora_A', None)\n            self.register_parameter('lora_B', None)\n\n    def forward(self, x):\n        base = self.linear(x)\n        if self.r > 0:\n            # compute adapter output (initially 0 due to B = 0)\n            x_d = self.dropout(x)\n            a_out = F.linear(x_d, self.lora_A)       # (batch, r)\n            b_out = F.linear(a_out, self.lora_B)     # (batch, out)\n            return base + b_out * self.scaling\n        else:\n            return base","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:45.950049Z","iopub.execute_input":"2025-11-19T11:42:45.950905Z","iopub.status.idle":"2025-11-19T11:42:45.956664Z","shell.execute_reply.started":"2025-11-19T11:42:45.950880Z","shell.execute_reply":"2025-11-19T11:42:45.956022Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The purpose of implementing **LoRA** is to maintain minimal burden","metadata":{}},{"cell_type":"code","source":"# 2. LoRA-Enhanced Hybrid Model\nclass HybridModel(nn.Module):\n    \"\"\"Hybrid model combining CNN and LSTM features with LoRA for safe fine-tuning\"\"\"\n\n    def __init__(self, input_dim = config.embeds_dim[\"T5\"], num_classes = config.num_labels, dropout_rate= 0.3, lora_r= 8, lora_alpha= 16, lora_dropout=0.025):\n        super().__init__()\n\n        # ----- CNN branch (unchanged) -----\n        self.conv1 = nn.Conv1d(1, 32, kernel_size=5, padding=2)\n        self.conv2 = nn.Conv1d(32, 64, kernel_size=3, padding=1)\n        self.pool = nn.AdaptiveMaxPool1d(128)\n\n        # ----- LSTM branch (unchanged) -----\n        self.lstm = nn.LSTM(input_dim, 256, num_layers=2,\n                            bidirectional=True, dropout=dropout_rate, batch_first=True)\n\n        # Optional small projection for LSTM output to apply LoRA safely\n        self.lstm_proj = LoRALinear(512, 512, r=lora_r, alpha=lora_alpha, dropout=lora_dropout)\n\n        # ----- Fusion and Classification (LoRA applied here) -----\n        self.fc1 = LoRALinear(64 * 128 + 512, 512, r=lora_r, alpha=lora_alpha, dropout=lora_dropout)\n        self.dropout = nn.Dropout(dropout_rate)\n        self.fc2 = LoRALinear(512, 256, r=lora_r, alpha=lora_alpha, dropout=lora_dropout)\n        self.fc3 = LoRALinear(256, num_classes, r=lora_r, alpha=lora_alpha, dropout=lora_dropout)\n\n    def forward(self, x):\n        batch_size = x.shape[0]\n\n        # ----- CNN branch -----\n        cnn_x = x.unsqueeze(1)           # (B, 1, seq_len)\n        cnn_x = F.relu(self.conv1(cnn_x))\n        cnn_x = F.relu(self.conv2(cnn_x))\n        cnn_x = self.pool(cnn_x)\n        cnn_x = cnn_x.view(batch_size, -1)\n\n        # ----- LSTM branch -----\n        lstm_x = x.unsqueeze(1)          # maintain consistency with original\n        lstm_out, _ = self.lstm(lstm_x)\n        lstm_x = lstm_out[:, -1, :]      # take final hidden state\n        lstm_x = self.lstm_proj(lstm_x)  # safe LoRA projection\n\n        # ----- Concatenate and classify -----\n        combined = torch.cat([cnn_x, lstm_x], dim=1)\n\n        x = F.relu(self.fc1(combined))\n        x = self.dropout(x)\n        x = F.relu(self.fc2(x))\n        x = self.dropout(x)\n        x = self.fc3(x)\n\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:46.976092Z","iopub.execute_input":"2025-11-19T11:42:46.976705Z","iopub.status.idle":"2025-11-19T11:42:46.984462Z","shell.execute_reply.started":"2025-11-19T11:42:46.976683Z","shell.execute_reply":"2025-11-19T11:42:46.983756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# new loss criterion : dynamic loss\nclass DynamicSmoothBCEWithLogitsLoss(nn.Module):\n#initial function\n    def __init__(\n        self,\n        smoothing_start=2e-8,\n        smoothing_min=1e-10,\n        smoothing_max=3e-4,\n        increase_factor=1.25,\n        decrease_factor=0.98,\n        spike_threshold=0.015\n    ):\n        super().__init__()\n        self.smoothing = smoothing_start\n        self.smoothing_min = smoothing_min\n        self.smoothing_max = smoothing_max\n        self.increase_factor = increase_factor\n        self.decrease_factor = decrease_factor\n        self.spike_threshold = spike_threshold\n\n        self.loss = nn.BCEWithLogitsLoss()\n        self.prev_loss = None\n\n    def update_smoothing(self, current_loss):\n        if self.prev_loss is None:\n            self.prev_loss = current_loss\n            return\n\n        delta = abs(current_loss - self.prev_loss)\n\n        # If loss fluctuates → increase smoothing\n        if delta > self.spike_threshold:\n            self.smoothing = min(self.smoothing * self.increase_factor, self.smoothing_max)\n        else:\n            # Stable → reduce smoothing slightly\n            self.smoothing = max(self.smoothing * self.decrease_factor, self.smoothing_min)\n\n        self.prev_loss = current_loss\n\n    def forward(self, pred, target):\n        smoothed_target = target * (1 - self.smoothing) + 0.5 * self.smoothing\n        result = self.loss(pred, smoothed_target)\n        self.update_smoothing(result.item())\n        return result\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:48.268110Z","iopub.execute_input":"2025-11-19T11:42:48.268825Z","iopub.status.idle":"2025-11-19T11:42:48.275350Z","shell.execute_reply.started":"2025-11-19T11:42:48.268802Z","shell.execute_reply":"2025-11-19T11:42:48.274528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Gaining LoRA parameters \ndef get_lora_params(model): \n    return [p for p in model.parameters() if p.requires_grad]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:50.254257Z","iopub.execute_input":"2025-11-19T11:42:50.254871Z","iopub.status.idle":"2025-11-19T11:42:50.258391Z","shell.execute_reply.started":"2025-11-19T11:42:50.254849Z","shell.execute_reply":"2025-11-19T11:42:50.257739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#optional modificaiton in LoRA training\ndef mark_only_lora_trainable(model):\n    for name, param in model.named_parameters():\n        param.requires_grad = False\n    for module in model.modules():\n        if isinstance(module, LoRALinear):\n            if module.lora_A is not None:\n                module.lora_A.requires_grad = True\n            if module.lora_B is not None:\n                module.lora_B.requires_grad = True","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:50.435632Z","iopub.execute_input":"2025-11-19T11:42:50.436282Z","iopub.status.idle":"2025-11-19T11:42:50.440422Z","shell.execute_reply.started":"2025-11-19T11:42:50.436259Z","shell.execute_reply":"2025-11-19T11:42:50.439850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Gaining LoRA parameters \ndef get_lora_params(model): \n    return [p for p in model.parameters() if p.requires_grad]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:50.766892Z","iopub.execute_input":"2025-11-19T11:42:50.767556Z","iopub.status.idle":"2025-11-19T11:42:50.771008Z","shell.execute_reply.started":"2025-11-19T11:42:50.767533Z","shell.execute_reply":"2025-11-19T11:42:50.770350Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Extra adjutments in schedule","metadata":{}},{"cell_type":"code","source":"# Making scheduler before training\ndef get_scheduler(optimizer, config):\n    def lr_lambda(current_step):\n        warmup_steps = 10\n        total_steps = config.n_epochs\n        if current_step < warmup_steps:\n            return float(current_step) / float(max(1, warmup_steps))\n        progress = float(current_step - warmup_steps) / float(max(1, total_steps - warmup_steps))\n        return max(0.1, 0.5 * (1.0 + np.cos(np.pi * progress)))\n    return LambdaLR(optimizer, lr_lambda)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:52.197780Z","iopub.execute_input":"2025-11-19T11:42:52.198072Z","iopub.status.idle":"2025-11-19T11:42:52.203035Z","shell.execute_reply.started":"2025-11-19T11:42:52.198053Z","shell.execute_reply":"2025-11-19T11:42:52.202114Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# The **'Running phase'** in training and validation","metadata":{}},{"cell_type":"code","source":"#gc case\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:42:53.543271Z","iopub.execute_input":"2025-11-19T11:42:53.544007Z","iopub.status.idle":"2025-11-19T11:42:53.743234Z","shell.execute_reply.started":"2025-11-19T11:42:53.543982Z","shell.execute_reply":"2025-11-19T11:42:53.742647Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## running phase","metadata":{}},{"cell_type":"code","source":"#only warning : sudden stop\nclass EarlyStopping:\n   #initial sudden stop\n    def __init__(self, patience= 15, min_delta= 2e-5):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.best = None\n        self.should_stop = False\n\n    def step(self, value):\n        if self.best is None:\n            self.best = value\n            return False\n\n        # improvement?\n        if value - self.best > self.min_delta:\n            self.best = value\n            self.counter = 0\n        else:\n            self.counter += 1\n\n        if self.counter >= self.patience:\n            self.should_stop = True\n\n        return self.should_stop\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:43:05.137856Z","iopub.execute_input":"2025-11-19T11:43:05.138145Z","iopub.status.idle":"2025-11-19T11:43:05.143331Z","shell.execute_reply.started":"2025-11-19T11:43:05.138125Z","shell.execute_reply":"2025-11-19T11:43:05.142663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#training phase\ndef train_epoch(model, dataloader, criterion, optimizer, device):\n    \"\"\"Train for one epoch\"\"\"\n    model.train()\n    losses = []\n    \n    for embeddings, labels in tqdm(dataloader, desc=\"Training\"):\n        embeddings, labels = embeddings.to(device), labels.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(embeddings)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        losses.append(loss.item())\n    \n    return np.mean(losses)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:43:05.995523Z","iopub.execute_input":"2025-11-19T11:43:05.996257Z","iopub.status.idle":"2025-11-19T11:43:06.000885Z","shell.execute_reply.started":"2025-11-19T11:43:05.996236Z","shell.execute_reply":"2025-11-19T11:43:06.000227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#validation\ndef validate_epoch(model, dataloader, criterion, metric, device):\n    \"\"\"Validate the result for one epoch\"\"\"\n    model.eval()\n    losses = []\n    scores = []\n    \n    with torch.no_grad():\n        for embeddings, labels in dataloader:\n            embeddings, labels = embeddings.to(device), labels.to(device)\n            \n            outputs = model(embeddings)\n            loss = criterion(outputs, labels)\n            score = metric(torch.sigmoid(outputs), labels.int())\n            \n            losses.append(loss.item())\n            scores.append(score.item())\n    \n    return np.mean(losses), np.mean(scores)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:43:06.288755Z","iopub.execute_input":"2025-11-19T11:43:06.289012Z","iopub.status.idle":"2025-11-19T11:43:06.294466Z","shell.execute_reply.started":"2025-11-19T11:43:06.288993Z","shell.execute_reply":"2025-11-19T11:43:06.293755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#running the training function with stabilizer\ndef train_model(embeddings_source=\"T5\", model_type=\"hybrid\", train_ratio= 0.8):\n    print(f\"\\nTraining {model_type} model with {embeddings_source} embeddings...\")\n\n    # Dataset split\n    dataset = ProteinDataset(\"train\", embeddings_source, config)\n    train_size = int(len(dataset) * train_ratio)\n    val_size = len(dataset) - train_size\n    train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n    train_loader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=2)\n    val_loader = DataLoader(val_dataset, batch_size=config.batch_size, shuffle=False, num_workers=2)\n\n    # Model\n    input_dim = config.embeds_dim[embeddings_source]\n    model = HybridModel(input_dim, config.num_labels, dropout_rate= 0.035, lora_r= 8, lora_alpha= 8, lora_dropout= 0.125).to(config.device)\n\n    # Freeze all except LoRA\n    mark_only_lora_trainable(model)\n    lora_params = get_lora_params(model)\n\n    # Optimizer + Scheduler\n    criterion = DynamicSmoothBCEWithLogitsLoss(smoothing_start=2e-7, smoothing_min=1e-10, smoothing_max=3e-2)\n    optimizer = torch.optim.AdamW(lora_params, lr=config.lr * 1.5, weight_decay= 4.4e-8)\n    scheduler = get_scheduler(optimizer, config)\n    metric = MultilabelF1Score(num_labels=config.num_labels, average='micro').to(config.device)\n\n    best_val_score = 0\n    train_losses, val_losses, val_scores, lrs = [], [], [], []\n    # sole callback : sudden stop\n    early_stopper = EarlyStopping(patience=config.patience, min_delta= 2e-5)\n    # ---------- Training Loop ----------\n    for epoch in range(config.n_epochs):\n        model.train()\n        epoch_losses = []\n        grad_means = []\n\n        for embeddings, labels in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{config.n_epochs}\"):\n            embeddings, labels = embeddings.to(config.device), labels.to(config.device)\n            optimizer.zero_grad()\n            outputs = model(embeddings)\n            loss = criterion(outputs, labels)\n            loss.backward()\n\n            # Gradient clipping for stability\n            torch.nn.utils.clip_grad_norm_(lora_params, max_norm=1.0)\n\n            optimizer.step()\n            epoch_losses.append(loss.item())\n\n            # Track mean gradient magnitude (diagnostic)\n            with torch.no_grad():\n                grads = [p.grad.abs().mean().item() for p in lora_params if p.grad is not None]\n                if grads:\n                    grad_means.append(np.mean(grads))\n\n        train_loss = np.mean(epoch_losses)\n        mean_grad = np.mean(grad_means) if grad_means else 0.0\n\n        # ---------- Validation ----------\n        val_loss, val_score = validate_epoch(model, val_loader, criterion, metric, config.device)\n\n        # ---------- Scheduler step ----------\n        scheduler.step()\n        current_lr = optimizer.param_groups[0][\"lr\"]\n        lrs.append(current_lr)\n\n        # ---------- Logging ----------\n        train_losses.append(train_loss)\n        val_losses.append(val_loss)\n        val_scores.append(val_score)\n\n        print(f\"Epoch {epoch+1}/{config.n_epochs} | \"\n              f\"Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | \"\n              f\"Val F1: {val_score:.4f} | LR: {current_lr:.6e} | Mean Grad: {mean_grad:.6f}\")\n\n        # Save best model\n        if val_score > best_val_score:\n            best_val_score = val_score\n            best_model_state = model.state_dict().copy()\n        # ---- EARLY STOPPING ----\n        if early_stopper.step(val_score):\n            print(f\"\\n⛔ Early stopping triggered at epoch {epoch+1}\")\n            break\n\n    # ---------- Return ----------\n    return model, {\n        'train_losses': train_losses,\n        'val_losses': val_losses,\n        'val_scores': val_scores,\n        'lrs': lrs,\n        'best_score': best_val_score\n    }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:43:16.025119Z","iopub.execute_input":"2025-11-19T11:43:16.025650Z","iopub.status.idle":"2025-11-19T11:43:16.036799Z","shell.execute_reply.started":"2025-11-19T11:43:16.025593Z","shell.execute_reply":"2025-11-19T11:43:16.035949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#another gc\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:43:18.243895Z","iopub.execute_input":"2025-11-19T11:43:18.244164Z","iopub.status.idle":"2025-11-19T11:43:18.442229Z","shell.execute_reply.started":"2025-11-19T11:43:18.244145Z","shell.execute_reply":"2025-11-19T11:43:18.441651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# actual running\nmodel, history = train_model(\n    embeddings_source=\"T5\",\n    model_type=\"hybrid\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:43:19.764100Z","iopub.execute_input":"2025-11-19T11:43:19.764685Z","iopub.status.idle":"2025-11-19T11:46:59.502750Z","shell.execute_reply.started":"2025-11-19T11:43:19.764663Z","shell.execute_reply":"2025-11-19T11:46:59.501876Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## training evaluation","metadata":{}},{"cell_type":"code","source":"#plot in training\ndef plot_training_curves(history):\n    epochs = range(1, len(history['train_losses']) + 1)\n\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(epochs, history['train_losses'], label='Train Loss', color='black')\n    plt.plot(epochs, history['val_losses'], label='Validation Loss', color='red')\n    plt.title('Training & Validation Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend()\n\n    plt.subplot(1, 2, 2)\n    plt.plot(epochs, history['val_scores'], label='Validation F1', color='green')\n    plt.title('Validation F1 Score')\n    plt.xlabel('Epoch')\n    plt.ylabel('F1 Score')\n    plt.legend()\n\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:46:59.505005Z","iopub.execute_input":"2025-11-19T11:46:59.505264Z","iopub.status.idle":"2025-11-19T11:46:59.511756Z","shell.execute_reply.started":"2025-11-19T11:46:59.505243Z","shell.execute_reply":"2025-11-19T11:46:59.510990Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#plot in learning curve\ndef plot_learning_rate(history):\n    plt.figure(figsize=(6,4))\n    plt.plot(history['lrs'], marker='o', color='purple')\n    plt.title('Learning Rate over Epochs')\n    plt.xlabel('Epoch')\n    plt.ylabel('LR')\n    plt.yscale('log')\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:46:59.512404Z","iopub.execute_input":"2025-11-19T11:46:59.512650Z","iopub.status.idle":"2025-11-19T11:46:59.533848Z","shell.execute_reply.started":"2025-11-19T11:46:59.512631Z","shell.execute_reply":"2025-11-19T11:46:59.533118Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#tunning training plot\nplot_training_curves(history)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:46:59.535250Z","iopub.execute_input":"2025-11-19T11:46:59.535504Z","iopub.status.idle":"2025-11-19T11:46:59.911365Z","shell.execute_reply.started":"2025-11-19T11:46:59.535488Z","shell.execute_reply":"2025-11-19T11:46:59.910660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#running learning curve plot\nplot_learning_rate(history)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:46:59.912201Z","iopub.execute_input":"2025-11-19T11:46:59.912506Z","iopub.status.idle":"2025-11-19T11:47:00.161266Z","shell.execute_reply.started":"2025-11-19T11:46:59.912488Z","shell.execute_reply":"2025-11-19T11:47:00.160615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#final result display\nprint(f\"Best Validation F1 Score: {history['best_score']:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:47:00.162209Z","iopub.execute_input":"2025-11-19T11:47:00.162396Z","iopub.status.idle":"2025-11-19T11:47:00.166490Z","shell.execute_reply.started":"2025-11-19T11:47:00.162382Z","shell.execute_reply":"2025-11-19T11:47:00.165654Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## prediction inside the validation","metadata":{}},{"cell_type":"code","source":"#function to predict actual results\ndef predict(model, embeddings_source=\"T5\"):\n    \"\"\"Generate predictions for test set\"\"\"\n    \n    print(\"\\nGenerating predictions...\")\n    \n    # Load test dataset\n    test_dataset = ProteinDataset(\"test\", embeddings_source, config)\n    test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False)\n    \n    # Get label names\n    labels_df = pd.read_csv(config.train_labels_path, sep=\"\\t\", names=[\"EntryID\", \"term\", \"aspect\"])\n    top_terms = labels_df.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\n    label_names = top_terms[:config.num_labels].index.tolist()\n    \n    # Generate predictions\n    model.eval()\n    predictions = []\n    \n    with torch.no_grad():\n        for embeddings, protein_id in tqdm(test_loader, desc=\"Predicting\"):\n            embeddings = embeddings.to(config.device)\n            outputs = torch.sigmoid(model(embeddings)).cpu().numpy().squeeze()\n            \n            for i, conf in enumerate(outputs):\n                if conf > 0.05:  # Only include predictions above threshold\n                    predictions.append({\n                        'Id': protein_id[0],\n                        'GO term': label_names[i],\n                        'Confidence': min(conf, 0.95)  # Scientific standard\n                    })\n    \n    return pd.DataFrame(predictions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:47:00.167352Z","iopub.execute_input":"2025-11-19T11:47:00.167588Z","iopub.status.idle":"2025-11-19T11:47:00.181979Z","shell.execute_reply.started":"2025-11-19T11:47:00.167567Z","shell.execute_reply":"2025-11-19T11:47:00.181298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#model prediciton\npreds = predict(model, embeddings_source=\"T5\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:47:00.182695Z","iopub.execute_input":"2025-11-19T11:47:00.182896Z","iopub.status.idle":"2025-11-19T11:53:38.699909Z","shell.execute_reply.started":"2025-11-19T11:47:00.182876Z","shell.execute_reply":"2025-11-19T11:53:38.698567Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Submission","metadata":{}},{"cell_type":"code","source":"#declaring main prediction\nexisting_pred_path = f\"{main_path}/blast-quick-sprof-zero-pred/submission.tsv\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:53:38.702757Z","iopub.execute_input":"2025-11-19T11:53:38.703064Z","iopub.status.idle":"2025-11-19T11:53:38.709999Z","shell.execute_reply.started":"2025-11-19T11:53:38.703049Z","shell.execute_reply":"2025-11-19T11:53:38.709268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#preparing submission\nexisting = pd.read_csv(existing_pred_path, sep='\\t', header=None, names=['Id', 'GO term', 'Confidence'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:53:38.712239Z","iopub.execute_input":"2025-11-19T11:53:38.712857Z","iopub.status.idle":"2025-11-19T11:53:45.031426Z","shell.execute_reply.started":"2025-11-19T11:53:38.712841Z","shell.execute_reply":"2025-11-19T11:53:45.030573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#merging the data for actual submission\nmerged = pd.merge(existing, preds, on=['Id', 'GO term'], how='outer', suffixes=('_existing', '_new'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T11:53:45.032253Z","iopub.execute_input":"2025-11-19T11:53:45.032460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#emergency gc\ngc.collect()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Combine confidences (take maximum or average)\nmerged['Confidence'] = merged[['Confidence_existing', 'Confidence_new']].max(axis=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#actual submitted data\nfinal_predictions = merged[['Id', 'GO term', 'Confidence']].copy()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Saving submission\nprint(\"\\nSaving submission file...\")\nfinal_predictions.to_csv('submission.tsv', sep='\\t', header=False, index=False)\nprint(f\"Submission saved with {len(final_predictions)} predictions\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}