{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":116062,"databundleVersionId":14875579,"sourceType":"competition"},{"sourceId":13482672,"sourceType":"datasetVersion","datasetId":8559888},{"sourceId":13665986,"sourceType":"datasetVersion","datasetId":8688768},{"sourceId":13680623,"sourceType":"datasetVersion","datasetId":8699749}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install BIO","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Sample submission\nimport pandas as pd\nsub = pd.read_csv(\"/kaggle/input/cafa-6-protein-function-prediction/sample_submission.tsv\", sep= \"\\t\", on_bad_lines='skip', header = None)\n#sub.columns = [\"The Protein ID\", \"The Gene Ontology term (GO) ID\", \"Predicted link probability that GO appear in Protein\"]\nsub.head(5)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"MAIN_DIR = \"/kaggle/input/cafa-6-protein-function-prediction\"\nclass config:\n    train_sequences_path = MAIN_DIR  + \"/Train/train_sequences.fasta\"\n    train_term_path = MAIN_DIR +\"/Train/train_terms.tsv\"\n    train_labels_path = MAIN_DIR + \"/Train/train_terms.tsv\"\n    test_sequences_path = MAIN_DIR + \"/Test/testsuperset.fasta\"\n    taxonomy_path = os.path.join(MAIN_DIR, \"Train/train_taxonomy.tsv\")\n    n_epochs = 50\n    batch_size = 128\n    lr = 1e-5\n    early_stopping = 7\n    obo_path = \"/kaggle/input/cafa-6-protein-function-prediction/Train/go-basic.obo\"\n    goa_path = \"/kaggle/input/protein-go-annotations/goa_uniprot_all.csv\"\n    device = torch.device('cpu')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data preprocessing","metadata":{}},{"cell_type":"code","source":"from Bio import SeqIO\ndef get_train_test_df():\n    train_df = pd.read_csv(config.train_term_path, sep=\"\\t\")\n    test_df = pd.read_csv(config.test_sequences_path, sep = \"\\t\")\n    train_ids = train_df[\"EntryID\"].unique().tolist()\n    test_ids = []\n    for record in SeqIO.parse(config.test_sequences_path, \"fasta\"):\n        test_id = record.id.split()[0]\n        test_ids.append(test_id)\n    return train_df, train_ids, test_df, test_ids\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_data(model_name = \"esm2\"):\n    train_df, train_ids, test_df, test_ids = get_train_test_df()\n    if model_name == \"esm2\":\n        ids = pd.read_csv(\"/kaggle/input/cafa6-protein-embeddings-esm2/protein_ids.csv\")[\"protein_id\"].tolist()\n        id_to_index = {pid: i for i, pid in enumerate(ids)}\n        #Converting to data to train , because the embedding is already in order for each protein\n        train_idx = [id_to_index[i] for i in train_ids if i in id_to_index]\n        test_idx  = [id_to_index[i] for i in test_ids  if i in id_to_index]\n        #init train and test\n        emb = np.load(\"/kaggle/input/cafa6-protein-embeddings-esm2/protein_embeddings.npy\")\n        X_train = emb[train_idx]\n        X_test  = emb[test_idx] \n    elif model_name ==\"t5\":\n        train_ids = np.load(\"/kaggle/input/cafa-6-t5-embeddings/train_ids.npy\")\n        train_ids = np.load(\"/kaggle/input/cafa-6-t5-embeddings/train_ids.npy\")\n        test_ids = np.load(\"/kaggle/input/cafa-6-t5-embeddings/test_ids.npy\")\n        train_ids = [train_id.split(\"|\")[1] for train_id in train_ids]\n        labels_names = train_df[\"term\"].unique()\n\n        id_to_index = {pid: i for i, pid in enumerate(train_ids)}\n        train_df[\"order\"] = train_df[\"EntryID\"].map(id_to_index)\n        train_df = train_df.sort_values(\"order\").reset_index(drop=True)\n        test_ids = list(np.load(\"/kaggle/input/cafa-6-t5-embeddings/test_ids.npy\"))\n\n        X_train = np.load(\"/kaggle/input/cafa-6-t5-embeddings/train_embeds.npy\")\n        X_test  = np.load(\"/kaggle/input/cafa-6-t5-embeddings/test_embeds.npy\")\n        \n    np.save(\"X_train.npy\", X_train)\n    np.save(\"X_test.npy\", X_test)\n    pd.DataFrame(train_ids).to_csv(\"y_train_ids.csv\", index=False)\n    pd.DataFrame(test_ids).to_csv(\"y_test_ids.csv\", index=False)\n\n    from sklearn.preprocessing import MultiLabelBinarizer\n    # Group into ID and Go function for training\n    labels = train_df.groupby(\"EntryID\")[\"term\"].apply(list)\n    Y_train = MultiLabelBinarizer().fit_transform(labels[train_ids])\n    \n    return X_train, X_test, Y_train    \n    \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Pre process data \nX_train, X_test, Y_train = preprocess_data(\"t5\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torch.cuda.amp import autocast, GradScaler\nimport numpy as np\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ProteinDataset(Dataset):\n    def __init__(self, X, Y):\n        # Dung de wrap vao trong pytorch\n        self.X = torch.tensor(X, dtype=torch.float32)\n        self.Y = torch.tensor(Y, dtype=torch.float32)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, i):\n        return self.X[i], self.Y[i]\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"\nclass MLP(nn.Module):\n    def __init__(self, input_dim=1280, output_dim=None, NN_tuning = [1024,512]):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(input_dim, NN_tuning[0]),\n            nn.BatchNorm1d(NN_tuning[0]),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n\n            nn.Linear(NN_tuning[0],NN_tuning[1]),\n            nn.BatchNorm1d(NN_tuning[1]),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n\n            nn.Linear(NN_tuning[1], output_dim)  \n        )\n\n    def forward(self, x):\n        return self.net(x)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# Preparing the data for training - uncomment to use the training pipeline\n# X = X_train\n# y = Y_train\n# #training\n# #Wrapping in wrapper class\n# from sklearn.model_selection import train_test_split\n# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1)\n# #wrapper\n# train_dataset = ProteinDataset(X_train, y_train)\n# val_dataset = ProteinDataset(X_test,y_test)\n\n# batch_size = config.batch_size\n# train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n# val_loader = DataLoader(val_dataset, batch_size=batch_size)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"NN_tunings = [\n\n[1024,512], #Epoch 23/30 | Train Loss: 0.000846 | Val Loss: 0.001295\n[1024,600], # Epoch 21/30 | Train Loss: 0.000947 | Val Loss: 0.001271\n]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import copy\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\ndef train_model():\n    epochs = config.n_epochs\n    lr = config.lr\n    model_list = []  \n    final_loss = dict()\n    losses_dict = dict()  \n\n    for NN_tuning in NN_tunings:\n        model = MLP(input_dim=X_train.shape[1], output_dim=Y_train.shape[1], NN_tuning = NN_tuning).to(device)\n        \n        \n        optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=1e-4)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n            optimizer, mode='min', factor=0.5, patience=2, verbose=True\n        )\n        \n        criterion = nn.BCEWithLogitsLoss()\n        scaler = torch.amp.GradScaler()\n        best_val_loss = float(\"inf\")\n        counter =  0\n        train_losses = []\n        val_losses = []\n        for epoch in range(epochs):\n            #Early stopping\n            if (counter>=config.early_stopping):\n                print(\"Early stopping!!!\")\n                break\n            elif (epoch>10) and (best_val_loss>0.0013):\n                break;\n            model.train()\n            train_loss = 0\n            for X_batch , y_batch in train_loader:\n                X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n                #Resetting the gradient\n                optimizer.zero_grad()\n                #Mixed between float 16 and 32\n                with torch.amp.autocast(device_type=\"cuda\"):\n                    logits = model(X_batch)\n                    loss = criterion(logits, y_batch)\n                #Prevent vanishing or exploding gradient\n                scaler.scale(loss).backward()\n                #Wrap trong scaler de xem co safe de update ko\n                scaler.step(optimizer)\n                scaler.update()\n                train_loss+= loss.item() * X_batch.size(0)\n            train_loss /= len(train_loader.dataset)\n            train_losses.append(train_loss)\n\n            #val \n            model.eval()\n            val_loss = 0\n            with torch.no_grad():\n                for Xb, Yb in val_loader:\n                    Xb, Yb = Xb.to(device), Yb.to(device)\n                    with torch.amp.autocast(device_type=\"cuda\"):\n                        logits = model(Xb)\n                        loss = criterion(logits, Yb)\n                    val_loss += loss.item() * Xb.size(0)\n        \n            val_loss /= len(val_loader.dataset)\n            val_losses.append(val_loss)\n\n            #For examining val loss and changing lr \n            scheduler.step(val_loss)\n            print(f\"Epoch {epoch+1}/{epochs} | Train Loss: {train_loss:.6f} | Val Loss: {val_loss:.6f}\")\n\n\n            if val_loss < best_val_loss:\n                print(f\"saved best model at {epoch}\")\n                best_model = copy.deepcopy(model)\n                best_val_loss = val_loss\n                counter = 0\n            else:\n                print(f\"Model hasn't improve for {counter+1} epochs\")\n                counter+=1\n        print(f\"Finished training for:L1: {NN_tuning[0]}, L2:{NN_tuning[1]}\")\n        final_loss[f\"L1: {NN_tuning[0]}, L2:{NN_tuning[1]}\"] = f\"Train_loss: {train_losses[-1]}, Val_loss:{val_losses[-1]}\"\n        losses_dict[f\"L1: {NN_tuning[0]}, L2:{NN_tuning[1]}\"] = np.array([train_losses, val_losses])\n        model_list.append(best_model)\n\n        # free memory before next tuning\n        del model, optimizer, scheduler, scaler\n        torch.cuda.empty_cache()\n\n    return model_list, final_loss, losses_dict\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Stopped training because we've found the optimized model\n#model_list, final_loss, losses_dict = train_model()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Using the best model i've trained\nmodel = MLP(input_dim=X_train.shape[1], output_dim=Y_train.shape[1], NN_tuning = NN_tunings[0]).to(device)\nstate_dict = torch.load(\"/kaggle/input/model-weights/model_weights0.00124_LOSS_0.2_(RELU_1024_512).pt\", map_location=\"cpu\")\nmodel.load_state_dict(state_dict)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Predicting","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom tqdm import tqdm\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\ntorch.cuda.empty_cache()\n\n\ndef predict_protein():\n    model = model.to(device)\n    model.eval()\n    model = model.half()        \n       \n    \n    #X_test = np.load(\"/kaggle/working/X_test.npy\", mmap_mode=\"r\")\n    X_test = np.load(\"/kaggle/working/X_test.npy\")\n    \n    test_ids = pd.read_csv(\"/kaggle/working/y_test_ids.csv\")[\"0\"].tolist()\n    test_id_to_index = {pid: i for i, pid in enumerate(test_ids)}\n    \n    train_terms = pd.read_csv(\n        \"/kaggle/input/cafa-6-protein-function-prediction/Train/train_terms.tsv\",\n        sep=\"\\t\"\n    )\n    all_go_terms = sorted(train_terms[\"term\"].unique())\n    \n    batch_size = 32\n    min_prob = 0.02      \n    top_k = 25        \n    \n    submission_rows = []\n    \n    for i in tqdm(range(0, len(test_ids), batch_size), desc=\"Predicting & writing\"):\n        batch_ids = test_ids[i:i+batch_size]\n        batch_idx = [test_id_to_index[pid] for pid in batch_ids]\n    \n        batch = X_test[batch_idx].astype(np.float16)\n        batch = torch.tensor(batch, dtype=torch.float16).to(device)\n    \n        with torch.no_grad():\n            logits = model(batch)\n            probs = torch.sigmoid(logits).cpu().numpy()\n    \n        for j, pid in enumerate(batch_ids):\n            p = probs[j]\n    \n            # ----- Lọc theo min_prob -----\n            idx = np.where(p >= min_prob)[0]\n    \n            # ----- (Tuỳ chọn) thêm top_k nếu ít kết quả quá -----\n            if top_k is not None and len(idx) < top_k:\n                top_extra = p.argsort()[-top_k:]\n                idx = np.unique(np.concatenate([idx, top_extra]))\n    \n            # Sắp xếp theo xác suất giảm dần\n            idx = idx[np.argsort(-p[idx])]\n    \n            # Ghi vào submission\n            for k in idx:\n                term = all_go_terms[k]\n                score = float(p[k])\n                submission_rows.append([pid, term, score])\n        if len(submission_rows) > 5000:\n            df = pd.DataFrame(submission_rows, columns=[\"Id\", \"term\", \"score\"])\n            df.to_csv(\"submission1.tsv\",sep=\"\\t\", mode=\"a\", header=False, index=False)\n            submission_rows = []    \n        del batch, logits, probs\n    \n    torch.cuda.empty_cache()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predict_protein()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Post processing","metadata":{}},{"cell_type":"code","source":"from typing import Dict, List, Set, Tuple, Optional\nimport pandas as pd\ndef parse_obo(obo_file: str) -> Tuple[Dict[str, List[str]], Dict[str, List[str]]]:\n    go_parents = {}   \n    go_children = {}  \n    current_term = None\n    is_obsolete = False\n    \n    print(f\"Parsing GO hierarchy from {obo_file}...\")\n    \n    with open(obo_file, 'r') as f:\n        for line in f:\n            line = line.strip()\n            \n            if line == \"[Term]\":\n                current_term = None\n                is_obsolete = False\n                \n            elif line.startswith(\"id: GO:\"):\n                current_term = line.split(\"id: \")[1]\n                go_parents[current_term] = []\n                go_children.setdefault(current_term, [])\n                \n            elif line.startswith(\"is_obsolete: true\"):\n                is_obsolete = True\n                \n            elif current_term and not is_obsolete:\n                if line.startswith(\"is_a:\"):\n                    parent = line.split(\"is_a: \")[1].split(\" !\")[0].strip()\n                    go_parents[current_term].append(parent)\n                    go_children.setdefault(parent, []).append(current_term)\n                    \n                elif line.startswith(\"relationship: part_of\"):\n                    parent = line.split(\"part_of \")[1].split(\" !\")[0].strip()\n                    go_parents[current_term].append(parent)\n                    go_children.setdefault(parent, []).append(current_term)\n    \n    print(f\"Loaded {len(go_parents)} GO terms\")\n    return go_parents, go_children\n\n\ndef get_all_ancestors(term: str, go_parents: Dict, cache: Dict = None) -> Set[str]:\n    \"\"\"Get all ancestors of a GO term .\"\"\"\n    if cache is None:\n        cache = {}\n    if term in cache:\n        return cache[term]\n    \n    ancestors = set()\n    stack = [term]\n    while stack:\n        cur = stack.pop()\n        for parent in go_parents.get(cur, []):\n            if parent not in ancestors:\n                ancestors.add(parent)\n                stack.append(parent)\n    \n    cache[term] = ancestors\n    return ancestors\n\n\ndef get_all_descendants(term: str, go_children: Dict, cache: Dict = None) -> Set[str]:\n    \"\"\"Get all descendants of a GO term .\"\"\"\n    if cache is None:\n        cache = {}\n    if term in cache:\n        return cache[term]\n    \n    descendants = set()\n    stack = [term]\n    while stack:\n        cur = stack.pop()\n        for child in go_children.get(cur, []):\n            if child not in descendants:\n                descendants.add(child)\n                stack.append(child)\n    \n    cache[term] = descendants\n    return descendants\n\n\ndef propagate_predictions(\n    predictions_df: pd.DataFrame, \n    go_parents: Dict[str, List[str]]\n) -> pd.DataFrame:\n    print(\"Propagating predictions to ancestor GO terms...\")\n    ancestor_cache = {}\n    propagated_rows = []\n    \n    for pid, group in tqdm(predictions_df.groupby('pid'), desc=\"Propagating\"):\n        term_scores = {}\n        \n        for _, row in group.iterrows():\n            term, score = row['term'], row['p']\n            term_scores[term] = max(term_scores.get(term, 0), score)\n            \n            for ancestor in get_all_ancestors(term, go_parents, ancestor_cache):\n                term_scores[ancestor] = max(term_scores.get(ancestor, 0), score)\n        \n        for term, score in term_scores.items():\n            propagated_rows.append({'pid': pid, 'term': term, 'p': score})\n    \n    propagated_df = pd.DataFrame(propagated_rows)\n    print(f\"Before: {len(predictions_df)}, After: {len(propagated_df)} predictions\")\n    return propagated_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load GOA Uniprot Data","metadata":{}},{"cell_type":"code","source":"def load_goa_annotations(\n    goa_path: str, \n    go_children: Dict[str, List[str]]\n) -> Tuple[Set[str], Optional[pd.DataFrame]]:\n\n    if not os.path.exists(goa_path):\n        print(f\"[WARNING] GOA file not found: {goa_path}\")\n        return set(), None\n    \n    print(f\"Loading GOA annotations from {goa_path}...\")\n    goa_df = pd.read_csv(goa_path).drop_duplicates()\n    print(f\"Loaded {len(goa_df)} GOA annotations\")\n    \n    # Extract NEGATIVE \n    print(\"Processing negative annotations...\")\n    neg = goa_df[goa_df['qualifier'].str.contains('NOT', na=False)]\n    neg = neg[['protein_id', 'go_term']].drop_duplicates()\n    neg_by_protein = neg.groupby('protein_id')['go_term'].apply(list).to_dict()\n    \n    # Negative\n    desc_cache = {}\n    negative_keys = set()\n    for protein, terms in tqdm(neg_by_protein.items(), desc=\"Propagating negatives\"):\n        all_neg = set(terms)\n        for term in terms:\n            all_neg |= get_all_descendants(term, go_children, desc_cache)\n        for term in all_neg:\n            negative_keys.add(f\"{protein}_{term}\")\n    print(f\"Total negative pairs: {len(negative_keys)}\")\n    \n    # Postivie\n    print(\"Processing positive annotations...\")\n    pos = goa_df[~goa_df['qualifier'].str.contains('NOT', na=False)]\n    pos = pos[['protein_id', 'go_term']].drop_duplicates()\n    pos['score'] = 1.0\n    pos['pred_key'] = pos['protein_id'].astype(str) + '_' + pos['go_term'].astype(str)\n    pos = pos[~pos['pred_key'].isin(negative_keys)]\n    print(f\"Total positive pairs: {len(pos)}\")\n    \n    return negative_keys, pos\n\n\ndef apply_negative_propagation(\n    predictions_df: pd.DataFrame, \n    negative_keys: Set[str]\n) -> pd.DataFrame:\n    \"\"\"Remove predictions matching negative protein-GO pairs.\"\"\"\n    if not negative_keys:\n        return predictions_df\n    \n    print(\"Removing negative predictions...\")\n    df = predictions_df.copy()\n    df['pred_key'] = df['pid'].astype(str) + '_' + df['term'].astype(str)\n    before = len(df)\n    df = df[~df['pred_key'].isin(negative_keys)]\n    df = df.drop(columns=['pred_key'])\n    print(f\"Removed {before - len(df)} predictions ({before} -> {len(df)})\")\n    return df\n\n\ndef add_goa_ground_truth(\n    predictions_df: pd.DataFrame, \n    goa_positive_df: Optional[pd.DataFrame]\n) -> pd.DataFrame:\n    \"\"\"Add GOA ground truth annotations with score 1.0.\"\"\"\n    if goa_positive_df is None or len(goa_positive_df) == 0:\n        return predictions_df\n    \n    print(\"Adding GOA ground truth annotations...\")\n    test_proteins = set(predictions_df['pid'].unique())\n    \n    goa = goa_positive_df[goa_positive_df['protein_id'].isin(test_proteins)].copy()\n    goa = goa.rename(columns={'protein_id': 'pid', 'go_term': 'term', 'score': 'p'})\n    goa = goa[['pid', 'term', 'p']]\n    print(f\"Found {len(goa)} GOA annotations for test proteins\")\n    \n    combined = pd.concat([predictions_df, goa], ignore_index=True)\n    combined = combined.groupby(['pid', 'term'])['p'].max().reset_index()\n    print(f\"After adding GOA: {len(predictions_df)} -> {len(combined)}\")\n    return combined","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef merge_goa(file_path = \"/kaggle/input/best-submission/submission1.tsv\"):\n    df = pd.read_csv(file_path, sep=\"\\t\", header=None)\n    \n\n    df = df.rename(columns={0: 'pid', 1: 'term', 2: 'p'})\n    \n    go_parents, go_children = parse_obo(config.obo_path)\n    \n    #load\n    negative_keys, goa_positive_df = load_goa_annotations(config.goa_path, go_children)\n    \n    #propagteted ancestor and descendant\n\n    propagated_df = propagate_predictions(df, go_parents)\n    \n    #negative propaggation # Can stop here for 0.313 model\n    cleaned_df = apply_negative_propagation(propagated_df, negative_keys)\n    final_df = cleaned_df\n    # adding ground truth only work for 0.3 model # goa ground truth only hurt the model prediction\n    final_df = add_goa_ground_truth(cleaned_df, goa_positive_df)\n\n    final_df = final_df.sort_values(by='p', ascending=False)Q\n    final_df.to_csv('submission.tsv', sep='\\t', index=False, header=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"merge_goa(\"/kaggle/working/submission.tsv\")\ndf = pd.read_csv(\"submission.tsv\", sep = \"\\t\", header = None)\ndf_sorted = df.sort_values(\n    by=[0, 2], \n    ascending=[True, False]  \n)\ndf_sorted.to_csv(\n    'submission.tsv',  \n    sep='\\t',          \n    header=False,     \n    index=False        \n)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}