{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from Bio import SeqIO\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-04T16:44:26.915524Z","iopub.execute_input":"2023-07-04T16:44:26.915886Z","iopub.status.idle":"2023-07-04T16:44:27.083230Z","shell.execute_reply.started":"2023-07-04T16:44:26.915856Z","shell.execute_reply":"2023-07-04T16:44:27.082001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAIN_DIR = \"/kaggle/input/cafa-5-protein-function-prediction\"\n\n# UTILITARIES\nimport numpy as np\nfrom tqdm import tqdm\nimport time\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\n\n# TORCH MODULES FOR METRICS COMPUTATION :\nimport torch\nfrom torch.utils.data import Dataset\nfrom torch import nn\nfrom torch.utils.data import random_split\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torchmetrics.classification import MultilabelF1Score\nfrom torchmetrics.classification import MultilabelAccuracy\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.loggers import WandbLogger\n\n# WANDB FOR LIGHTNING :\nimport wandb\n\n# FILES VISUALIZATION\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:44:30.629002Z","iopub.execute_input":"2023-07-04T16:44:30.629383Z","iopub.status.idle":"2023-07-04T16:44:47.140434Z","shell.execute_reply.started":"2023-07-04T16:44:30.629349Z","shell.execute_reply":"2023-07-04T16:44:47.139396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    train_sequences_path = MAIN_DIR  + \"/Train/train_sequences.fasta\"\n    train_labels_path = MAIN_DIR + \"/Train/train_terms.tsv\"\n    test_sequences_path = MAIN_DIR + \"/Test (Targets)/testsuperset.fasta\"\n    \n    num_labels = 500\n    n_epochs = 18\n    batch_size = 128\n    lr = 0.01\n    \n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:44:51.128737Z","iopub.execute_input":"2023-07-04T16:44:51.129112Z","iopub.status.idle":"2023-07-04T16:44:51.150123Z","shell.execute_reply.started":"2023-07-04T16:44:51.129081Z","shell.execute_reply":"2023-07-04T16:44:51.149168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Load ProtBERT Model...\")\n# PROT BERT LOADING :\nfrom transformers import BertModel, BertTokenizer\ntokenizer = BertTokenizer.from_pretrained(\"Rostlab/prot_bert\", do_lower_case=False )\nmodel = BertModel.from_pretrained(\"Rostlab/prot_bert\").to(config.device)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:44:54.937732Z","iopub.execute_input":"2023-07-04T16:44:54.938151Z","iopub.status.idle":"2023-07-04T16:45:29.148962Z","shell.execute_reply.started":"2023-07-04T16:44:54.938120Z","shell.execute_reply":"2023-07-04T16:45:29.147970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_bert_embedding(\n#     sequence : str,\n#     len_seq_limit : int\n# ):\n#     '''\n#     Function to collect last hidden state embedding vector from pre-trained ProtBERT Model\n\n#     INPUTS:\n#     - sequence (str) : protein sequence (ex : AAABBB) from fasta file\n#     - len_seq_limit (int) : maximum sequence lenght (i.e nb of letters) for truncation\n\n#     OUTPUTS:\n#     - output_hidden : last hidden state embedding vector for input sequence of length 1024\n#     '''\n#     sequence_w_spaces = ' '.join(list(sequence))\n#     encoded_input = tokenizer(\n#         sequence_w_spaces,\n#         truncation=True,\n#         max_length=len_seq_limit,\n#         padding='max_length',\n#         return_tensors='pt').to(config.device)\n#     output = model(**encoded_input)\n#     output_hidden = output['last_hidden_state'][:,0][0].detach().cpu().numpy()\n#     assert len(output_hidden)==1024\n#     return output_hidden","metadata":{"execution":{"iopub.status.busy":"2023-07-04T14:46:40.690006Z","iopub.execute_input":"2023-07-04T14:46:40.692199Z","iopub.status.idle":"2023-07-04T14:46:40.701988Z","shell.execute_reply.started":"2023-07-04T14:46:40.692164Z","shell.execute_reply":"2023-07-04T14:46:40.700919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ### COLLECTING FOR TRAIN SAMPLES :\n# print(\"Loading train set ProtBERT Embeddings...\")\n# fasta_train = SeqIO.parse(config.train_sequences_path, \"fasta\")\n# print(\"Total Nb of Elements : \", len(list(fasta_train)))\n# fasta_train = SeqIO.parse(config.train_sequences_path, \"fasta\")\n# ids_list = []\n# embed_vects_list = []\n# t0 = time.time()\n# checkpoint = 0\n# for item in tqdm(fasta_train):\n#     ids_list.append(item.id)\n#     embed_vects_list.append(\n#         get_bert_embedding(sequence = item.seq, len_seq_limit = 1200))\n#     checkpoint+=1\n#     if checkpoint>=100:\n#         df_res = pd.DataFrame(data={\"id\" : ids_list, \"embed_vect\" : embed_vects_list})\n#         np.save('/kaggle/working/train_ids.npy',np.array(ids_list))\n#         np.save('/kaggle/working/train_embeddings.npy',np.array(embed_vects_list))\n#         checkpoint=0\n\n# np.save('/kaggle/working/train_ids.npy',np.array(ids_list))\n# np.save('/kaggle/working/train_embeddings.npy',np.array(embed_vects_list))\n# print('Total Elapsed Time:',time.time()-t0)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T19:07:49.035312Z","iopub.execute_input":"2023-07-03T19:07:49.035713Z","iopub.status.idle":"2023-07-03T19:08:27.704055Z","shell.execute_reply.started":"2023-07-03T19:07:49.035680Z","shell.execute_reply":"2023-07-03T19:08:27.702669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ### COLLECTING FOR TEST SAMPLES :\n# print(\"Loading test set ProtBERT Embeddings...\")\n# fasta_test = SeqIO.parse(config.test_sequences_path, \"fasta\")\n# print(\"Total Nb of Elements : \", len(list(fasta_test)))\n# fasta_test = SeqIO.parse(config.test_sequences_path, \"fasta\")\n# ids_list = []\n# embed_vects_list = []\n# t0 = time.time()\n# checkpoint=0\n# for item in tqdm(fasta_test):\n#     ids_list.append(item.id)\n#     embed_vects_list.append(\n#         get_bert_embedding(sequence = item.seq, len_seq_limit = 1200))\n#     checkpoint+=1\n#     if checkpoint>=100:\n#         np.save('/kaggle/working/test_ids.npy',np.array(ids_list))\n#         np.save('/kaggle/working/test_embeddings.npy',np.array(embed_vects_list))\n#         checkpoint=0\n\n# np.save('/kaggle/working/test_ids.npy',np.array(ids_list))\n# np.save('/kaggle/working/test_embeddings.npy',np.array(embed_vects_list))\n# print('Total Elasped Time:',time.time()-t0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"GENERATE TARGETS FOR ENTRY IDS (\"+str(config.num_labels)+\" MOST COMMON GO TERMS)\")\nids = np.load(\"/kaggle/input/protbert-embeddings-for-cafa5/train_ids.npy\")\nlabels = pd.read_csv(config.train_labels_path, sep = \"\\t\")\n\ntop_terms = labels.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\nlabels_names = top_terms[:config.num_labels].index.values\ntrain_labels_sub = labels[(labels.term.isin(labels_names)) & (labels.EntryID.isin(ids))]\nid_labels = train_labels_sub.groupby('EntryID')['term'].apply(list).to_dict()\n\ngo_terms_map = {label: i for i, label in enumerate(labels_names)}\nlabels_matrix = np.empty((len(ids), len(labels_names)))\n\nfor index, id in tqdm(enumerate(ids)):\n    id_gos_list = id_labels[id]\n    temp = [go_terms_map[go] for go in labels_names if go in id_gos_list]\n    labels_matrix[index, temp] = 1\n\nnp.save(\"/kaggle/working/train_targets_top\"+str(config.num_labels)+\".npy\", np.array(labels_matrix))\nprint(\"GENERATION FINISHED!\")","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:47:55.722210Z","iopub.execute_input":"2023-07-04T16:47:55.722726Z","iopub.status.idle":"2023-07-04T16:49:16.427962Z","shell.execute_reply.started":"2023-07-04T16:47:55.722685Z","shell.execute_reply":"2023-07-04T16:49:16.425965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pickle\n\n# # Assuming you already have the variables 'labels', 'id_labels', and 'labels_names' from your previous code\n# # Ensure that 'labels_names' contains the top 500 most common GO terms\n\n# # Filter the 'labels' DataFrame to keep only the top 500 most common GO terms\n# top_500_labels = labels[labels['term'].isin(labels_names)]\n\n# # Save 'top_500_labels' and 'id_labels' to a dictionary\n# data_dict = {'top_500_labels': top_500_labels, 'id_labels': id_labels}\n\n# # Define the filename for the pickle file\n# pickle_file = \"/kaggle/working/train_targets_top\"+str(config.num_labels)+\".pkl\"\n\n# # Open the file in binary write mode and save the data_dict\n# with open(pickle_file, 'wb') as f:\n#     pickle.dump(data_dict, f)\n\n# print(\"Pickle file created successfully at:\", pickle_file)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T14:48:30.342763Z","iopub.execute_input":"2023-07-04T14:48:30.343201Z","iopub.status.idle":"2023-07-04T14:48:31.887858Z","shell.execute_reply.started":"2023-07-04T14:48:30.343171Z","shell.execute_reply":"2023-07-04T14:48:31.886696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pandas as pd\n\n# # Load the pickled dictionary\n# data_dict = pd.read_pickle(\"/kaggle/working/train_targets_top\"+str(config.num_labels)+\".pkl\")\n\n# # Get the number of entries (assuming the dictionary is the same as DataFrame rows)\n# num_entries = len(data_dict)\n# print(\"Number of entries:\", num_entries)\n\n# # Get the dimensions of the dictionary (if the value is a pandas DataFrame or similar)\n# dimensions = {}\n# for key, value in data_dict.items():\n#     if isinstance(value, pd.DataFrame) or isinstance(value, pd.Series):\n#         dimensions[key] = value.shape\n#     elif isinstance(value, list):\n#         dimensions[key] = len(value)\n#     else:\n#         dimensions[key] = \"Unknown\"\n\n# print(\"Dimensions of the dictionary:\")\n# for key, value in dimensions.items():\n#     print(f\"Key: {key}, Dimensions: {value}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:09:38.064462Z","iopub.execute_input":"2023-07-04T15:09:38.064839Z","iopub.status.idle":"2023-07-04T15:09:39.673841Z","shell.execute_reply.started":"2023-07-04T15:09:38.064788Z","shell.execute_reply":"2023-07-04T15:09:39.672737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class ProteinSequenceDataset(Dataset):\n    \n#     def __init__(self, datatype, embeddings_source):\n#         super(ProteinSequenceDataset).__init__()\n#         self.datatype = datatype\n        \n#         if embeddings_source in [\"ProtBERT\", \"EMS2\"]:\n#             embeds = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_embeddings.npy\")\n#             ids = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_ids.npy\")\n        \n#         if embeddings_source == \"T5\":\n#             embeds = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_embeds.npy\")\n#             ids = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_ids.npy\")\n            \n#         embeds_list = []\n#         for l in range(embeds.shape[0]):\n#             embeds_list.append(embeds[l,:])\n#         self.df = pd.DataFrame(data={\"EntryID\": ids, \"embed\" : embeds_list})\n        \n#         if datatype==\"train\":\n#             df_labels = pd.read_pickle(\n#                 \"/kaggle/working/train_targets_top\"+str(config.num_labels)+\".pkl\")\n\n#             # Assuming self.df is your existing DataFrame and df_labels is the dictionary with \"top_500_labels\" and \"id_labels\" keys\n#             # Extract the \"top_500_labels\" DataFrame from the dictionary\n#             df_labels = df_labels[\"top_500_labels\"]\n\n#             # Merge the two DataFrames on \"EntryID\" column\n#             self.df = self.df.merge(df_labels, on=\"EntryID\")\n\n\n#             # Merge the two DataFrames on \"EntryID\" column\n#             #self.df = self.df.merge(df_labels, on=\"EntryID\")\n#             #self.df = self.df.merge(df_labels, on=\"EntryID\")\n        \n#     def __len__(self):\n#         return len(self.df)\n    \n#     def __getitem__(self, index):\n#         embed = torch.tensor(self.df.iloc[index][\"embed\"] , dtype = torch.float32)\n#         if self.datatype==\"train\":\n#             targets = torch.tensor(self.df.iloc[index][\"labels_vect\"], dtype = torch.float32)\n#             return embed, targets\n#         if self.datatype==\"test\":\n#             id = self.df.iloc[index][\"EntryID\"]\n#             return embed, id\n        ","metadata":{"execution":{"iopub.status.busy":"2023-07-04T15:29:07.964666Z","iopub.execute_input":"2023-07-04T15:29:07.965476Z","iopub.status.idle":"2023-07-04T15:29:07.981379Z","shell.execute_reply.started":"2023-07-04T15:29:07.965433Z","shell.execute_reply":"2023-07-04T15:29:07.980004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ProteinSequenceDataset(Dataset):\n    \n    def __init__(self, datatype, embeddings_source):\n        super(ProteinSequenceDataset).__init__()\n        self.datatype = datatype\n        \n        if embeddings_source in [\"ProtBERT\", \"EMS2\"]:\n            embeds = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_embeddings.npy\")\n            ids = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_ids.npy\")\n        \n        if embeddings_source == \"T5\":\n            embeds = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_embeds.npy\")\n            ids = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_ids.npy\")\n            \n        embeds_list = []\n        for l in range(embeds.shape[0]):\n            embeds_list.append(embeds[l,:])\n        self.df = pd.DataFrame(data={\"EntryID\": ids, \"embed\" : embeds_list})\n        \n        if datatype==\"train\":\n            df_labels = np.load(\"/kaggle/working/train_targets_top500.npy\")\n            self.df[\"labels_vect\"] = df_labels.tolist()\n            #self.df = self.df.merge(df_labels, on=\"EntryID\")\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        embed = torch.tensor(self.df.iloc[index][\"embed\"] , dtype = torch.float32)\n        if self.datatype==\"train\":\n            targets = torch.tensor(self.df.iloc[index][\"labels_vect\"], dtype = torch.float32)\n            return embed, targets\n        if self.datatype==\"test\":\n            id = self.df.iloc[index][\"EntryID\"]\n            return embed, id\n        ","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:49:42.079633Z","iopub.execute_input":"2023-07-04T16:49:42.080012Z","iopub.status.idle":"2023-07-04T16:49:42.094369Z","shell.execute_reply.started":"2023-07-04T16:49:42.079982Z","shell.execute_reply":"2023-07-04T16:49:42.092988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# from torch.utils.data import Dataset\n# import numpy as np\n# import pandas as pd\n\n# class ProteinSequenceDataset(Dataset):\n    \n#     def __init__(self, datatype, embeddings_source):\n#         super(ProteinSequenceDataset, self).__init__()\n#         self.datatype = datatype\n        \n#         if embeddings_source in [\"ProtBERT\", \"EMS2\"]:\n#             embeds = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_embeddings.npy\")\n#             ids = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_ids.npy\")\n        \n#         if embeddings_source == \"T5\":\n#             embeds = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_embeds.npy\")\n#             ids = np.load(\"/kaggle/input/\"+embeds_map[embeddings_source]+\"/\"+datatype+\"_ids.npy\")\n            \n#         embeds_list = []\n#         for l in range(embeds.shape[0]):\n#             embeds_list.append(embeds[l,:])\n#         self.df = pd.DataFrame(data={\"EntryID\": ids, \"embed\" : embeds_list})\n        \n#         if datatype == \"train\":\n#             # Read df_labels as a DataFrame\n#             #df_labels = pd.read_pickle(\"/kaggle/working/train_targets_top\"+str(config.num_labels)+\".pkl\")\n#             df_labels = pd.read_pickle(\"/kaggle/working/train_targets_top500.pkl\")\n#             # Check if df_labels is a DataFrame and contains 'EntryID' and 'labels_vect' columns\n#             if not isinstance(df_labels, pd.DataFrame) or \"EntryID\" not in df_labels.columns or \"labels_vect\" not in df_labels.columns:\n#                 raise ValueError(\"Invalid df_labels format. It should be a DataFrame with 'EntryID' and 'labels_vect' columns.\")\n#             self.df = self.df.merge(df_labels, on=\"EntryID\")\n        \n#     def __len__(self):\n#         return len(self.df)\n    \n#     def __getitem__(self, index):\n#         embed = torch.tensor(self.df.iloc[index][\"embed\"], dtype=torch.float32)\n#         if self.datatype == \"train\":\n#             targets = torch.tensor(self.df.iloc[index][\"labels_vect\"], dtype=torch.float32)\n#             return embed, targets\n#         if self.datatype == \"test\":\n#             entry_id = self.df.iloc[index][\"EntryID\"]\n#             return embed, entry_id\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T14:49:10.756524Z","iopub.execute_input":"2023-07-04T14:49:10.756903Z","iopub.status.idle":"2023-07-04T14:49:10.770680Z","shell.execute_reply.started":"2023-07-04T14:49:10.756868Z","shell.execute_reply":"2023-07-04T14:49:10.769745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MultiLayerPerceptron(torch.nn.Module):\n\n    def __init__(self, input_dim, num_classes):\n        super(MultiLayerPerceptron, self).__init__()\n\n        self.linear1 = torch.nn.Linear(input_dim, 1012)\n        self.activation1 = torch.nn.ReLU()\n        self.linear2 = torch.nn.Linear(1012, 712)\n        self.activation2 = torch.nn.ReLU()\n        self.linear3 = torch.nn.Linear(712, num_classes)\n\n    def forward(self, x):\n        x = self.linear1(x)\n        x = self.activation1(x)\n        x = self.linear2(x)\n        x = self.activation2(x)\n        x = self.linear3(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:49:45.780172Z","iopub.execute_input":"2023-07-04T16:49:45.780534Z","iopub.status.idle":"2023-07-04T16:49:45.788598Z","shell.execute_reply.started":"2023-07-04T16:49:45.780506Z","shell.execute_reply":"2023-07-04T16:49:45.787384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(embeddings_source, model_type=\"linear\", train_size=0.9):\n    \n    train_dataset = ProteinSequenceDataset(datatype=\"train\", embeddings_source = embeddings_source)\n    \n    train_set, val_set = random_split(train_dataset, lengths = [int(len(train_dataset)*train_size), len(train_dataset)-int(len(train_dataset)*train_size)])\n    train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=config.batch_size, shuffle=True)\n    val_dataloader = torch.utils.data.DataLoader(val_set, batch_size=config.batch_size, shuffle=True)\n\n    if model_type == \"linear\":\n        model = MultiLayerPerceptron(input_dim=embeds_dim[embeddings_source], num_classes=config.num_labels).to(config.device)\n    if model_type == \"convolutional\":\n        model = CNN1D(input_dim=embeds_dim[embeddings_source], num_classes=config.num_labels).to(config.device)\n\n    optimizer = torch.optim.Adam(model.parameters(), lr = config.lr)\n    scheduler = ReduceLROnPlateau(optimizer, factor=0.1, patience=1)\n    CrossEntropy = torch.nn.CrossEntropyLoss()\n    f1_score = MultilabelF1Score(num_labels=config.num_labels).to(config.device)\n    n_epochs = config.n_epochs\n\n    print(\"BEGIN TRAINING...\")\n    train_loss_history=[]\n    val_loss_history=[]\n    \n    train_f1score_history=[]\n    val_f1score_history=[]\n    for epoch in range(n_epochs):\n        print(\"EPOCH \", epoch+1)\n        ## TRAIN PHASE :\n        losses = []\n        scores = []\n        for embed, targets in tqdm(train_dataloader):\n            embed, targets = embed.to(config.device), targets.to(config.device)\n            optimizer.zero_grad()\n            preds = model(embed)\n            loss= CrossEntropy(preds, targets)\n            score=f1_score(preds, targets)\n            losses.append(loss.item()) \n            scores.append(score.item())\n            loss.backward()\n            optimizer.step()\n        avg_loss = np.mean(losses)\n        avg_score = np.mean(scores)\n        print(\"Running Average TRAIN Loss : \", avg_loss)\n        print(\"Running Average TRAIN F1-Score : \", avg_score)\n        train_loss_history.append(avg_loss)\n        train_f1score_history.append(avg_score)\n        \n        ## VALIDATION PHASE : \n        losses = []\n        scores = []\n        for embed, targets in val_dataloader:\n            embed, targets = embed.to(config.device), targets.to(config.device)\n            preds = model(embed)\n            loss= CrossEntropy(preds, targets)\n            score=f1_score(preds, targets)\n            losses.append(loss.item())\n            scores.append(score.item())\n        avg_loss = np.mean(losses)\n        avg_score = np.mean(scores)\n        print(\"Running Average VAL Loss : \", avg_loss)\n        print(\"Running Average VAL F1-Score : \", avg_score)\n        val_loss_history.append(avg_loss)\n        val_f1score_history.append(avg_score)\n        \n        scheduler.step(avg_loss)\n        print(\"\\n\")\n        \n    print(\"TRAINING FINISHED\")\n    print(\"FINAL TRAINING SCORE : \", train_f1score_history[-1])\n    print(\"FINAL VALIDATION SCORE : \", val_f1score_history[-1])\n    \n    losses_history = {\"train\" : train_loss_history, \"val\" : val_loss_history}\n    scores_history = {\"train\" : train_f1score_history, \"val\" : val_f1score_history}\n    \n    return model, losses_history, scores_history","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:49:47.536181Z","iopub.execute_input":"2023-07-04T16:49:47.536547Z","iopub.status.idle":"2023-07-04T16:49:47.556513Z","shell.execute_reply.started":"2023-07-04T16:49:47.536519Z","shell.execute_reply":"2023-07-04T16:49:47.555384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Directories for the different embedding vectors : \nembeds_map = {\n    \"T5\" : \"t5embeds\",\n    \"ProtBERT\" : \"protbert-embeddings-for-cafa5\",\n    \"EMS2\" : \"cafa-5-ems-2-embeddings-numpy\"\n}\n\n# Length of the different embedding vectors :\nembeds_dim = {\n    \"T5\" : 1024,\n    \"ProtBERT\" : 1024,\n    \"EMS2\" : 1280\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:49:49.851299Z","iopub.execute_input":"2023-07-04T16:49:49.851674Z","iopub.status.idle":"2023-07-04T16:49:49.857610Z","shell.execute_reply.started":"2023-07-04T16:49:49.851643Z","shell.execute_reply":"2023-07-04T16:49:49.856279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"protbert_model, protbert_losses, protbert_scores = train_model(embeddings_source=\"ProtBERT\",model_type=\"linear\")","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:49:54.221810Z","iopub.execute_input":"2023-07-04T16:49:54.222546Z","iopub.status.idle":"2023-07-04T17:00:30.789792Z","shell.execute_reply.started":"2023-07-04T16:49:54.222508Z","shell.execute_reply":"2023-07-04T17:00:30.788630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 4))\nplt.plot(protbert_losses[\"val\"], label = \"ProtBERT\")\nplt.title(\"Validation Losses for # Vector Embeddings\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Average Loss\")\nplt.legend()\nplt.show()\n\nplt.figure(figsize = (10, 4))\nplt.plot(protbert_scores[\"val\"], label = \"ProtBERT\")\nplt.title(\"Validation F1-Scores for # Vector Embeddings\")\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Average F1-Score\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-04T17:00:40.926320Z","iopub.execute_input":"2023-07-04T17:00:40.926701Z","iopub.status.idle":"2023-07-04T17:00:41.629856Z","shell.execute_reply.started":"2023-07-04T17:00:40.926669Z","shell.execute_reply":"2023-07-04T17:00:41.628858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(embeddings_source):\n    \n    test_dataset = ProteinSequenceDataset(datatype=\"test\", embeddings_source = embeddings_source)\n    test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False)\n    \n    if embeddings_source == \"T5\":\n        model = t5_model\n    if embeddings_source == \"ProtBERT\":\n        model = protbert_model\n    if embeddings_source == \"EMS2\":\n        model = ems2_model\n        \n    model.eval()\n    \n    labels = pd.read_csv(config.train_labels_path, sep = \"\\t\")\n    top_terms = labels.groupby(\"term\")[\"EntryID\"].count().sort_values(ascending=False)\n    labels_names = top_terms[:config.num_labels].index.values\n    print(\"GENERATE PREDICTION FOR TEST SET...\")\n\n    ids_ = np.empty(shape=(len(test_dataloader)*config.num_labels,), dtype=object)\n    go_terms_ = np.empty(shape=(len(test_dataloader)*config.num_labels,), dtype=object)\n    confs_ = np.empty(shape=(len(test_dataloader)*config.num_labels,), dtype=np.float32)\n\n    for i, (embed, id) in tqdm(enumerate(test_dataloader)):\n        embed = embed.to(config.device)\n        confs_[i*config.num_labels:(i+1)*config.num_labels] = torch.nn.functional.sigmoid(model(embed)).squeeze().detach().cpu().numpy()\n        ids_[i*config.num_labels:(i+1)*config.num_labels] = id[0]\n        go_terms_[i*config.num_labels:(i+1)*config.num_labels] = labels_names\n\n    submission_df = pd.DataFrame(data={\"Id\" : ids_, \"GO term\" : go_terms_, \"Confidence\" : confs_})\n    print(\"PREDICTIONS DONE\")\n    return submission_df","metadata":{"execution":{"iopub.status.busy":"2023-07-04T17:00:48.479230Z","iopub.execute_input":"2023-07-04T17:00:48.479605Z","iopub.status.idle":"2023-07-04T17:00:48.492537Z","shell.execute_reply.started":"2023-07-04T17:00:48.479574Z","shell.execute_reply":"2023-07-04T17:00:48.491098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = predict(\"ProtBERT\")","metadata":{"execution":{"iopub.status.busy":"2023-07-04T17:00:56.618526Z","iopub.execute_input":"2023-07-04T17:00:56.618947Z","iopub.status.idle":"2023-07-04T17:02:38.350444Z","shell.execute_reply.started":"2023-07-04T17:00:56.618895Z","shell.execute_reply":"2023-07-04T17:02:38.348528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#np.save(\"pred.npy\", submission_df)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T17:02:43.517044Z","iopub.execute_input":"2023-07-04T17:02:43.517444Z","iopub.status.idle":"2023-07-04T17:03:22.062442Z","shell.execute_reply.started":"2023-07-04T17:02:43.517413Z","shell.execute_reply":"2023-07-04T17:03:22.061294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2023-07-04T16:41:19.646691Z","iopub.execute_input":"2023-07-04T16:41:19.647390Z","iopub.status.idle":"2023-07-04T16:41:19.669571Z","shell.execute_reply.started":"2023-07-04T16:41:19.647354Z","shell.execute_reply":"2023-07-04T16:41:19.668209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.tsv', sep='\\t', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T17:21:18.753028Z","iopub.execute_input":"2023-07-04T17:21:18.753978Z","iopub.status.idle":"2023-07-04T17:25:57.731686Z","shell.execute_reply.started":"2023-07-04T17:21:18.753943Z","shell.execute_reply":"2023-07-04T17:25:57.730524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}