{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":8931188,"sourceType":"datasetVersion","datasetId":5372786}],"dockerImageVersionId":30716,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Устанавливаем необходимые пакеты","metadata":{}},{"cell_type":"markdown","source":"# Считываем данные\n","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom tqdm import tqdm\nimport warnings\nimport numpy as np\nimport torch\n\nwarnings.filterwarnings('ignore')\n\ndata = pd.read_csv('/kaggle/input/unique-cpp/filtered_sequences_unique.csv')\nX = data['Sequence'].tolist()\ny = data['Label'].tolist()","metadata":{"execution":{"iopub.status.busy":"2024-07-12T08:11:33.472226Z","iopub.execute_input":"2024-07-12T08:11:33.47337Z","iopub.status.idle":"2024-07-12T08:11:33.496346Z","shell.execute_reply.started":"2024-07-12T08:11:33.473325Z","shell.execute_reply":"2024-07-12T08:11:33.495571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Разделение данных на обучающую и тестовую выборки\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-07-12T08:11:33.49802Z","iopub.execute_input":"2024-07-12T08:11:33.498349Z","iopub.status.idle":"2024-07-12T08:11:33.50459Z","shell.execute_reply.started":"2024-07-12T08:11:33.498322Z","shell.execute_reply":"2024-07-12T08:11:33.503557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Подключаем CUDA","metadata":{}},{"cell_type":"code","source":"import multiprocess as mp\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-07-12T08:11:33.505691Z","iopub.execute_input":"2024-07-12T08:11:33.506012Z","iopub.status.idle":"2024-07-12T08:11:33.51755Z","shell.execute_reply.started":"2024-07-12T08:11:33.505982Z","shell.execute_reply":"2024-07-12T08:11:33.516657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Используем эмбеддинги из Prot-Bert","metadata":{}},{"cell_type":"code","source":"from transformers import BertModel, BertTokenizer\n\ntokenizer = BertTokenizer.from_pretrained(\"Rostlab/prot_bert\", do_lower_case=False)\nbert = BertModel.from_pretrained(\"Rostlab/prot_bert\")\n\ndef mean_pooling(model_output, attention_mask):\n    token_embeddings = model_output[0] #First element of model_output contains all token embeddings\n    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n    res = torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n    return res.squeeze()\n\ndef get_embedding(protein):\n    with torch.inference_mode():\n        tokenized = tokenizer(\n            protein,\n            return_tensors=\"pt\",\n            truncation=True,\n            max_length=10 # 25\n        ) #.to(device)\n        return mean_pooling(bert(**tokenized), tokenized[\"attention_mask\"])","metadata":{"execution":{"iopub.status.busy":"2024-07-12T08:11:33.518582Z","iopub.execute_input":"2024-07-12T08:11:33.518828Z","iopub.status.idle":"2024-07-12T08:11:37.544895Z","shell.execute_reply.started":"2024-07-12T08:11:33.518806Z","shell.execute_reply":"2024-07-12T08:11:37.544088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset\n\nclass ProtDataset(Dataset):\n    def __init__(self, seqs, labels) -> None:\n        self.seqs = seqs\n        self.labels = labels\n\n    def __getitem__(self, index: int) -> tuple:\n        embedding = get_embedding(self.seqs[index])\n        label = self.labels[index]\n        return (embedding, label)\n\n    def __len__(self) -> int:\n        return len(self.labels)","metadata":{"execution":{"iopub.status.busy":"2024-07-12T08:11:37.547186Z","iopub.execute_input":"2024-07-12T08:11:37.547493Z","iopub.status.idle":"2024-07-12T08:11:37.553882Z","shell.execute_reply.started":"2024-07-12T08:11:37.547468Z","shell.execute_reply":"2024-07-12T08:11:37.552996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Гиперпараметры","metadata":{}},{"cell_type":"code","source":"batch_size = 128\nlearning_rate = 1e-3\nweight_decay = 1e-5\nnum_epochs = 11","metadata":{"execution":{"iopub.status.busy":"2024-07-12T08:11:37.554948Z","iopub.execute_input":"2024-07-12T08:11:37.555241Z","iopub.status.idle":"2024-07-12T08:11:37.56587Z","shell.execute_reply.started":"2024-07-12T08:11:37.555196Z","shell.execute_reply":"2024-07-12T08:11:37.565096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = ProtDataset(X_train, y_train)\ntrainloader = torch.utils.data.DataLoader(\n    train, batch_size=batch_size, shuffle=True, num_workers=4\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-12T08:11:37.566943Z","iopub.execute_input":"2024-07-12T08:11:37.56729Z","iopub.status.idle":"2024-07-12T08:11:37.578312Z","shell.execute_reply.started":"2024-07-12T08:11:37.567258Z","shell.execute_reply":"2024-07-12T08:11:37.577531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(trainloader)","metadata":{"execution":{"iopub.status.busy":"2024-07-12T08:11:37.579482Z","iopub.execute_input":"2024-07-12T08:11:37.579734Z","iopub.status.idle":"2024-07-12T08:11:37.591219Z","shell.execute_reply.started":"2024-07-12T08:11:37.579712Z","shell.execute_reply":"2024-07-12T08:11:37.590114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Нейронная сеть","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\n\nclass RNN(nn.Module):\n    def __init__(self, input_size=1024, output_size=1, hidden_size=32):\n        super(RNN, self).__init__()\n        \n        self.hidden_size = hidden_size\n        #self.conv = nn.Conv1d(in_channels=128, out_channels=128, kernel_size=3,  stride=1, padding=1)\n        self.rnn = nn.RNN(input_size, hidden_size, nonlinearity='relu')\n        self.relu = nn.ReLU()\n        #self.pool = nn.MaxPool1d(kernel_size=2, stride=2)\n        self.fc = nn.Linear(hidden_size, output_size)\n\n    def forward(self, x):\n        #output = self.conv(x)\n        output = self.relu(x)\n        #output = self.pool(output)\n        #print(output.shape)\n        \n        h0 = torch.zeros(1, x.size(0), self.hidden_size).to(device)\n        output, hidden = self.rnn(x.unsqueeze(0), h0)\n        #output = self.pool(output)\n        \n        output = self.fc(output[-1, :])\n        return output","metadata":{"execution":{"iopub.status.busy":"2024-07-12T09:59:03.87576Z","iopub.execute_input":"2024-07-12T09:59:03.876143Z","iopub.status.idle":"2024-07-12T09:59:03.884991Z","shell.execute_reply.started":"2024-07-12T09:59:03.876104Z","shell.execute_reply":"2024-07-12T09:59:03.883977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\n\nmodel = RNN().to(device)\nmodel.to(device)\noptimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\ncriterion = nn.MSELoss() # nn.MSELoss() # nn.CrossEntropyLoss()\nmodel.train()","metadata":{"execution":{"iopub.status.busy":"2024-07-12T09:59:05.998655Z","iopub.execute_input":"2024-07-12T09:59:05.999515Z","iopub.status.idle":"2024-07-12T09:59:06.011737Z","shell.execute_reply.started":"2024-07-12T09:59:05.99948Z","shell.execute_reply":"2024-07-12T09:59:06.010696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Обучение модели","metadata":{}},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    train_loss = 0.0\n    correct, total = 0, 0\n    for data, target in tqdm(trainloader):\n        print(data.shape)\n        data = data.to(device)\n        target = target.to(torch.float32).to(device)\n        optimizer.zero_grad()\n        output = model(data)\n        print(output.shape, target.shape)\n        loss = criterion(output, target)\n        loss.backward()\n        optimizer.step()\n        total += target.size(0)\n        _, predicted = torch.max(output.data, 1) # , 1\n        correct += (predicted == target).sum().item()\n        train_loss += loss.item() * data.size(0)\n        \n    train_loss = train_loss / len(trainloader.dataset)\n    acc = 100 * correct / total\n    print(f'Epoch: {epoch+1} \\tTraining Loss: {train_loss:.6f} \\t Training Acc: {acc:.3f}')","metadata":{"execution":{"iopub.status.busy":"2024-07-12T09:59:08.794856Z","iopub.execute_input":"2024-07-12T09:59:08.795904Z"},"trusted":true},"execution_count":null,"outputs":[]}]}