{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Log Classification using BERT","metadata":{"id":"EKOTlwcmxmej"}},{"cell_type":"code","source":"import tensorflow as tf\n# Checking for the GPU\ndevice_name = tf.test.gpu_device_name()\nprint(device_name)","metadata":{"id":"DEfSbAA4QHas","outputId":"8431a054-3835-4c33-a465-3b63542a903f","execution":{"iopub.status.busy":"2022-12-13T01:08:29.107030Z","iopub.execute_input":"2022-12-13T01:08:29.107572Z","iopub.status.idle":"2022-12-13T01:08:37.411283Z","shell.execute_reply.started":"2022-12-13T01:08:29.107456Z","shell.execute_reply":"2022-12-13T01:08:37.410253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ndevice = torch.device(\"cuda\")","metadata":{"id":"oYsV4H8fCpZ-","execution":{"iopub.status.busy":"2022-12-13T01:08:37.412743Z","iopub.execute_input":"2022-12-13T01:08:37.413233Z","iopub.status.idle":"2022-12-13T01:08:39.026385Z","shell.execute_reply.started":"2022-12-13T01:08:37.413198Z","shell.execute_reply":"2022-12-13T01:08:39.025432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:08:39.027782Z","iopub.execute_input":"2022-12-13T01:08:39.028137Z","iopub.status.idle":"2022-12-13T01:08:39.033898Z","shell.execute_reply.started":"2022-12-13T01:08:39.028101Z","shell.execute_reply":"2022-12-13T01:08:39.032630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install transformers","metadata":{"id":"0NmMdkZO8R6q","outputId":"09e7d809-20ee-40d1-c45b-9c569992aed0","execution":{"iopub.status.busy":"2022-12-13T01:08:39.036601Z","iopub.execute_input":"2022-12-13T01:08:39.037491Z","iopub.status.idle":"2022-12-13T01:08:49.763680Z","shell.execute_reply.started":"2022-12-13T01:08:39.037437Z","shell.execute_reply":"2022-12-13T01:08:49.762537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def log_preprocess(logs):\n    logs = logs.str.lower()\n    logs = logs.replace(r'\\S*\\d\\S*','',regex=True).str.strip()\n    return logs","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:08:49.765623Z","iopub.execute_input":"2022-12-13T01:08:49.766527Z","iopub.status.idle":"2022-12-13T01:08:49.772891Z","shell.execute_reply.started":"2022-12-13T01:08:49.766469Z","shell.execute_reply":"2022-12-13T01:08:49.771957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the dataset into a pandas dataframe.\ndf = pd.read_csv(\"/kaggle/input/convolve-train/train_logs.csv\",names=['Log','Label'])\ndf_test = pd.read_csv('/kaggle/input/convolve-train/test.csv')\ndf.drop_duplicates(inplace=True)\n# Report the number of sentences.\nprint('Number of training sentences: {:,}\\n'.format(df.shape[0]))\nprint('Number of test sentences: {:,}\\n'.format(df_test.shape[0]))\n\n# Display 10 random rows from the data.\ndf.sample(10)","metadata":{"id":"_UkeC7SG2krJ","outputId":"e20220fa-7ba5-4f59-d84d-772e95f18eff","execution":{"iopub.status.busy":"2022-12-13T01:08:49.774449Z","iopub.execute_input":"2022-12-13T01:08:49.774807Z","iopub.status.idle":"2022-12-13T01:09:07.412124Z","shell.execute_reply.started":"2022-12-13T01:08:49.774774Z","shell.execute_reply":"2022-12-13T01:09:07.410985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.sample()","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:07.413749Z","iopub.execute_input":"2022-12-13T01:09:07.415056Z","iopub.status.idle":"2022-12-13T01:09:07.438709Z","shell.execute_reply.started":"2022-12-13T01:09:07.415017Z","shell.execute_reply":"2022-12-13T01:09:07.437660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.iloc[20,:]","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:07.440104Z","iopub.execute_input":"2022-12-13T01:09:07.440548Z","iopub.status.idle":"2022-12-13T01:09:07.448688Z","shell.execute_reply.started":"2022-12-13T01:09:07.440512Z","shell.execute_reply":"2022-12-13T01:09:07.447470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.Label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:07.451120Z","iopub.execute_input":"2022-12-13T01:09:07.451380Z","iopub.status.idle":"2022-12-13T01:09:07.462081Z","shell.execute_reply.started":"2022-12-13T01:09:07.451355Z","shell.execute_reply":"2022-12-13T01:09:07.460910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the lists of sentences and their labels.\nlogs = df.Log.values[1:]\nlabels = df.Label.values[1:]\ntest_logs = log_preprocess(df_test[' Log'].str.split(n=5,expand=True)[5])\ntest_logs = test_logs.values","metadata":{"id":"GuE5BqICAne2","execution":{"iopub.status.busy":"2022-12-13T01:09:07.466863Z","iopub.execute_input":"2022-12-13T01:09:07.467128Z","iopub.status.idle":"2022-12-13T01:09:13.045527Z","shell.execute_reply.started":"2022-12-13T01:09:07.467088Z","shell.execute_reply":"2022-12-13T01:09:13.044424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_logs[0])\nprint(len(test_logs))","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:13.047033Z","iopub.execute_input":"2022-12-13T01:09:13.047412Z","iopub.status.idle":"2022-12-13T01:09:13.055096Z","shell.execute_reply.started":"2022-12-13T01:09:13.047376Z","shell.execute_reply":"2022-12-13T01:09:13.054148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import BertTokenizer\n\n# Load the BERT tokenizer.\n\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)","metadata":{"id":"Z474sSC6oe7A","outputId":"5cac4edc-a347-4d50-99d6-9dd1f4fd99d2","execution":{"iopub.status.busy":"2022-12-13T01:09:13.056342Z","iopub.execute_input":"2022-12-13T01:09:13.057308Z","iopub.status.idle":"2022-12-13T01:09:15.740099Z","shell.execute_reply.started":"2022-12-13T01:09:13.057263Z","shell.execute_reply":"2022-12-13T01:09:15.737045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print the original sentence.\nprint(' Original: ', logs[0])\n\n# Print the sentence split into tokens.\nprint('Tokenized: ', tokenizer.tokenize(logs[0]))\n\n# Print the sentence mapped to token ids.\nprint('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(logs[0])))","metadata":{"id":"dLIbudgfh6F0","outputId":"73326d52-d489-4e06-b706-99e6a5f7d57a","execution":{"iopub.status.busy":"2022-12-13T01:09:15.741617Z","iopub.status.idle":"2022-12-13T01:09:15.742415Z","shell.execute_reply.started":"2022-12-13T01:09:15.742149Z","shell.execute_reply":"2022-12-13T01:09:15.742175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.743937Z","iopub.status.idle":"2022-12-13T01:09:15.744701Z","shell.execute_reply.started":"2022-12-13T01:09:15.744435Z","shell.execute_reply":"2022-12-13T01:09:15.744460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logs[19]","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.746098Z","iopub.status.idle":"2022-12-13T01:09:15.746908Z","shell.execute_reply.started":"2022-12-13T01:09:15.746635Z","shell.execute_reply":"2022-12-13T01:09:15.746660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tokenize all of the sentences and map the tokens to thier word IDs.\ninput_ids = []\n\n# For every sentence...\nfor log in tqdm(logs):\n    # `encode` will:\n    #   (1) Tokenize the sentence.\n    #   (2) Prepend the `[CLS]` token to the start.\n    #   (3) Append the `[SEP]` token to the end.\n    #   (4) Map tokens to their IDs.\n    encoded_log = tokenizer.encode(\n                        log\n                   )\n    \n    # Add the encoded sentence to the list.\n    input_ids.append(encoded_log)\n\n# Print sentence 0, now as a list of IDs.\nprint('Original: ', logs[0])\nprint('Token IDs:', input_ids[0])","metadata":{"id":"2bBdb3pt8LuQ","outputId":"043462be-49c9-4a91-92c7-268acf61a6d3","execution":{"iopub.status.busy":"2022-12-13T01:09:15.748309Z","iopub.status.idle":"2022-12-13T01:09:15.749121Z","shell.execute_reply.started":"2022-12-13T01:09:15.748861Z","shell.execute_reply":"2022-12-13T01:09:15.748886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Same for test\ntest_input_ids = []\n\n# For every sentence...\nfor log in tqdm(test_logs):\n    # `encode` will:\n    #   (1) Tokenize the sentence.\n    #   (2) Prepend the `[CLS]` token to the start.\n    #   (3) Append the `[SEP]` token to the end.\n    #   (4) Map tokens to their IDs.\n    encoded_log = tokenizer.encode(\n                        log\n                   )\n    \n    # Add the encoded sentence to the list.\n    test_input_ids.append(encoded_log)","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.750548Z","iopub.status.idle":"2022-12-13T01:09:15.751316Z","shell.execute_reply.started":"2022-12-13T01:09:15.751058Z","shell.execute_reply":"2022-12-13T01:09:15.751083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Max sentence length: ', max([len(sen) for sen in test_input_ids]))","metadata":{"id":"JhUZO9vc_l6T","outputId":"20d43d20-5b2c-4c51-f3b1-dd59f20646da","execution":{"iopub.status.busy":"2022-12-13T01:09:15.752753Z","iopub.status.idle":"2022-12-13T01:09:15.753570Z","shell.execute_reply.started":"2022-12-13T01:09:15.753297Z","shell.execute_reply":"2022-12-13T01:09:15.753322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We will use some utility function from tensorflow\nfrom keras.preprocessing.sequence import pad_sequences\n\nMAX_LEN = 128\n\n#Padding the input to the max length\ninput_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", \n                          value=0, truncating=\"post\", padding=\"post\")\ntest_input_ids = pad_sequences(test_input_ids, maxlen=MAX_LEN, dtype=\"long\", \n                          value=0, truncating=\"post\", padding=\"post\")","metadata":{"id":"Cp9BPRd1tMIo","execution":{"iopub.status.busy":"2022-12-13T01:09:15.754973Z","iopub.status.idle":"2022-12-13T01:09:15.755752Z","shell.execute_reply.started":"2022-12-13T01:09:15.755502Z","shell.execute_reply":"2022-12-13T01:09:15.755526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating the attention masks\nattention_masks = []\n\n# For each sentence...\nfor sent in input_ids:\n    \n    # Create the attention mask.\n    #   - If a token ID is 0, then it's padding, set the mask to 0.\n    #   - If a token ID is > 0, then it's a real token, set the mask to 1.\n    att_mask = [int(token_id > 0) for token_id in sent]\n    \n    # Store the attention mask for this sentence.\n    attention_masks.append(att_mask)","metadata":{"id":"cDoC24LeEv3N","execution":{"iopub.status.busy":"2022-12-13T01:09:15.757162Z","iopub.status.idle":"2022-12-13T01:09:15.757958Z","shell.execute_reply.started":"2022-12-13T01:09:15.757688Z","shell.execute_reply":"2022-12-13T01:09:15.757713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For test\n# Creating the attention masks\ntest_attention_masks = []\n\n# For each sentence...\nfor sent in test_input_ids:\n    \n    # Create the attention mask.\n    #   - If a token ID is 0, then it's padding, set the mask to 0.\n    #   - If a token ID is > 0, then it's a real token, set the mask to 1.\n    test_att_mask = [int(token_id > 0) for token_id in sent]\n    \n    # Store the attention mask for this sentence.\n    test_attention_masks.append(att_mask)","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.759412Z","iopub.status.idle":"2022-12-13T01:09:15.760249Z","shell.execute_reply.started":"2022-12-13T01:09:15.759962Z","shell.execute_reply":"2022-12-13T01:09:15.759987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = [1 if x == \"normal\" else 0 for x in labels]","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.761661Z","iopub.status.idle":"2022-12-13T01:09:15.762531Z","shell.execute_reply.started":"2022-12-13T01:09:15.762236Z","shell.execute_reply":"2022-12-13T01:09:15.762263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We will call the train_test_split() function from sklearn\nfrom sklearn.model_selection import train_test_split\n\n\ntrain_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n                                                            random_state=2018, test_size=0.1, stratify=labels)\n# Performing same steps on the attention masks\ntrain_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,\n                                             random_state=2018, test_size=0.1, stratify=labels)","metadata":{"id":"aFbE-UHvsb7-","execution":{"iopub.status.busy":"2022-12-13T01:09:15.763985Z","iopub.status.idle":"2022-12-13T01:09:15.764761Z","shell.execute_reply.started":"2022-12-13T01:09:15.764498Z","shell.execute_reply":"2022-12-13T01:09:15.764523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Converting the input data to the tensor , which can be feeded to the model\ntrain_inputs = torch.tensor(train_inputs)\nvalidation_inputs = torch.tensor(validation_inputs)\n\ntrain_labels = torch.tensor(train_labels)\nvalidation_labels = torch.tensor(validation_labels)\n\ntrain_masks = torch.tensor(train_masks)\nvalidation_masks = torch.tensor(validation_masks)\n\ntest_inputs = torch.tensor(test_input_ids)\n\ntest_masks = torch.tensor(test_attention_masks)","metadata":{"id":"jw5K2A5Ko1RF","execution":{"iopub.status.busy":"2022-12-13T01:09:15.766508Z","iopub.status.idle":"2022-12-13T01:09:15.767278Z","shell.execute_reply.started":"2022-12-13T01:09:15.767021Z","shell.execute_reply":"2022-12-13T01:09:15.767046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import TensorDataset, DataLoader\n#Creating the DataLoader which will help us to load data into the GPU/CPU\nbatch_size = 32\n\n# Create the DataLoader for our training set.\ntrain_data = TensorDataset(train_inputs, train_masks, train_labels)\ntrain_dataloader = DataLoader(train_data, batch_size=batch_size)\n# Create the DataLoader for our validation set.\nvalidation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\nvalidation_dataloader = DataLoader(validation_data,batch_size=batch_size)\n#For Test \ntest_data = TensorDataset(test_inputs, test_masks)\ntest_dataloader = DataLoader(test_data, batch_size=batch_size*2)","metadata":{"id":"GEgLpFVlo1Z-","execution":{"iopub.status.busy":"2022-12-13T01:09:15.768734Z","iopub.status.idle":"2022-12-13T01:09:15.769544Z","shell.execute_reply.started":"2022-12-13T01:09:15.769267Z","shell.execute_reply":"2022-12-13T01:09:15.769292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Loading the pre-trained BERT model from huggingface library\n\nfrom transformers import BertForSequenceClassification, AdamW, BertConfig\n\n# Load BertForSequenceClassification, the pretrained BERT model with a single \n# linear classification layer on top. \nmodel = BertForSequenceClassification.from_pretrained(\n    \"bert-base-uncased\", \n    num_labels = 2,   \n    output_attentions = False, \n    output_hidden_states = False, )\n\n# Teeling the model to run on GPU\nmodel.cuda()","metadata":{"id":"gFsCTp_mporB","outputId":"60b3ee08-16e0-464a-f784-684df32628c3","execution":{"iopub.status.busy":"2022-12-13T01:09:15.770936Z","iopub.status.idle":"2022-12-13T01:09:15.771722Z","shell.execute_reply.started":"2022-12-13T01:09:15.771453Z","shell.execute_reply":"2022-12-13T01:09:15.771492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# AdamW is an optimizer which is a Adam Optimzier with weight-decay-fix\noptimizer = AdamW(model.parameters(),\n                  lr = 2e-5, \n                  eps = 1e-8 \n                )\n","metadata":{"id":"GLs72DuMODJO","execution":{"iopub.status.busy":"2022-12-13T01:09:15.773130Z","iopub.status.idle":"2022-12-13T01:09:15.773920Z","shell.execute_reply.started":"2022-12-13T01:09:15.773652Z","shell.execute_reply":"2022-12-13T01:09:15.773677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#defining a loss with weights for class imbalance\nloss_weights = [80.0,1.0]\nloss_weights = torch.FloatTensor(loss_weights).cuda()\ncriterion = torch.nn.CrossEntropyLoss(weight = loss_weights)","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.775747Z","iopub.status.idle":"2022-12-13T01:09:15.776537Z","shell.execute_reply.started":"2022-12-13T01:09:15.776259Z","shell.execute_reply":"2022-12-13T01:09:15.776283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import get_linear_schedule_with_warmup\n\n# Number of training epochs\nepochs = 10\n\n# Total number of training steps is number of batches * number of epochs.\ntotal_steps = len(train_dataloader) * epochs\n\n# Create the learning rate scheduler.\nscheduler = get_linear_schedule_with_warmup(optimizer, \n                                            num_warmup_steps = 0, # Default value in run_glue.py\n                                            num_training_steps = total_steps)\nscheduler","metadata":{"id":"-p0upAhhRiIx","outputId":"3a075bc4-4570-4f37-b8f8-1d31a91d32a9","execution":{"iopub.status.busy":"2022-12-13T01:09:15.777965Z","iopub.status.idle":"2022-12-13T01:09:15.778753Z","shell.execute_reply.started":"2022-12-13T01:09:15.778502Z","shell.execute_reply":"2022-12-13T01:09:15.778528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define a helper function for calculating accuracy.","metadata":{"id":"pE5B99H5H2-W"}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.metrics import f1_score\n# Function to calculate the accuracy,f1 of our predictions vs labels\ndef flat_accuracy(preds, labels):\n    pred_flat = np.argmax(preds, axis=1).flatten()\n    labels_flat = labels.flatten()\n    return np.sum(pred_flat == labels_flat) / len(labels_flat)\ndef flat_f1_score(preds,labels):\n    return f1_score(labels.flatten(),np.argmax(preds,axis=1).flatten())","metadata":{"id":"9cQNvaZ9bnyy","execution":{"iopub.status.busy":"2022-12-13T01:09:15.780158Z","iopub.status.idle":"2022-12-13T01:09:15.781007Z","shell.execute_reply.started":"2022-12-13T01:09:15.780692Z","shell.execute_reply":"2022-12-13T01:09:15.780717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Creating the helper function to have a watch on elapsed time\n\nimport time\nimport datetime\n\ndef format_time(elapsed):\n    '''\n    Takes a time in seconds and returns a string hh:mm:ss\n    '''\n    # Round to the nearest second.\n    elapsed_rounded = int(round((elapsed)))\n    \n    # Format as hh:mm:ss\n    return str(datetime.timedelta(seconds=elapsed_rounded))\n","metadata":{"id":"gpt6tR83keZD","execution":{"iopub.status.busy":"2022-12-13T01:09:15.782565Z","iopub.status.idle":"2022-12-13T01:09:15.783345Z","shell.execute_reply.started":"2022-12-13T01:09:15.783084Z","shell.execute_reply":"2022-12-13T01:09:15.783109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's start the training process\n\nimport random\n\n# This training code is based on the `run_glue.py` script here:\n# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n\n\n# Set the seed value all over the place to make this reproducible.\nseed_val = 42\n\nrandom.seed(seed_val)\nnp.random.seed(seed_val)\ntorch.manual_seed(seed_val)\ntorch.cuda.manual_seed_all(seed_val)\n\n# Store the average loss after each epoch so we can plot them.\nloss_values = []\n\n# For each epoch...\nfor epoch_i in range(0, epochs):\n    \n    # ========================================\n    #               Training\n    # ========================================\n    \n    # Perform one full pass over the training set.\n\n    print(\"\")\n    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n    print('Training...')\n\n    # Measure how long the training epoch takes.\n    t0 = time.time()\n\n    # Reset the total loss for this epoch.\n    total_loss = 0\n\n    # Put the model into training mode. Don't be mislead--the call to \n    # `train` just changes the *mode*, it doesn't *perform* the training.\n    # `dropout` and `batchnorm` layers behave differently during training\n    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n    model.train()\n\n    # For each batch of training data...\n    for step, batch in enumerate(train_dataloader):\n\n        # Progress update every 40 batches.\n        if step % 40 == 0 and not step == 0:\n            # Calculate elapsed time in minutes.\n            elapsed = format_time(time.time() - t0)\n            \n            # Report progress.\n            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n\n        # Unpack this training batch from our dataloader. \n        #\n        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n        # `to` method.\n        #\n        # `batch` contains three pytorch tensors:\n        #   [0]: input ids \n        #   [1]: attention masks\n        #   [2]: labels \n        b_input_ids = batch[0].to(device)\n        b_input_mask = batch[1].to(device)\n        b_labels = batch[2].to(device)\n\n        # Always clear any previously calculated gradients before performing a\n        # backward pass. PyTorch doesn't do this automatically because \n        # accumulating the gradients is \"convenient while training RNNs\". \n        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n        model.zero_grad()        \n\n        # Perform a forward pass (evaluate the model on this training batch).\n        # This will return the loss (rather than the model output) because we\n        # have provided the `labels`.\n        # The documentation for this `model` function is here: \n        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n        outputs = model(b_input_ids, \n                    token_type_ids=None, \n                    attention_mask=b_input_mask, \n                    labels=b_labels)\n        \n        # The call to `model` always returns a tuple, so we need to pull the \n        # loss value out of the tuple.\n        # loss = outputs[0]\n        #Using custom loss\n        loss = criterion(outputs[1], b_labels) # 1 for logits as we pass labels\n\n\n        # Accumulate the training loss over all of the batches so that we can\n        # calculate the average loss at the end. `loss` is a Tensor containing a\n        # single value; the `.item()` function just returns the Python value \n        # from the tensor.\n        total_loss += loss.item()\n\n        # Perform a backward pass to calculate the gradients.\n        loss.backward()\n\n        # Clip the norm of the gradients to 1.0.\n        # This is to help prevent the \"exploding gradients\" problem.\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n\n        # Update parameters and take a step using the computed gradient.\n        # The optimizer dictates the \"update rule\"--how the parameters are\n        # modified based on their gradients, the learning rate, etc.\n        optimizer.step()\n\n        # Update the learning rate.\n        scheduler.step()\n\n    # Calculate the average loss over the training data.\n    avg_train_loss = total_loss / len(train_dataloader)            \n    \n    # Store the loss value for plotting the learning curve.\n    loss_values.append(avg_train_loss)\n\n    print(\"\")\n    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n    print(\"  Training epoch took: {:}\".format(format_time(time.time() - t0)))\n        \n    # ========================================\n    #               Validation\n    # ========================================\n    # After the completion of each training epoch, measure our performance on\n    # our validation set.\n\n    print(\"\")\n    print(\"Running Validation...\")\n\n    t0 = time.time()\n\n    # Put the model in evaluation mode--the dropout layers behave differently\n    # during evaluation.\n    model.eval()\n\n    # Tracking variables \n    eval_loss, eval_accuracy = 0, 0\n    nb_eval_steps, nb_eval_examples = 0, 0\n    eval_f1 = 0\n\n    # Evaluate data for one epoch\n    for batch in validation_dataloader:\n        \n        # Add batch to GPU\n        batch = tuple(t.to(device) for t in batch)\n        \n        # Unpack the inputs from our dataloader\n        b_input_ids, b_input_mask, b_labels = batch\n        \n        # Telling the model not to compute or store gradients, saving memory and\n        # speeding up validation\n        with torch.no_grad():        \n\n            # Forward pass, calculate logit predictions.\n            # This will return the logits rather than the loss because we have\n            # not provided labels.\n            # token_type_ids is the same as the \"segment ids\", which \n            # differentiates sentence 1 and 2 in 2-sentence tasks.\n            # The documentation for this `model` function is here: \n            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n            outputs = model(b_input_ids, \n                            token_type_ids=None, \n                            attention_mask=b_input_mask)\n        \n        # Get the \"logits\" output by the model. The \"logits\" are the output\n        # values prior to applying an activation function like the softmax.\n        logits = outputs[0]\n\n        # Move logits and labels to CPU\n        logits = logits.detach().cpu().numpy()\n        label_ids = b_labels.to('cpu').numpy()\n        \n        # Calculate the accuracy for this batch of test sentences.\n        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n        tmp_eval_f1 = flat_f1_score(logits, label_ids)\n        \n        # Accumulate the total accuracy.\n        eval_accuracy += tmp_eval_accuracy\n        eval_f1 += tmp_eval_f1\n        \n\n        # Track the number of batches\n        nb_eval_steps += 1\n\n    # Report the final accuracy for this validation run.\n    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n    print(\"  F1: {0:.2f}\".format(eval_f1/nb_eval_steps))\n\n    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n\nprint(\"\")\nprint(\"Training complete!\")","metadata":{"id":"6J-FYdx6nFE_","outputId":"40adc853-1e64-49ef-a340-7c033918903a","execution":{"iopub.status.busy":"2022-12-13T01:09:15.784972Z","iopub.status.idle":"2022-12-13T01:09:15.785904Z","shell.execute_reply.started":"2022-12-13T01:09:15.785634Z","shell.execute_reply":"2022-12-13T01:09:15.785658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(input_ids)","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.787338Z","iopub.status.idle":"2022-12-13T01:09:15.788160Z","shell.execute_reply.started":"2022-12-13T01:09:15.787900Z","shell.execute_reply":"2022-12-13T01:09:15.787926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Evaluating our model on the test set\n\n# Prediction on test set\n\nprint('Predicting labels for {:,} test sentences...'.format(len(test_inputs)))\npredictions = []\n\n# Put model in evaluation mode\nmodel.eval()\n\n# Predict \nfor batch in tqdm(test_dataloader):\n  # Add batch to GPU\n  batch = tuple(t.to(device) for t in batch)\n  \n  # Unpack the inputs from our dataloader\n  b_input_ids, b_input_mask = batch\n  \n  # Telling the model not to compute or store gradients, saving memory and \n  # speeding up prediction\n  with torch.no_grad():\n      # Forward pass, calculate logit predictions\n      outputs = model(b_input_ids, token_type_ids=None, \n                      attention_mask=b_input_mask)\n\n  logits = outputs[0]\n\n  # Move logits and labels to CPU\n  logits = logits.detach().cpu().numpy()\n  \n  # Store predictions\n  predictions.append(logits)","metadata":{"id":"Hba10sXR7Xi6","outputId":"8c65c0d5-4498-44a4-e025-3b29b4e86923","execution":{"iopub.status.busy":"2022-12-13T01:09:15.789594Z","iopub.status.idle":"2022-12-13T01:09:15.790365Z","shell.execute_reply.started":"2022-12-13T01:09:15.790104Z","shell.execute_reply":"2022-12-13T01:09:15.790129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(predictions)","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.791793Z","iopub.status.idle":"2022-12-13T01:09:15.792583Z","shell.execute_reply.started":"2022-12-13T01:09:15.792309Z","shell.execute_reply":"2022-12-13T01:09:15.792333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.794004Z","iopub.status.idle":"2022-12-13T01:09:15.794785Z","shell.execute_reply.started":"2022-12-13T01:09:15.794532Z","shell.execute_reply":"2022-12-13T01:09:15.794558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nfor batch in tqdm(predictions):\n    for instance in batch:\n        preds.append(np.argmax(instance))","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-12-13T01:09:15.796185Z","iopub.status.idle":"2022-12-13T01:09:15.796973Z","shell.execute_reply.started":"2022-12-13T01:09:15.796702Z","shell.execute_reply":"2022-12-13T01:09:15.796727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set(preds)","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.798369Z","iopub.status.idle":"2022-12-13T01:09:15.799183Z","shell.execute_reply.started":"2022-12-13T01:09:15.798925Z","shell.execute_reply":"2022-12-13T01:09:15.798952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame({\"ID\":df_test.ID, \" Label\":[\"normal\" if x==1 else \"anomaly\" for x in preds]})\nsubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-13T01:09:15.800619Z","iopub.status.idle":"2022-12-13T01:09:15.801401Z","shell.execute_reply.started":"2022-12-13T01:09:15.801141Z","shell.execute_reply":"2022-12-13T01:09:15.801165Z"},"trusted":true},"execution_count":null,"outputs":[]}]}