{"cells":[{"metadata":{},"cell_type":"markdown","source":"# NLP\n## Description\nThe following notebook takes a pretrained BERT model and trains a new model on top (a 2-layer neural network). For efficiency, we can simply compute the pretrained BERT model on the data first and then use the outputs of BERT (which are simply 1d tensors) as the inputs for our network. This leads to much faster training.\n\nThe motivation for using BERT was to transfer the performance of a strong NLP model to our problem of classifying the sentences in Government documents."},{"metadata":{},"cell_type":"markdown","source":"## BERT -> multilabel classification"},{"metadata":{"trusted":true},"cell_type":"code","source":"TOKEN_LENGTH = 512\nBATCH_SIZE = 16\nSEED = 42","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport torch\nimport pandas as pd\n\n# Check GPU availability\ndevice_name = tf.test.gpu_device_name()\nif device_name == '/device:GPU:0':\n    print('Found GPU at: {}'.format(device_name))\nelse:\n    print('No GPU found.')\n    \n\n# Initialise pytorch with GPU\nif torch.cuda.is_available():    \n    device = torch.device('cuda')\n    torch.cuda.empty_cache()\n    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n    print('We will use the GPU:', torch.cuda.get_device_name(0))\nelse:\n    print('No GPU available, using the CPU instead.')\n    device = torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fpath = '../input/cdp-30-35-54-translated-with-labels/bert_translations_01122020_1.csv'\ndf = pd.read_csv(fpath, header=0, encoding='ISO-8859-1')\nrequired_columns = [\n    'response_answer_translated',\n    'Employment',\n    'Inclusion',\n    'Health',\n    'Congestion'\n]\ndf = df[[c for c in required_columns if 'Unnamed' not in c]].dropna()\nlabel_cols = ['Employment', 'Inclusion', 'Health', 'Congestion']\nfor c in label_cols:\n    df[c] = df[c].astype(int)\ndf['label'] = df[label_cols].values.tolist()\nprint(f\"Loaded dataset with {df['label'].shape[0]} labelled examples.\")\ndf.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create test set\nfrom sklearn.model_selection import train_test_split\ndf, df_test = train_test_split(df, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nsequences = df['response_answer_translated'].values\nlabels = np.array([np.array(l, dtype=np.float16) for l in df['label'].values])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import BertTokenizer\n\n# Load the BERT tokenizer.\nprint('Loading BERT tokenizer...')\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\nmax_len = 0\nfor s in sequences:\n    input_ids = tokenizer.encode(s, add_special_tokens=True)\n    max_len = max(max_len, len(input_ids))  # keep track of longest seq\n\n\nprint('Max sequence length: ', max_len)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Tokenize all of the sentences and map the tokens to thier word IDs.\ninput_ids = []\nattention_masks = []\n\n# For every sentence...\nfor s in sequences:\n    # `Tokenize sentence, add special chars\n    encoded_dict = tokenizer.encode_plus(\n                        s,\n                        add_special_tokens = True,\n                        max_length = TOKEN_LENGTH,\n                        pad_to_max_length = True,\n                        return_attention_mask = True,\n                        return_tensors = 'pt',\n                   )\n    \n    input_ids.append(encoded_dict['input_ids'])\n    attention_masks.append(encoded_dict['attention_mask'])   # mask padding\n\n# Convert the lists into tensors.\ninput_ids = torch.cat(input_ids, dim=0)\nattention_masks = torch.cat(attention_masks, dim=0)\nlabels = torch.tensor(labels, dtype=torch.float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import TensorDataset, random_split\n\ndataset = TensorDataset(input_ids, attention_masks, labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First run BERT on the whole dataset, so that we get encodings for all of our inputs."},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n\nall_dataloader = DataLoader(\n    dataset,\n    sampler=SequentialSampler(dataset),\n    batch_size=BATCH_SIZE,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers.modeling_bert import BertPreTrainedModel, BertModel\nfrom torch.nn import BCEWithLogitsLoss\n\n# Modified from Kaushal Trivedi's multilabel example\n# https://medium.com/huggingface/multi-label-text-classification-using-bert-the-mighty-transformer-69714fa3fb3d\n# Source: https://github.com/kaushaltrivedi/bert-toxic-comments-multilabel/blob/master/toxic-bert-multilabel-classification.ipynb\nclass BertFromPreTrained(BertPreTrainedModel):\n    \"\"\"\n    BERT model for encoding.\n    \"\"\"\n    def __init__(self, config):\n        super().__init__(config)\n        self.bert = BertModel(config)\n\n    def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):\n        _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask)\n        \n        return pooled_output\n        \n    def freeze_bert_encoder(self):\n        for param in self.bert.parameters():\n            param.requires_grad = False\n    \n    def unfreeze_bert_encoder(self):\n        for param in self.bert.parameters():\n            param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_bert = BertFromPreTrained.from_pretrained(\n    'bert-base-uncased',  # Use the 12-layer BERT model, with an uncased vocab.\n    output_attentions=False,\n    output_hidden_states=False,\n)\n\npretrained_bert.to(device)\npretrained_bert.freeze_bert_encoder()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_encodings = []\nfor batch in all_dataloader:\n    b_input_ids = batch[0].to(device)\n    b_input_mask = batch[1].to(device)\n    b_labels = batch[2].to(device)\n\n    # Forward pass (predict)\n    with torch.no_grad():\n        latents = pretrained_bert(b_input_ids, None, b_input_mask, b_labels)\n    \n    bert_encodings.append(latents)\n        \nbert_encodings = torch.cat(bert_encodings, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import TensorDataset, random_split\n\nencoded_dataset = TensorDataset(bert_encodings, attention_masks, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Divide the dataset by randomly selecting samples.\ntrain_size = int(0.8 * len(dataset))\nval_size = len(dataset) - train_size\ntrain_dataset, val_dataset = random_split(encoded_dataset, [train_size, val_size], generator=torch.Generator().manual_seed(0))\nprint('{:>5,} training samples'.format(train_size))\nprint('{:>5,} validation samples'.format(val_size))\n\n# Create data loaders to feed information to the GPU\ntrain_dataloader = DataLoader(\n            train_dataset,\n            sampler=RandomSampler(train_dataset),  # random example to batch selection\n            batch_size=BATCH_SIZE \n        )\n\n# Train-val set\nseq_train_dataloader = DataLoader(\n            train_dataset,\n            sampler=SequentialSampler(train_dataset), \n            batch_size=BATCH_SIZE \n        )\n\nvalidation_dataloader = DataLoader(\n            val_dataset,\n            sampler=SequentialSampler(val_dataset),\n            batch_size=BATCH_SIZE\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\nUSE_WANDB = False\n\n# if USE_WANDB:\n#     wandb.login()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\nfrom sklearn.datasets import make_multilabel_classification\nfrom torch import optim\n\nclass Network(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.dropout = torch.nn.Dropout(0.1)\n        self.layer1 = torch.nn.Linear(768, 32)\n        # self.layer2 = torch.nn.Linear(32, 32)\n        self.logits = torch.nn.Linear(32, 4)\n        # self.relu = torch.nn.ReLU\n        \n    def forward(self, x):\n        x = self.dropout(x)\n        x = self.layer1(x)\n        x = F.relu(x)\n        #x = self.layer2(x)\n        #x = F.relu(x)\n        return self.logits(x)\n\n\nmodel = Network()\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from transformers import get_linear_schedule_with_warmup\nfrom transformers import AdamW\n\n# Set up optimiser to manage modification of model weights\n# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n# lr in (5e-5, 3e-5, 2e-5)\nlr = 1e-3\neps = 1e-8\nepochs = 800\n\noptimizer = AdamW(model.parameters(), lr=lr, eps=eps)\n\ntotal_steps = len(train_dataloader) * epochs  # batches * epochs\nscheduler = get_linear_schedule_with_warmup(\n    optimizer, num_warmup_steps=0, num_training_steps=total_steps\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # 1. Start a new run\ntag = 5\n\n# if USE_WANDB:\n#     wandb.init(project=\"cdp-abc\", tags=list(str(tag)))\n    \n#     # 2. Save model inputs and hyperparameters\n#     config = wandb.config\n#     config.learning_rate = lr\n#     config.eps = eps\n\n#     # 3. Log gradients and model parameters\n#     wandb.watch(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport time\nimport datetime\n\n\ndef flat_accuracy(preds, labels):\n    return np.mean(preds == labels)\n    \n\ndef loss_fn(outputs, targets):\n    return torch.nn.BCEWithLogitsLoss()(outputs, targets)\n\n\n# Not yet implemented\n# def f_beta(y_pred: Tensor, y_true: Tensor, thresh: float = 0.2, \n#            beta: float = 2, eps: float = 1e-9, sigmoid: bool = True):\n#     beta2 = beta ** 2\n#     if sigmoid: y_pred = y_pred.sigmoid()\n#     y_pred = (y_pred>thresh).float()\n#     y_true = y_true.float()\n#     tp = (y_pred*y_true).sum(dim=1)\n#     prec = TP/(y_pred.sum(dim=1)+eps)\n#     rec = TP/(y_true.sum(dim=1)+eps)\n#     res = (prec*rec)/(prec*beta2+rec+eps)*(1+beta2)\n#     return res.mean().item()\n\n\ndef format_time(elapsed):\n    \"\"\" Takes a time in seconds and returns a string hh:mm:ss. \"\"\"\n    elapsed_rounded = int(round((elapsed)))\n    return str(datetime.timedelta(seconds=elapsed_rounded))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Load model\n# tag = 3\n# load_model = False\n# if load_model:\n#     chosen_tag = 2\n#     chosen_epoch = 150\n#     fpath = f'./model_{chosen_tag}_{chosen_epoch}'\n#     print(f'Model fpath: {fpath}')\n#     model.load_state_dict(torch.load(fpath))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn.functional as F\nfrom sklearn.metrics import roc_auc_score\n\ndef validation(model, dataloader, criterion, threshold=0.5):\n    print('Running Validation...')\n    t0 = time.time()\n\n    model.eval()  # evaluation mode, change dropout behaviour\n\n    total_eval_accuracy = 0\n    total_eval_loss = 0\n    nb_eval_steps = 0\n\n    tp = 0\n    fp = 0\n    fn = 0\n    preds_l = []\n    labels_l = []\n    for batch in dataloader:  #validation_dataloader:\n        b_x = batch[0].to(device)\n        b_input_mask = batch[1].to(device)\n        b_labels = batch[2].to(device)\n\n        # Forward pass (predict)\n        with torch.no_grad():\n            logits = model(b_x)\n            \n        loss = criterion(logits, b_labels)\n\n        preds = F.sigmoid(logits).cpu().numpy()\n        preds_l.append(preds)\n        logits = logits.detach().cpu().numpy()  # move logits and labels to CPU\n        label_ids = b_labels.to('cpu').numpy()\n        labels_l.append(label_ids)\n        \n        preds = (preds > threshold).astype(float)\n        tp += np.sum((label_ids == 1) * (preds == 1)) \n        fp += np.sum((label_ids == 0) * (preds == 1))\n        fn += np.sum((label_ids == 1) * (preds == 0))\n\n        total_eval_loss += loss.item()\n        total_eval_accuracy += flat_accuracy(preds, label_ids)\n\n    all_preds = np.concatenate(preds_l, axis=0)\n    all_labels = np.concatenate(labels_l, axis=0)\n    auc = roc_auc_score(y_true=all_labels.ravel(), y_score=all_preds.ravel())\n        \n    recall = tp / (tp + fn)\n    precision = tp / (tp + fp)\n    avg_val_accuracy = total_eval_accuracy / len(dataloader)\n    avg_val_loss = total_eval_loss / len(validation_dataloader)\n    validation_time = format_time(time.time() - t0)\n    print('  Accuracy: {0:.2f}'.format(avg_val_accuracy))\n    print('  Validation Loss: {0:.2f}'.format(avg_val_loss))\n    print('  Validation took: {:}'.format(validation_time))\n    return {\n        'ava': avg_val_accuracy, \n        'avl': avg_val_loss, \n        'vt': validation_time, \n        'preds': preds, \n        'recall': recall, \n        'precision': precision,\n        'auc': auc,\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nimport numpy as np\n\n# This training code is based on the `run_glue.py` script here:\n# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\n\n\ntraining_stats = []\ntotal_t0 = time.time()\n\ncriterion = torch.nn.BCEWithLogitsLoss()\n\nfor epoch_i in range(0, epochs):\n    \n    # TRAINING\n    print('\\n======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n    print('Training...')\n    t0 = time.time()\n    total_train_accuracy = 0\n    total_train_loss = 0\n\n    model.train()  # training mode, change dropout behaviour\n    \n    for step, batch in enumerate(train_dataloader):\n        if step % 25 == 0 and not step == 0:\n            elapsed = format_time(time.time() - t0)\n            msg = '  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'\n            print(msg.format(step, len(train_dataloader), elapsed))\n\n        b_input_ids = batch[0].to(device)  # copy batch to GPU\n        b_input_mask = batch[1].to(device)\n        b_labels = batch[2].to(device)\n\n        model.zero_grad()  # clear gradients before backwards pass     \n\n        # Forward pass (train)\n        b_token_type_ids = None\n        logits = model(b_input_ids)\n        loss = criterion(logits, b_labels)\n        \n        total_train_loss += loss.item()  # collect training loss\n        \n        _logits = logits.detach().cpu().numpy()\n        _label_ids = b_labels.to('cpu').numpy()\n        total_train_accuracy += flat_accuracy(_logits, _label_ids)\n\n        # Backward pass (calculate gradients)\n        loss.backward()\n\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)  # prevent exploding grad\n        optimizer.step()  # update parameters, take step using gradient\n        scheduler.step()  # update learning rate\n\n    avg_train_accuracy = total_train_accuracy / len(train_dataloader)\n    avg_train_loss = total_train_loss / len(train_dataloader)            \n    training_time = format_time(time.time() - t0)\n    print('  Average training loss: {0:.2f}'.format(avg_train_loss))\n    print('  Training epcoh took: {:}'.format(training_time))\n        \n    # VALIDATION\n    train_stat = validation(model, seq_train_dataloader, criterion)\n    val_stat = validation(model, validation_dataloader, criterion)\n    \n    wandb_stats = {\n        'loss_trn': train_stat['avl'],\n        'loss_val': val_stat['avl'],\n        'accuracy_trn': train_stat['ava'],\n        'accuracy_val': val_stat['ava'],\n        'recall_trn': train_stat['recall'],\n        'recall_val': val_stat['recall'],\n        'precision_trn': train_stat['precision'],\n        'precision_val': val_stat['precision'],\n        'auc_trn': train_stat['auc'],\n        'auc_val': val_stat['auc'],\n    }\n    \n    training_stats.append({\n        'epoch': epoch_i + 1,\n        'Training Time': training_time,\n        'validation_time_trn': train_stat['vt'],\n        'validation_time_val': val_stat['vt'],\n        **wandb_stats,\n    })\n    \n    if epoch_i % 50 == 0:\n        torch.save(model.state_dict(), f'model_{tag}_{epoch_i}')\n        \n#     # Log to WandB\n#     try:\n#         if USE_WANDB:\n#             wandb.log(wandb_stats)\n#     except Exception as e:\n#         print(e)\n\nprint('Training took {:} (h:mm:ss)'.format(format_time(time.time() - total_t0)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(2, 2, figsize=(20, 10))\n\nax = axs[0][0]\nax.plot([stats['auc_val'] for stats in training_stats], label=f\"{training_stats[-1]['auc_val']:.02f}\")\nax.grid()\nax.legend()\nax.set_title('Validation AUC')\n\nax = axs[0][1]\nax.plot([stats['precision_val'] for stats in training_stats], label=f\"{training_stats[-1]['precision_val']:.02f}\")\nax.grid()\nax.legend()\nax.set_title('Validation Precision')\n\nax = axs[1][0]\nax.plot([stats['recall_val'] for stats in training_stats], label=f\"{training_stats[-1]['recall_val']:.02f}\")\nax.grid()\nax.legend()\nax.set_title('Validation Recall')\n\nax = axs[1][1]\nax.plot([stats['accuracy_val'] for stats in training_stats], label=f\"{training_stats[-1]['accuracy_val']:.02f}\")\nax.grid()\nax.legend()\nax.set_title('Validation Accuracy')\npass","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We are able to achieve AUC on the validation set of 0.83, precision of 0.68, accuracy of 0.85 and recall of 0.49. A random model would have obtained precision of about 20%, since the labels in our validation set are only about 20% 1s."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# criterion = torch.nn.BCEWithLogitsLoss()\n\n# acc_l = []\n# loss_l = []\n# for epoch in range(2000):\n#     optimizer.zero_grad()\n#     logits = network(x)\n#     loss = criterion(logits, y)\n#     loss.backward()\n#     optimizer.step()\n#     acc_l.append(\n#         flat_accuracy(\n#             (F.sigmoid(logits) > 0.5).cpu().numpy(), \n#             y.cpu().numpy()\n#         )\n#     )\n#     # print('Loss: {:.3f}'.format(loss.item()))\n#     loss_l.append(loss.item())\n    \n# plt.plot(loss_l)\n# plt.plot(acc_l)\n# acc_l[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import pandas as pd\n\n# df_stats = pd.DataFrame(data=training_stats)\n# df_stats = df_stats.set_index('epoch')\n# df_stats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Learning curve\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n\n# sns.set(style='darkgrid')\n# plt.rcParams['figure.figsize'] = (12, 6)\n\n# alpha = 0.6\n# plt.plot(df_stats['Training Loss'], 'b-o', label='Training loss', alpha=alpha)\n# plt.plot(df_stats['Valid. Loss'], 'g-o', label='Validation loss', alpha=alpha)\n# plt.plot(df_stats['train_accuracy'], 'r-o', label='Training accuracy', alpha=alpha)\n# plt.plot(df_stats['validation_accuracy'], 'y-o', label='Validation accuracy', alpha=alpha)\n# plt.title('Loss + accuracy')\n# plt.xlabel('Epoch')\n# plt.ylabel('Loss')\n# plt.legend()\n# xs = []\n# for e in range(epochs):\n#     if e % 50 == 0:\n#         xs.append(e)\n# plt.xticks(xs)\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import pandas as pd\n\n# # df_test = df_cheeky_test.copy()\n# print('Number of test sequences: {:,}\\n'.format(df_test.shape[0]))\n\n# sequences = df_test['response_answer_translated'].values\n# labels = np.array([np.array(l, dtype=np.float16) for l in df_test['label'].values])\n\n# input_ids = []\n# attention_masks = []\n# for s in sequences:\n#     encoded_dict = tokenizer.encode_plus(\n#         s,\n#         add_special_tokens = True,\n#         max_length = TOKEN_LENGTH,\n#         pad_to_max_length = True,\n#         return_attention_mask = True,\n#         return_tensors = 'pt',\n#     )\n     \n#     input_ids.append(encoded_dict['input_ids'])\n#     attention_masks.append(encoded_dict['attention_mask'])\n\n\n# input_ids = torch.cat(input_ids, dim=0)\n# attention_masks = torch.cat(attention_masks, dim=0)\n# labels = torch.tensor(labels)\n\n# prediction_data = TensorDataset(input_ids, attention_masks, labels)\n# prediction_sampler = SequentialSampler(prediction_data)\n# prediction_dataloader = DataLoader(\n#     prediction_data, sampler=prediction_sampler, batch_size=BATCH_SIZE\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Prediction on test set\n# print('Predicting labels for {} test sequences...'.format(len(input_ids)))\n\n# model.eval()\n\n# predictions = [] \n# true_labels = []\n# for batch in prediction_dataloader:\n#     batch = tuple(t.to(device) for t in batch)\n#     b_input_ids, b_input_mask, b_labels = batch\n    \n#     # Forward pass, predict\n#     with torch.no_grad():\n#         b_token_type_ids = None\n#         outputs = model(b_input_ids, b_token_type_ids, b_input_mask)  # no labels\n#         m = torch.nn.Sigmoid()\n#         outputs = m(outputs)\n\n#     logits = outputs.detach().cpu().numpy()\n#     label_ids = b_labels.to('cpu').numpy()\n#     predictions.append(logits)\n#     true_labels.append(label_ids)\n\n# predictions = np.vstack(predictions)\n# true_labels = np.vstack(true_labels)\n# print('Completed predictions.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def precision_recall(preds, labels):\n#     tp = np.sum((preds == 1) * (labels == 1))\n#     fp = np.sum((preds == 1) * (labels == 0))\n#     fn = np.sum((preds == 0) * (labels == 1))\n#     precision = tp / (tp + fp)\n#     recall = tp / (tp + fn)\n#     return precision, recall\n\n\n# xs = np.linspace(0, 1, 11)\n# prec = [precision_recall((predictions > x), true_labels)[0] for x in xs]\n# recs = [precision_recall((predictions > x), true_labels)[1] for x in xs]\n# plt.plot(xs, prec, label='precision')\n# plt.plot(xs, recs, label='recall')\n# plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.metrics import multilabel_confusion_matrix, roc_auc_score\n\n# rocauc_micro = roc_auc_score(true_labels, predictions, average='micro')\n# print('ROC-AUC: {:.3f}\\n'.format(rocauc_micro))\n# print('Confusion matrices:')\n# print(multilabel_confusion_matrix(true_labels, (predictions > 0.5)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# threshold = 0.5\n# output_col_names = ['employment', 'inclusion', 'health', 'congestion']\n# df_preds = pd.DataFrame(predictions, columns=output_col_names)\n# _df_test = pd.concat([df_test.reset_index(), df_preds], axis=1)\n# _df_test[['employment', 'inclusion', 'health', 'congestion']] = (\n#     _df_test[['employment', 'inclusion', 'health', 'congestion']].apply(lambda x: (x > threshold).astype(float))\n# )\n# _df_test['label_preds'] = _df_test[['employment', 'inclusion', 'health', 'congestion']].apply(list, axis=1)\n# _df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Miscellaneous"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import torch\n# import torch.nn.functional as F\n# from sklearn.datasets import make_multilabel_classification\n# from torch import optim\n\n# class Network(torch.nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.layer1 = torch.nn.Linear(768, 32)\n#         # self.layer2 = torch.nn.Linear(32, 32)\n#         self.logits = torch.nn.Linear(32, 4)\n#         self.dropout = torch.nn.Dropout(0.1)\n#         # self.relu = torch.nn.ReLU\n        \n#     def forward(self, x):\n#         x = self.dropout(x)\n#         x = self.layer1(x)\n#         x = F.relu(x)\n#         #x = self.layer2(x)\n#         #x = F.relu(x)\n#         return self.logits(x)\n\n    \n# network = Network()\n# network.to(device)\n\n# # x = torch.randn(16, 4).cuda()\n# x = model.forward(next(iter(train_dataloader))[0].cuda())\n# _, y = make_multilabel_classification(n_samples=16, n_features=4, n_classes=4, n_labels=2)\n# # y = torch.Tensor(y).cuda()\n# y = torch.Tensor(labels).cuda()\n\n# network.train()\n\n# optimizer = optim.AdamW(network.parameters(), lr=1e-3)\n\n# acc_l = []\n# loss_l = []\n# for epoch in range(2000):\n#     optimizer.zero_grad()\n#     logits = network(x)\n#     loss = criterion(logits, y)\n#     loss.backward()\n#     optimizer.step()\n#     acc_l.append(\n#         flat_accuracy(\n#             (F.sigmoid(logits) > 0.5).cpu().numpy(), \n#             y.cpu().numpy()\n#         )\n#     )\n#     # print('Loss: {:.3f}'.format(loss.item()))\n#     loss_l.append(loss.item())\n    \n# plt.plot(loss_l)\n# plt.plot(acc_l)\n# acc_l[-1]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}