{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# General information\nThis notebook is used for the course: Machine Learning in Practice. This specific notebook is used in combination with \"Without internet - BERT NER - MLiP 28\". As the competition requires a notebook that does not use internet, this notebook was split from the ohter notebook to make sure we could use internet for the parts that needed internet. \n\nThis notebook is adapted from the notebook by Tung M. Phung: https://www.kaggle.com/tungmphung/coleridge-matching-bert-ner\n\nIf something is added or changed by our team it is stated in or above that code cell.\n\n","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.\n\nMAX_LENGTH = 64 #max no. words for each sentence\nOVERLAP = 20 #if a sentence exeeds MAX_LENGTH, we split it into multiple sentences with overlapping","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:25:43.820132Z","iopub.execute_input":"2021-06-14T09:25:43.820478Z","iopub.status.idle":"2021-06-14T09:25:43.825895Z","shell.execute_reply.started":"2021-06-14T09:25:43.820403Z","shell.execute_reply":"2021-06-14T09:25:43.824871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:25:43.868707Z","iopub.execute_input":"2021-06-14T09:25:43.868967Z","iopub.status.idle":"2021-06-14T09:26:12.414992Z","shell.execute_reply.started":"2021-06-14T09:25:43.868942Z","shell.execute_reply":"2021-06-14T09:26:12.414032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)\n\n# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-14T09:26:12.419089Z","iopub.execute_input":"2021-06-14T09:26:12.419378Z","iopub.status.idle":"2021-06-14T09:26:13.970507Z","shell.execute_reply.started":"2021-06-14T09:26:12.419348Z","shell.execute_reply":"2021-06-14T09:26:13.969551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:26:13.974090Z","iopub.execute_input":"2021-06-14T09:26:13.974356Z","iopub.status.idle":"2021-06-14T09:27:04.927514Z","shell.execute_reply.started":"2021-06-14T09:26:13.974329Z","shell.execute_reply":"2021-06-14T09:27:04.926594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For the literal matching later on we need a knowledge bank. This is created here since\n# we now create our validation set in the same loop that we process the train dataset and\n# in the Coleridge:Matching notebook this was done before grouping the training labels\n\nall_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:27:04.928772Z","iopub.execute_input":"2021-06-14T09:27:04.929103Z","iopub.status.idle":"2021-06-14T09:27:04.987123Z","shell.execute_reply.started":"2021-06-14T09:27:04.929068Z","shell.execute_reply":"2021-06-14T09:27:04.986292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#group by publication, training labels should have the same form as expected output.\ntrain = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')\n\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:27:04.990665Z","iopub.execute_input":"2021-06-14T09:27:04.990927Z","iopub.status.idle":"2021-06-14T09:27:13.290970Z","shell.execute_reply.started":"2021-06-14T09:27:04.990902Z","shell.execute_reply":"2021-06-14T09:27:13.290177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create training and validation sets\nHere we create the training and validation sets based on the original train data. To do this we start with defining the original functions and then adapt the for loop of the train data to assign the last 30% of","metadata":{}},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, list(zip(sentence_words, nes))\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:27:13.294948Z","iopub.execute_input":"2021-06-14T09:27:13.295210Z","iopub.status.idle":"2021-06-14T09:27:13.308169Z","shell.execute_reply.started":"2021-06-14T09:27:13.295184Z","shell.execute_reply":"2021-06-14T09:27:13.307300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"create validation set","metadata":{}},{"cell_type":"code","source":"#Changed by our team for evaluation purposes\n\n# Data is already shuffled in previous cell, so we just take first part for training and second part for validation\ntrain_val_split = 0.7 # 70% as training data\ntrain_val_split = int(train_val_split * len(train))\ntraining_set = True\ntest_rows = []\nliteral_preds = [] # this is for the test set\npaper_length = [] # also for test set\n\ncnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nner_data = []\npbar = tqdm(total=train_val_split)\n\niter = 0\nfor i, id, dataset_label in train[['Id', 'dataset_label']].itertuples():\n    if iter == train_val_split:\n        training_set = False\n         \n    # paper\n    paper = papers[id]\n    \n    # labels\n    labels = dataset_label.split('|')\n    labels = [clean_training_text(label) for label in labels]\n\n    # sentences\n    sentences = set([clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.') \n                ])\n    \n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    \n    if training_set:\n        for sentence in sentences:\n            is_positive, tags = tag_sentence(sentence, labels)\n            if is_positive:\n                cnt_pos += 1\n                ner_data.append(tags)\n            elif any(word in sentence.lower() for word in ['data', 'study']): \n                ner_data.append(tags)\n                cnt_neg += 1\n\n        # process bar\n        pbar.update(1)\n        pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\n            \n    else: # we're creating the validation set using the same logic as for the test set in the other notebook\n        sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n        for sentence in sentences:\n            sentence_words = sentence.split()\n            dummy_tags = ['O']*len(sentence_words)\n            test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n            \n        # this is copied from the Coleride:Matching notebook and is used for matching on test data  \n        text_1 = '. '.join(section['text'] for section in paper).lower()\n        text_2 = totally_clean_text(text_1)\n\n        labels = set()\n        for label in all_labels:\n            if label in text_1 or label in text_2:\n                labels.add(clean_text(label))\n        literal_preds.append('|'.join(labels))\n        \n        paper_length.append(len(sentences))\n        # until here    \n        \n    iter = iter+1\n #shuffling\nrandom.shuffle(ner_data)\n    \nwith open('train_ner_validation_version.json', 'w') as f:\n    for row in ner_data:\n        words, nes = list(zip(*row))\n        row_json = {'tokens' : words, 'tags' : nes}\n        json.dump(row_json, f)\n        f.write('\\n')\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:27:13.309612Z","iopub.execute_input":"2021-06-14T09:27:13.310248Z","iopub.status.idle":"2021-06-14T09:31:02.492728Z","shell.execute_reply.started":"2021-06-14T09:27:13.310207Z","shell.execute_reply":"2021-06-14T09:31:02.491937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Added by our team for evaluation purposes\n\n# to illustrate\nprint(ner_data[6])\nprint(test_rows[0])\nprint(\"\\n\")\n\nn_train = len(ner_data)\nn_val = len(test_rows)\nprint('n_train:', n_train)\nprint('n_val:', n_val)\n\nper_train = (100/(n_train+n_val))*n_train\nprint(f\"train/val split: {per_train}/{100-per_train}\")\nprint(\"\\n\")\n\nprint(literal_preds[:5])","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:02.494376Z","iopub.execute_input":"2021-06-14T09:31:02.494806Z","iopub.status.idle":"2021-06-14T09:31:02.503272Z","shell.execute_reply.started":"2021-06-14T09:31:02.494762Z","shell.execute_reply":"2021-06-14T09:31:02.501068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:02.504558Z","iopub.execute_input":"2021-06-14T09:31:02.504959Z","iopub.status.idle":"2021-06-14T09:31:02.554651Z","shell.execute_reply.started":"2021-06-14T09:31:02.504919Z","shell.execute_reply":"2021-06-14T09:31:02.553973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:02.555811Z","iopub.execute_input":"2021-06-14T09:31:02.556139Z","iopub.status.idle":"2021-06-14T09:31:02.602769Z","shell.execute_reply.started":"2021-06-14T09:31:02.556101Z","shell.execute_reply":"2021-06-14T09:31:02.601880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"literal_preds = []\n\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if label in text_1 or label in text_2:\n            labels.add(clean_text(label))\n    \n    literal_preds.append('|'.join(labels))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:02.603967Z","iopub.execute_input":"2021-06-14T09:31:02.604305Z","iopub.status.idle":"2021-06-14T09:31:02.801906Z","shell.execute_reply.started":"2021-06-14T09:31:02.604271Z","shell.execute_reply":"2021-06-14T09:31:02.801061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:02.803024Z","iopub.execute_input":"2021-06-14T09:31:02.803379Z","iopub.status.idle":"2021-06-14T09:31:02.812033Z","shell.execute_reply.started":"2021-06-14T09:31:02.803342Z","shell.execute_reply":"2021-06-14T09:31:02.810973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\n#Changed by our team since we wanted to try different BERT based models.\n#BERT:\n#PRETRAINED_PATH = '../input/coleridge-bert-models/output'\n#SciBERT: \n#PRETRAINED_PATH = 'allenai/scibert_scivocab_cased'\n#PRETRAINED_PATH = '../input/scibert/output'\n#BioBERT:\nPRETRAINED_PATH = 'dmis-lab/biobert-base-cased-v1.1'\n\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = './train_ner_validation_version.json'\nVAL_PATH = './train_ner_validation_version.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:02.813748Z","iopub.execute_input":"2021-06-14T09:31:02.814582Z","iopub.status.idle":"2021-06-14T09:31:02.820374Z","shell.execute_reply.started":"2021-06-14T09:31:02.814542Z","shell.execute_reply":"2021-06-14T09:31:02.819711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:02.821926Z","iopub.execute_input":"2021-06-14T09:31:02.822743Z","iopub.status.idle":"2021-06-14T09:31:03.171908Z","shell.execute_reply.started":"2021-06-14T09:31:02.822708Z","shell.execute_reply":"2021-06-14T09:31:03.170990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in sample_submission['Id']:\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:03.173188Z","iopub.execute_input":"2021-06-14T09:31:03.173725Z","iopub.status.idle":"2021-06-14T09:31:03.359680Z","shell.execute_reply.started":"2021-06-14T09:31:03.173686Z","shell.execute_reply":"2021-06-14T09:31:03.358798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:03.361270Z","iopub.execute_input":"2021-06-14T09:31:03.361634Z","iopub.status.idle":"2021-06-14T09:31:03.366765Z","shell.execute_reply.started":"2021-06-14T09:31:03.361597Z","shell.execute_reply":"2021-06-14T09:31:03.365797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:03.368221Z","iopub.execute_input":"2021-06-14T09:31:03.368908Z","iopub.status.idle":"2021-06-14T09:31:04.068751Z","shell.execute_reply.started":"2021-06-14T09:31:03.368856Z","shell.execute_reply":"2021-06-14T09:31:04.067808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:04.070263Z","iopub.execute_input":"2021-06-14T09:31:04.070596Z","iopub.status.idle":"2021-06-14T09:31:04.079262Z","shell.execute_reply.started":"2021-06-14T09:31:04.070557Z","shell.execute_reply":"2021-06-14T09:31:04.078447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:31:04.080601Z","iopub.execute_input":"2021-06-14T09:31:04.081119Z","iopub.status.idle":"2021-06-14T09:32:08.039331Z","shell.execute_reply.started":"2021-06-14T09:31:04.081084Z","shell.execute_reply":"2021-06-14T09:32:08.038404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:32:08.040727Z","iopub.execute_input":"2021-06-14T09:32:08.041069Z","iopub.status.idle":"2021-06-14T09:32:08.048858Z","shell.execute_reply.started":"2021-06-14T09:32:08.041031Z","shell.execute_reply":"2021-06-14T09:32:08.046188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# store all dataset labels for each publication by looping over each paper and\n# adding the phrases which were predicted as labels\n\nbert_dataset_labels = []\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:32:08.050427Z","iopub.execute_input":"2021-06-14T09:32:08.050900Z","iopub.status.idle":"2021-06-14T09:32:08.063831Z","shell.execute_reply.started":"2021-06-14T09:32:08.050862Z","shell.execute_reply":"2021-06-14T09:32:08.062755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:32:08.065114Z","iopub.execute_input":"2021-06-14T09:32:08.065508Z","iopub.status.idle":"2021-06-14T09:32:08.073990Z","shell.execute_reply.started":"2021-06-14T09:32:08.065468Z","shell.execute_reply":"2021-06-14T09:32:08.073149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the following code filters out any duplicate predicted labels (assuming the similarity is large)\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n    \n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:32:08.077110Z","iopub.execute_input":"2021-06-14T09:32:08.077357Z","iopub.status.idle":"2021-06-14T09:32:08.503915Z","shell.execute_reply.started":"2021-06-14T09:32:08.077334Z","shell.execute_reply":"2021-06-14T09:32:08.502969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Added by our team for evaluation purposes\n# the following code zips the true labels and the predicted labels of each paper together\n# to determine the confusion matrix.\n# We will not keep track of TN, as this consists of all non-dataset words\n\nduplicates = []\ntp = [] # label and predicted\n#tn = [] # remainder\nfp = [] # predicted but no label match\nfn = [] # label but no predicted\n\nfor labels, preds in zip(literal_preds, filtered_bert_labels):\n    labels = labels.split('|')\n    preds = preds.split('|')\n    matched_preds = []\n    dup = []\n    \n    for label in labels:\n        best_pred = ('', 0)\n        for pred in preds:\n            j_sim = jaccard_similarity(label, pred)\n            if j_sim >= 0.5 and j_sim > best_pred[1]:\n                best_pred = (pred, j_sim)\n        if best_pred[0]:\n            if best_pred[0] in matched_preds:\n                dup.append(best_pred[0])\n            else:\n                matched_preds.append(best_pred[0])\n                tp.append(best_pred[0])\n        else:\n            fn.append(label)\n            \n    for pred in preds:\n        if pred not in matched_preds:\n            fp.append(pred)\n    if dup:\n        duplicates.append(dup)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:32:08.505157Z","iopub.execute_input":"2021-06-14T09:32:08.505483Z","iopub.status.idle":"2021-06-14T09:32:08.522460Z","shell.execute_reply.started":"2021-06-14T09:32:08.505447Z","shell.execute_reply":"2021-06-14T09:32:08.521639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Added by our team for evaluation purposes\nprint('true positives:', len(tp))\nprint('false positives:',len(fp))\nprint('false_negatives:',len(fn))\nprint('\\n')\n\nprint('duplicate true labels:', len(duplicates))\nprint(duplicates[:5])","metadata":{"execution":{"iopub.status.busy":"2021-06-14T09:32:08.523627Z","iopub.execute_input":"2021-06-14T09:32:08.524014Z","iopub.status.idle":"2021-06-14T09:32:08.530913Z","shell.execute_reply.started":"2021-06-14T09:32:08.523978Z","shell.execute_reply":"2021-06-14T09:32:08.529763Z"},"trusted":true},"execution_count":null,"outputs":[]}]}