{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:06.982470Z","iopub.execute_input":"2021-06-14T08:23:06.982831Z","iopub.status.idle":"2021-06-14T08:23:06.987865Z","shell.execute_reply.started":"2021-06-14T08:23:06.982796Z","shell.execute_reply":"2021-06-14T08:23:06.986996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:06.989643Z","iopub.execute_input":"2021-06-14T08:23:06.990353Z","iopub.status.idle":"2021-06-14T08:23:29.885029Z","shell.execute_reply.started":"2021-06-14T08:23:06.990290Z","shell.execute_reply":"2021-06-14T08:23:29.883860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-14T08:23:29.887112Z","iopub.execute_input":"2021-06-14T08:23:29.887463Z","iopub.status.idle":"2021-06-14T08:23:29.903909Z","shell.execute_reply.started":"2021-06-14T08:23:29.887427Z","shell.execute_reply":"2021-06-14T08:23:29.902785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"# read train data\nfull_df_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ndf = pd.read_csv(full_df_path)\ndf = df[:MAX_SAMPLE]\n\n# train papers added to papers (200)\npaper_df_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in df['Id'].unique():\n    with open(f'{paper_df_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:29.908970Z","iopub.execute_input":"2021-06-14T08:23:29.910994Z","iopub.status.idle":"2021-06-14T08:23:39.157910Z","shell.execute_reply.started":"2021-06-14T08:23:29.910840Z","shell.execute_reply":"2021-06-14T08:23:39.157105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the input train data to train and validation set based on Id\nCOVID_papers = df[df['dataset_title'] == \"Our World in Data COVID-19 dataset\"]\nCOVID_papers_IDS = set(COVID_papers['Id'])\ndf_train_set = df[~df['Id'].isin(COVID_papers_IDS)]\ndf_in_val_set = df[df['Id'].isin(COVID_papers_IDS)]\n\ntrain, validate = train_test_split(df_train_set['Id'].unique(), test_size=0.01, random_state=42)\n# print(\"Train Shape : \", train.shape)\n\nvalidate = np.append(df_in_val_set[\"Id\"].to_numpy(), validate) # add Id's with Our World in Data COVID-19 dataset to the validation set\nprint(\"Validate Shape : \", validate.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.159164Z","iopub.execute_input":"2021-06-14T08:23:39.159498Z","iopub.status.idle":"2021-06-14T08:23:39.184137Z","shell.execute_reply.started":"2021-06-14T08:23:39.159465Z","shell.execute_reply":"2021-06-14T08:23:39.182932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train papers with titles\n# train_ground_truth = df[df['Id'].isin(train)]\n# validation papers with titles\nvalidate_ground_truth = df[df['Id'].isin(validate)]\n\n# train papers\n# train_papers = {your_key: papers[your_key] for your_key in train}\n# validate papers\nvalidate_papers = {your_key: papers[your_key] for your_key in validate}\n\n#print(train_ground_truth.shape)\nprint(validate_ground_truth.shape)\n#print(len(train_papers))\nprint(len(validate_papers))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.185521Z","iopub.execute_input":"2021-06-14T08:23:39.185881Z","iopub.status.idle":"2021-06-14T08:23:39.199777Z","shell.execute_reply.started":"2021-06-14T08:23:39.185845Z","shell.execute_reply":"2021-06-14T08:23:39.198586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.201381Z","iopub.execute_input":"2021-06-14T08:23:39.201754Z","iopub.status.idle":"2021-06-14T08:23:39.211672Z","shell.execute_reply.started":"2021-06-14T08:23:39.201718Z","shell.execute_reply":"2021-06-14T08:23:39.210734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Known labels for literal matching\n\n# all_labels = set()\n\n# for label_1, label_2, label_3 in train_ground_truth[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n#     all_labels.add(str(label_1).lower())\n#     all_labels.add(str(label_2).lower())\n#     all_labels.add(str(label_3).lower())\n    \n# print(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.213116Z","iopub.execute_input":"2021-06-14T08:23:39.213482Z","iopub.status.idle":"2021-06-14T08:23:39.219351Z","shell.execute_reply.started":"2021-06-14T08:23:39.213447Z","shell.execute_reply":"2021-06-14T08:23:39.218352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#For literal matching\n\n# literal_preds = []\n\n# for paper_id in sample_submission['Id']:\n#     paper = train_papers[paper_id]\n#     text_1 = '. '.join(section['text'] for section in paper).lower()\n#     text_2 = totally_clean_text(text_1)\n    \n#     labels = set()\n#     for label in all_labels:\n#         if label in text_1 or label in text_2:\n#             labels.add(clean_text(label))\n    \n#     literal_preds.append('|'.join(labels))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.220994Z","iopub.execute_input":"2021-06-14T08:23:39.221450Z","iopub.status.idle":"2021-06-14T08:23:39.228338Z","shell.execute_reply.started":"2021-06-14T08:23:39.221413Z","shell.execute_reply":"2021-06-14T08:23:39.227613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\nPRETRAINED_PATH = '../input/bert-label-frequency-equal/output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = '../input/bert-label-frequency-equal/train_ner.json'\nVAL_PATH = '../input/bert-label-frequency-equal/train_ner.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.231339Z","iopub.execute_input":"2021-06-14T08:23:39.231610Z","iopub.status.idle":"2021-06-14T08:23:39.238743Z","shell.execute_reply.started":"2021-06-14T08:23:39.231585Z","shell.execute_reply":"2021-06-14T08:23:39.238078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"# train = train_ground_truth.groupby('Id').agg({\n#     'pub_title': 'first',\n#     'dataset_title': '|'.join,\n#     'dataset_label': '|'.join,\n#     'cleaned_label': '|'.join\n# }).reset_index()\n\n# print(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.240412Z","iopub.execute_input":"2021-06-14T08:23:39.240668Z","iopub.status.idle":"2021-06-14T08:23:39.248023Z","shell.execute_reply.started":"2021-06-14T08:23:39.240627Z","shell.execute_reply":"2021-06-14T08:23:39.247107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n        \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    labels = sorted(labels, key=len, reverse=True)\n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, sentence_words, nes\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, sentence_words, nes","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.252305Z","iopub.execute_input":"2021-06-14T08:23:39.252706Z","iopub.status.idle":"2021-06-14T08:23:39.265898Z","shell.execute_reply.started":"2021-06-14T08:23:39.252671Z","shell.execute_reply":"2021-06-14T08:23:39.265145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\n#for paper_id in sample_submission['Id']:\nfor paper_id in validate_ground_truth['Id']:\n    sentences = []\n    # load paper\n    # paper = train_papers[paper_id]\n    paper = validate_papers[paper_id]\n    \n    # extract sentences\n    sentences_all = [clean_training_text(sentence) for section in paper    # split the sentences\n            for sentence in section['text'].split('.') \n            ]\n    \n    sentences_short = shorten_sentences(sentences_all)\n    sentences = [sentence for sentence in sentences_short if len(sentence) > 10]    #keep the sentences with at least 10 characters\n\n#     for i in range(0, len(sentences_lengths)):\n#         if len(re.findall('([A-Z][a-z]+)', sentences_lengths[i])) > 3:\n#             sentences.append(sentences_lengths[i])\n        \n    # collect all sentences in json\n    labels = validate_ground_truth[validate_ground_truth['Id']==paper_id]['dataset_label']\n    labels = [clean_training_text(label) for label in labels]\n#     print(labels)\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        pos, sentence_words, tags = tag_sentence(sentence, labels)\n        test_rows.append({'tokens' : sentence_words, 'tags' : tags})\n        \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:39.267989Z","iopub.execute_input":"2021-06-14T08:23:39.268343Z","iopub.status.idle":"2021-06-14T08:23:43.342216Z","shell.execute_reply.started":"2021-06-14T08:23:39.268311Z","shell.execute_reply":"2021-06-14T08:23:43.341271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:43.343469Z","iopub.execute_input":"2021-06-14T08:23:43.343977Z","iopub.status.idle":"2021-06-14T08:23:43.348999Z","shell.execute_reply.started":"2021-06-14T08:23:43.343941Z","shell.execute_reply":"2021-06-14T08:23:43.348153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:43.350272Z","iopub.execute_input":"2021-06-14T08:23:43.350753Z","iopub.status.idle":"2021-06-14T08:23:44.014073Z","shell.execute_reply.started":"2021-06-14T08:23:43.350719Z","shell.execute_reply":"2021-06-14T08:23:44.013113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$TRAIN_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:44.015746Z","iopub.execute_input":"2021-06-14T08:23:44.016082Z","iopub.status.idle":"2021-06-14T08:23:44.022068Z","shell.execute_reply.started":"2021-06-14T08:23:44.016046Z","shell.execute_reply":"2021-06-14T08:23:44.020745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:23:44.023573Z","iopub.execute_input":"2021-06-14T08:23:44.024235Z","iopub.status.idle":"2021-06-14T08:36:11.043963Z","shell.execute_reply.started":"2021-06-14T08:23:44.024198Z","shell.execute_reply":"2021-06-14T08:36:11.043098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.045470Z","iopub.execute_input":"2021-06-14T08:36:11.045841Z","iopub.status.idle":"2021-06-14T08:36:11.108759Z","shell.execute_reply.started":"2021-06-14T08:36:11.045805Z","shell.execute_reply":"2021-06-14T08:36:11.108013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                #print(word)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.111922Z","iopub.execute_input":"2021-06-14T08:36:11.112195Z","iopub.status.idle":"2021-06-14T08:36:11.780812Z","shell.execute_reply.started":"2021-06-14T08:36:11.112169Z","shell.execute_reply":"2021-06-14T08:36:11.779993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n    \n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.782268Z","iopub.execute_input":"2021-06-14T08:36:11.782619Z","iopub.status.idle":"2021-06-14T08:36:11.792467Z","shell.execute_reply.started":"2021-06-14T08:36:11.782581Z","shell.execute_reply":"2021-06-14T08:36:11.791262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"final_predictions = []\nfor bert_pred in filtered_bert_labels:\n    #if literal_match:\n    #    final_predictions.append(literal_match)\n    #else:\n    final_predictions.append(bert_pred)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.793961Z","iopub.execute_input":"2021-06-14T08:36:11.794509Z","iopub.status.idle":"2021-06-14T08:36:11.801990Z","shell.execute_reply.started":"2021-06-14T08:36:11.794453Z","shell.execute_reply":"2021-06-14T08:36:11.800830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#if you want to submit to the competition\n\n# sample_submission['PredictionString'] = final_predictions\n# sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.803218Z","iopub.execute_input":"2021-06-14T08:36:11.803734Z","iopub.status.idle":"2021-06-14T08:36:11.812391Z","shell.execute_reply.started":"2021-06-14T08:36:11.803702Z","shell.execute_reply":"2021-06-14T08:36:11.811466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.813735Z","iopub.execute_input":"2021-06-14T08:36:11.814106Z","iopub.status.idle":"2021-06-14T08:36:11.820338Z","shell.execute_reply.started":"2021-06-14T08:36:11.814070Z","shell.execute_reply":"2021-06-14T08:36:11.819543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation","metadata":{}},{"cell_type":"code","source":"validate_ground_truth['pred'] = filtered_bert_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.821930Z","iopub.execute_input":"2021-06-14T08:36:11.822286Z","iopub.status.idle":"2021-06-14T08:36:11.835925Z","shell.execute_reply.started":"2021-06-14T08:36:11.822254Z","shell.execute_reply":"2021-06-14T08:36:11.835203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.839024Z","iopub.execute_input":"2021-06-14T08:36:11.839303Z","iopub.status.idle":"2021-06-14T08:36:11.844768Z","shell.execute_reply.started":"2021-06-14T08:36:11.839279Z","shell.execute_reply":"2021-06-14T08:36:11.843697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_precision_recall(tp, fp, fn):\n    precision = tp / (tp+fp)\n    recall = tp / (tp + fn)\n    return precision, recall\n\ndef fbeta_score(precision, recall, beta):\n    fbeta = (1+(beta*beta))*((precision*recall)/( (beta*beta*precision) + recall))\n    return fbeta","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.846254Z","iopub.execute_input":"2021-06-14T08:36:11.846990Z","iopub.status.idle":"2021-06-14T08:36:11.853729Z","shell.execute_reply.started":"2021-06-14T08:36:11.846952Z","shell.execute_reply":"2021-06-14T08:36:11.852811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def coleridge_initiative_jaccard(ground_truth, prediction, verbose=True):\n    gts = ground_truth.split('|')\n    if len(prediction) > 0:\n        pds = sorted(prediction.split('|'))\n    else:\n        pds = []\n    if verbose:\n        print(\"Ground truth : \" , gts)\n        print(\"Prediction : \", pds)\n        \n    js_scores = []\n    cf_matrix = []\n    \n    #### Counting True Positives (TP) and False Positives (FP)\n    \n    for pd in pds:\n        score = -1\n        for gt in gts:\n            js = jaccard(pd, gt)\n            if js > score:\n                score = js\n        if score >= 0.5:\n            js_scores.append(score)\n            cf_matrix.append(\"TP\")\n        else:\n            js_scores.append(score)\n            cf_matrix.append(\"FP\")\n            \n    #### Counting False Negatives (FN)\n    \n    for gt in gts:\n        score = 0\n        for pd in pds:\n            js = jaccard(gt, pd)\n            if js > score:\n                score = js\n        if score == 0:\n            js_scores.append(score)\n            cf_matrix.append(\"FN\")\n            \n    return js_scores, \" \".join(cf_matrix)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.855332Z","iopub.execute_input":"2021-06-14T08:36:11.855917Z","iopub.status.idle":"2021-06-14T08:36:11.865666Z","shell.execute_reply.started":"2021-06-14T08:36:11.855881Z","shell.execute_reply":"2021-06-14T08:36:11.864802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validate_ground_truth['evaluation'] = validate_ground_truth.apply(lambda x: coleridge_initiative_jaccard(x['cleaned_label'],\n                                                                                                         x['pred'], verbose=False), axis=1)\nvalidate_ground_truth['js_scores'] = validate_ground_truth['evaluation'].apply(lambda x : x[0])\nvalidate_ground_truth['pred_type'] = validate_ground_truth['evaluation'].apply(lambda x : x[1])\n\n\n#coleridge_initiative_jaccard(validate_ground_truth['cleaned_label'], validate_ground_truth['pred'] , verbose=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.866953Z","iopub.execute_input":"2021-06-14T08:36:11.867449Z","iopub.status.idle":"2021-06-14T08:36:11.890825Z","shell.execute_reply.started":"2021-06-14T08:36:11.867412Z","shell.execute_reply":"2021-06-14T08:36:11.889956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_count_tp_fp_fn(prediction, verbose=True):\n    preds = prediction.split(\" \")\n    if verbose:\n        print(preds)\n    tpc = 0\n    fpc = 0\n    fnc = 0\n    for pred in preds:\n        if pred == \"TP\":\n            tpc = tpc + 1\n        elif pred == \"FP\":\n            fpc = fpc + 1\n        elif pred == \"FN\":\n            fnc = fnc + 1\n    return [tpc, fpc, fnc]\n\ndef make_col_tp_fp_fn(df, col):\n    df['TP'] = df[col].apply(lambda x : x[0])\n    df['FP'] = df[col].apply(lambda x : x[1])\n    df['FN'] = df[col].apply(lambda x : x[2])\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.891996Z","iopub.execute_input":"2021-06-14T08:36:11.892469Z","iopub.status.idle":"2021-06-14T08:36:11.900356Z","shell.execute_reply.started":"2021-06-14T08:36:11.892434Z","shell.execute_reply":"2021-06-14T08:36:11.899435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validate_ground_truth['tp_fp_fn'] = validate_ground_truth['pred_type'].apply(lambda x : get_count_tp_fp_fn(x, verbose=False))\nvalidate_ground_truth = make_col_tp_fp_fn(validate_ground_truth, 'tp_fp_fn')\ntp = sum(validate_ground_truth['TP'])\nfp = sum(validate_ground_truth['FP'])\nfn = sum(validate_ground_truth['FN'])\n\nprint(\"True Positives (TP) : \", tp)\nprint(\"False Positives (FP) : \", fp)\nprint(\"False Negatives (FN) : \", fn)\n\nprecision, recall = get_precision_recall(tp, fp, fn)\nprint(\"Precision : \", precision)\nprint(\"Recall : \", recall)\n\nfbeta = fbeta_score(precision, recall, 0.5)\nprint(\"FBeta Score : \", fbeta)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.901740Z","iopub.execute_input":"2021-06-14T08:36:11.902182Z","iopub.status.idle":"2021-06-14T08:36:11.919376Z","shell.execute_reply.started":"2021-06-14T08:36:11.902148Z","shell.execute_reply":"2021-06-14T08:36:11.917221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_false_negatives = validate_ground_truth[(validate_ground_truth['FN'] > 0) & (validate_ground_truth['TP'] == 0) & (validate_ground_truth['FP'] == 0)]\n# False negative is when there is no prediction --> FN at least 1 and TP and FP equal to 0\n\nall_full_true_positive = validate_ground_truth[(validate_ground_truth['TP'] > 0) & (validate_ground_truth['FP'] == 0) & (validate_ground_truth['FN'] == 0)]\n# Fully correct predictions --> TP bigger at least 1 and FP and FN equal to 0\n\nall_only_wrong_prediction = validate_ground_truth[(validate_ground_truth['TP'] == 0) & (validate_ground_truth['FP'] > 0)]\n# only wrong predictions --> TP is equal to 0 and FP is at least 1, FN does not matter\n\nall_good_and_wrong_pred = validate_ground_truth[(validate_ground_truth['TP'] > 0) & (validate_ground_truth['FP'] > 0)]\n# at least one good prediction and at least one bad prediction, FN are always 0 in this case","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.921455Z","iopub.execute_input":"2021-06-14T08:36:11.921882Z","iopub.status.idle":"2021-06-14T08:36:11.946518Z","shell.execute_reply.started":"2021-06-14T08:36:11.921848Z","shell.execute_reply":"2021-06-14T08:36:11.945826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_labels = validate_ground_truth['dataset_label'].value_counts().rename_axis('label').reset_index(name='all_labels_count')\nfalse_negatives = all_false_negatives[\"dataset_label\"].value_counts().rename_axis('label').reset_index(name='false_negatives')\nfull_true_positive = all_full_true_positive[\"dataset_label\"].value_counts().rename_axis('label').reset_index(name='full_true_positive')\nonly_wrong_prediction = all_only_wrong_prediction[\"dataset_label\"].value_counts().rename_axis('label').reset_index(name='full_wrong')\ngood_and_wrong_pred = all_good_and_wrong_pred[\"dataset_label\"].value_counts().rename_axis('label').reset_index(name='good_and_wrong')\n\n\nx1 = pd.merge(all_labels, false_negatives , on='label', how = 'left')\nx2 = pd.merge(x1, full_true_positive , on='label', how = 'left')\nx3 = pd.merge(x2, only_wrong_prediction , on='label', how = 'left')\nx4 = pd.merge(x3, good_and_wrong_pred , on='label', how = 'left')\n\nx4.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:11.947769Z","iopub.execute_input":"2021-06-14T08:36:11.948099Z","iopub.status.idle":"2021-06-14T08:36:12.001821Z","shell.execute_reply.started":"2021-06-14T08:36:11.948064Z","shell.execute_reply":"2021-06-14T08:36:12.000985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option(\"display.max_rows\", None, \"display.max_columns\", None)","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:12.003379Z","iopub.execute_input":"2021-06-14T08:36:12.003748Z","iopub.status.idle":"2021-06-14T08:36:12.008046Z","shell.execute_reply.started":"2021-06-14T08:36:12.003706Z","shell.execute_reply":"2021-06-14T08:36:12.006956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Datasets die maar 1 keer voorkomen in de validatieset\n\nsingle_datasets = x4[(x4['all_labels_count'] ==1)].fillna(0)\nprint(\"False_negative % = \", sum(single_datasets[\"false_negatives\"]) / sum(single_datasets[\"all_labels_count\"]))\nprint(\"Full_true_positive % = \", sum(single_datasets[\"full_true_positive\"]) / sum(single_datasets[\"all_labels_count\"]))\nprint(\"Full_wrong % = \", sum(single_datasets[\"full_wrong\"]) / sum(single_datasets[\"all_labels_count\"]))\nprint(\"Good_and_wrong % = \", sum(single_datasets[\"good_and_wrong\"]) / sum(single_datasets[\"all_labels_count\"]))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:12.009923Z","iopub.execute_input":"2021-06-14T08:36:12.010308Z","iopub.status.idle":"2021-06-14T08:36:12.022164Z","shell.execute_reply.started":"2021-06-14T08:36:12.010275Z","shell.execute_reply":"2021-06-14T08:36:12.021145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Datasets die het vaakst voorkomen in de validatieset\n\ncommon_datasets = x4[(x4['all_labels_count'] > 30)].fillna(0)\nprint(\"False_negative % = \", sum(common_datasets[\"false_negatives\"]) / sum(common_datasets[\"all_labels_count\"]))\nprint(\"Full_true_positive % = \", sum(common_datasets[\"full_true_positive\"]) / sum(common_datasets[\"all_labels_count\"]))\nprint(\"Full_wrong % = \", sum(common_datasets[\"full_wrong\"]) / sum(common_datasets[\"all_labels_count\"]))\nprint(\"Good_and_wrong % = \", sum(common_datasets[\"good_and_wrong\"]) / sum(common_datasets[\"all_labels_count\"]))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:12.023944Z","iopub.execute_input":"2021-06-14T08:36:12.024730Z","iopub.status.idle":"2021-06-14T08:36:12.034176Z","shell.execute_reply.started":"2021-06-14T08:36:12.024691Z","shell.execute_reply":"2021-06-14T08:36:12.032118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_mostly_abbrev = [\"ADNI\", \"SLOSH model\", \"NOAA Tide Gauge\", \"IBTrACS\", \"NOAA C-CAP\"]\nlabels_containing_abrev = [\"ADNI\", \"Alzheimer's Disease Neuroimaging Initiative (ADNI)\", \"Baltimore Longitudinal Study of Aging (BLSA)\",\n                          \"SLOSH model\", \"NOAA Tide Gauge\", \"SARS-CoV-2 genome sequences\", \"IBTrACS\", \"genome sequence of SARS-CoV-2\", \n                          \"North American Breeding Bird Survey (BBS)\", \"genome sequences of SARS-CoV-2\", \"COVID-19 Open Research Dataset\",\n                          \"USDA Census of Agriculture\", \"SARS-CoV-2 genome sequence\", \"COVID-19 Open Research Dataset (CORD-19)\", \n                          \"NOAA Optimum Interpolation Sea Surface Temperature\", \"NSF Survey of Earned Doctorates\", \"NCES Common Core of Data\",\n                          \"NOAA C-CAP\", \"NOAA tide station\", \"NASS Census of Agriculture\", \"NOAA World Ocean Database\", \"ANSS Comprehensive Catalog\",\n                          \"COVID-19 Image Data Collection\", \"NSF Survey of Graduate Students and Postdoctorates in Science and Engineering\",\n                           \"genome sequence of 2019-nCoV\", \"NSF Survey of Industrial Research and Development\", \"SARS-CoV-2 full genome sequence\",\n                          \"genome sequences of 2019-nCoV\", \"ARMS Farm Financial and Crop Production Practices\", \"NSF Survey of Science and Engineering Research Facilities\",\n                          \"2019-nCoV complete genome sequences\", \"NOAA National Water Level Observation Network\", \"SARS-CoV-2 full genome sequences\",\n                          \"genome sequences of COVID-19\", \"COVID-19 genome sequences\", \"COVID-19 Open Research Data\", \"ANSS Comprehensive Earthquake Catalog\",\n                          \"Our World in Data COVID-19\", \"genome sequence of COVID-19\", \"NOAA water level station\", \"COVID-19 Death data\",\n                          \"The National Institute on Aging Genetics of Alzheimer's Disease Data Storage Site (NIAGADS)\", \"NOAA Sea, Lake, and Overland Surges from Hurricanes\"]\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:12.035828Z","iopub.execute_input":"2021-06-14T08:36:12.036445Z","iopub.status.idle":"2021-06-14T08:36:12.043824Z","shell.execute_reply.started":"2021-06-14T08:36:12.036406Z","shell.execute_reply":"2021-06-14T08:36:12.042683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# algemene score\n\nx4_nona = x4.fillna(0)\nprint(\"False_negative % = \", sum(x4_nona[\"false_negatives\"]) / sum(x4_nona[\"all_labels_count\"]))\nprint(\"Full_true_positive % = \", sum(x4_nona[\"full_true_positive\"]) / sum(x4_nona[\"all_labels_count\"]))\nprint(\"Full_wrong % = \", sum(x4_nona[\"full_wrong\"]) / sum(x4_nona[\"all_labels_count\"]))\nprint(\"Good_and_wrong % = \", sum(x4_nona[\"good_and_wrong\"]) / sum(x4_nona[\"all_labels_count\"]))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:12.045231Z","iopub.execute_input":"2021-06-14T08:36:12.045623Z","iopub.status.idle":"2021-06-14T08:36:12.057477Z","shell.execute_reply.started":"2021-06-14T08:36:12.045588Z","shell.execute_reply":"2021-06-14T08:36:12.055819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# datasets met vooral afkortingen\n\nresults_labels_mostly_abbrev = x4[x4['label'].isin(labels_mostly_abbrev)].fillna(0)\n\nprint(\"False_negative % = \", sum(results_labels_mostly_abbrev[\"false_negatives\"]) / sum(results_labels_mostly_abbrev[\"all_labels_count\"]))\nprint(\"Full_true_positive % = \", sum(results_labels_mostly_abbrev[\"full_true_positive\"]) / sum(results_labels_mostly_abbrev[\"all_labels_count\"]))\nprint(\"Full_wrong % = \", sum(results_labels_mostly_abbrev[\"full_wrong\"]) / sum(results_labels_mostly_abbrev[\"all_labels_count\"]))\nprint(\"Good_and_wrong % = \", sum(results_labels_mostly_abbrev[\"good_and_wrong\"]) / sum(results_labels_mostly_abbrev[\"all_labels_count\"]))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:12.059076Z","iopub.execute_input":"2021-06-14T08:36:12.059476Z","iopub.status.idle":"2021-06-14T08:36:12.071432Z","shell.execute_reply.started":"2021-06-14T08:36:12.059442Z","shell.execute_reply":"2021-06-14T08:36:12.069832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# datasets die afkorting bevatten\n\nresults_labels_contain_abbrev = x4[x4['label'].isin(labels_containing_abrev)].fillna(0)\nprint(\"False_negative % = \", sum(results_labels_contain_abbrev[\"false_negatives\"]) / sum(results_labels_contain_abbrev[\"all_labels_count\"]))\nprint(\"Full_true_positive % = \", sum(results_labels_contain_abbrev[\"full_true_positive\"]) / sum(results_labels_contain_abbrev[\"all_labels_count\"]))\nprint(\"Full_wrong % = \", sum(results_labels_contain_abbrev[\"full_wrong\"]) / sum(results_labels_contain_abbrev[\"all_labels_count\"]))\nprint(\"Good_and_wrong % = \", sum(results_labels_contain_abbrev[\"good_and_wrong\"]) / sum(results_labels_contain_abbrev[\"all_labels_count\"]))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:12.073007Z","iopub.execute_input":"2021-06-14T08:36:12.073495Z","iopub.status.idle":"2021-06-14T08:36:12.085199Z","shell.execute_reply.started":"2021-06-14T08:36:12.073460Z","shell.execute_reply":"2021-06-14T08:36:12.083583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# datasets zonder afkortingen\n\nresults_no_abbrev = x4[~x4['label'].isin(labels_containing_abrev)].fillna(0)\nprint(\"False_negative % = \", sum(results_no_abbrev[\"false_negatives\"]) / sum(results_no_abbrev[\"all_labels_count\"]))\nprint(\"Full_true_positive % = \", sum(results_no_abbrev[\"full_true_positive\"]) / sum(results_no_abbrev[\"all_labels_count\"]))\nprint(\"Full_wrong % = \", sum(results_no_abbrev[\"full_wrong\"]) / sum(results_no_abbrev[\"all_labels_count\"]))\nprint(\"Good_and_wrong % = \", sum(results_no_abbrev[\"good_and_wrong\"]) / sum(results_no_abbrev[\"all_labels_count\"]))","metadata":{"execution":{"iopub.status.busy":"2021-06-14T08:36:12.087447Z","iopub.execute_input":"2021-06-14T08:36:12.088123Z","iopub.status.idle":"2021-06-14T08:36:12.099875Z","shell.execute_reply.started":"2021-06-14T08:36:12.088084Z","shell.execute_reply":"2021-06-14T08:36:12.098740Z"},"trusted":true},"execution_count":null,"outputs":[]}]}