{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BERT NER & Literal Matching\n## Coleridge challenge\nThis notebook gives a simple combination of literal matching and Named Entity Recognition using a BERT-like model.\nThe training phase of the BERT model was done in another kernel: https://www.kaggle.com/isaiahvh/pytorch-bert-for-named-entity-recognition\n\n**MODEL USED: SciBERT**","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # Limits used sampled to a small number for experimentation, set None for production.\nCUSTOM_VALIDATION = 1 # 0 == identical train-test set, 1 == partially overlapping train-test set, 2 == disjoint train-test set\n# all own custom test set of equal size","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:43:20.688509Z","iopub.execute_input":"2021-06-09T12:43:20.688909Z","iopub.status.idle":"2021-06-09T12:43:20.693834Z","shell.execute_reply.started":"2021-06-09T12:43:20.688829Z","shell.execute_reply":"2021-06-09T12:43:20.692831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:43:20.712321Z","iopub.execute_input":"2021-06-09T12:43:20.712625Z","iopub.status.idle":"2021-06-09T12:44:52.097795Z","shell.execute_reply.started":"2021-06-09T12:43:20.712598Z","shell.execute_reply":"2021-06-09T12:44:52.096513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom collections import namedtuple\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-09T12:44:52.100264Z","iopub.execute_input":"2021-06-09T12:44:52.100696Z","iopub.status.idle":"2021-06-09T12:44:52.946005Z","shell.execute_reply.started":"2021-06-09T12:44:52.100652Z","shell.execute_reply":"2021-06-09T12:44:52.945112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"# # NOTE: We use training data (here only) for literal matching\n# # QUESTION: Why do we use both training and test data for `papers`?\n# train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\n# train = pd.read_csv(train_path)\n# train = train[:MAX_SAMPLE]\n\n# paper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n# papers = {}\n# for paper_id in train['Id'].unique():\n#     with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n#         paper = json.load(f)\n#         papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:44:52.947288Z","iopub.execute_input":"2021-06-09T12:44:52.947555Z","iopub.status.idle":"2021-06-09T12:44:52.955918Z","shell.execute_reply.started":"2021-06-09T12:44:52.947527Z","shell.execute_reply":"2021-06-09T12:44:52.954909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nif CUSTOM_VALIDATION == 2:\n    sample_submission_path = '../input/colridge-custom-dataset-split/data_subsets/Test_set.csv'\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\nelif CUSTOM_VALIDATION == 0:\n    sample_submission_path = '../input/colridge-custom-dataset-split/Full_Overlap_Test_set.csv'\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\nelif CUSTOM_VALIDATION == 1:\n    sample_submission_path = '../input/colridge-custom-dataset-split/Partial_Overlap_Test_set.csv'\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\nelse:\n    assert False, \"CUSTOM_VALIDATION has an invalid value\"\nsample_submission = pd.read_csv(sample_submission_path)\n\n\n\nif MAX_SAMPLE is not None:\n    sample_submission = sample_submission.head(MAX_SAMPLE)\n    \n\nsample_submission = sample_submission.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:44:52.957049Z","iopub.execute_input":"2021-06-09T12:44:52.957396Z","iopub.status.idle":"2021-06-09T12:44:58.251481Z","shell.execute_reply.started":"2021-06-09T12:44:52.957366Z","shell.execute_reply":"2021-06-09T12:44:58.250654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n# Also removes any duplicate (subsequent) spaces\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:44:58.260318Z","iopub.execute_input":"2021-06-09T12:44:58.260641Z","iopub.status.idle":"2021-06-09T12:44:58.275392Z","shell.execute_reply.started":"2021-06-09T12:44:58.260608Z","shell.execute_reply":"2021-06-09T12:44:58.274203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 48 # Max number of words for each sentence\nOVERLAP = 16 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\n# QUESTION: Is this words, characters, etc.?\nPREDICT_BATCH = 64000 \n\n# NOTE: Despite naming, doesn't necessarily contain distilbert\n# Check the version description of this dataset for the model context\nPRETRAINED_PATH = '../input/distilbertnerfordatasets/output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = '../input/distilbertnerfordatasets/train_ner.json'\n# SUGGESTION: Actually separate into train & validation data to limit overfitting\nVAL_PATH = '../input/distilbertnerfordatasets/train_ner.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:44:58.300028Z","iopub.execute_input":"2021-06-09T12:44:58.300333Z","iopub.status.idle":"2021-06-09T12:44:58.310144Z","shell.execute_reply.started":"2021-06-09T12:44:58.300304Z","shell.execute_reply":"2021-06-09T12:44:58.309143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:44:58.314006Z","iopub.execute_input":"2021-06-09T12:44:58.314415Z","iopub.status.idle":"2021-06-09T12:44:58.324468Z","shell.execute_reply.started":"2021-06-09T12:44:58.314377Z","shell.execute_reply":"2021-06-09T12:44:58.323263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # Test data in NER format\npaper_length = []\n\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    \n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    # QUESTION: Why do we load only sentences with 'data' or 'study'?\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n    # Collect 'dummy' labeled sentences as dictionary\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # Track number of sentence fragments each paper has\n    paper_length.append(len(sentences))\n    \nprint(f'Loaded {len(test_rows)} sentence fragments')","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:44:58.326734Z","iopub.execute_input":"2021-06-09T12:44:58.327231Z","iopub.status.idle":"2021-06-09T12:45:03.250421Z","shell.execute_reply.started":"2021-06-09T12:44:58.32712Z","shell.execute_reply":"2021-06-09T12:45:03.249274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:45:03.251747Z","iopub.execute_input":"2021-06-09T12:45:03.252075Z","iopub.status.idle":"2021-06-09T12:45:03.257557Z","shell.execute_reply.started":"2021-06-09T12:45:03.252043Z","shell.execute_reply":"2021-06-09T12:45:03.256459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:45:03.258713Z","iopub.execute_input":"2021-06-09T12:45:03.259014Z","iopub.status.idle":"2021-06-09T12:45:04.010043Z","shell.execute_reply.started":"2021-06-09T12:45:03.258971Z","shell.execute_reply":"2021-06-09T12:45:04.008837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# QUESTION: What are `train_file` and `validation_file` for if we are only interested in predictions?\n# Could we omit them, given the `--do_predict` flag?\n\ndef bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:45:04.012186Z","iopub.execute_input":"2021-06-09T12:45:04.01266Z","iopub.status.idle":"2021-06-09T12:45:04.019214Z","shell.execute_reply.started":"2021-06-09T12:45:04.012614Z","shell.execute_reply":"2021-06-09T12:45:04.018098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # Write test data to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # Remove previous output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # Perform prediction\n    bert_predict()\n    \n    # Read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:45:04.020847Z","iopub.execute_input":"2021-06-09T12:45:04.021302Z","iopub.status.idle":"2021-06-09T12:49:26.993374Z","shell.execute_reply.started":"2021-06-09T12:45:04.021259Z","shell.execute_reply":"2021-06-09T12:49:26.99118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"code","source":"# Retrieve test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:26.995494Z","iopub.status.idle":"2021-06-09T12:49:26.996476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if all(all(tag == 'O' for tag in sentence) for sentence in bert_outputs):\n    print(\"No predictions were made\")\nelse:\n    print(\"Some predictions were made\")","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:26.997347Z","iopub.status.idle":"2021-06-09T12:49:26.997763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # Dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        \n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # Start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # Continue the phrase\n                curr_phrase += ' ' + word\n            else: # End last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                    \n        # Add label if suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # Record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    # NOTE: One sloppy way to do indexing.\n    # QUESTION: Would it improve anything (i.e. RAM)?\n    del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:27.002615Z","iopub.status.idle":"2021-06-09T12:49:27.003004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Output sample predictions\nbert_dataset_labels[:5]\nif all(pred == set() for pred in bert_dataset_labels):\n    print(\"WARNING: No predictions were made!\")","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:27.004157Z","iopub.status.idle":"2021-06-09T12:49:27.004543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\n# Select from bert labels to prevent near-duplicates\nfinal_bert_selection = []\ntotal_duplicate_exclusion = 0\n\nfor labels in bert_dataset_labels:\n    label_selection = []\n    \n    for new_label in sorted(labels, key=len):\n        new_label = clean_text(new_label)\n        if len(label_selection) == 0 or all(jaccard_similarity(new_label, included_label) < 0.75 for included_label in label_selection):\n            label_selection.append(new_label)\n        else:\n            total_duplicate_exclusion += 1\n    \n    final_bert_selection.append('|'.join(label_selection))","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:27.005428Z","iopub.status.idle":"2021-06-09T12:49:27.005809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Output sample bert label selection\nprint(total_duplicate_exclusion)\nprint(final_bert_selection[:5])","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:27.006606Z","iopub.status.idle":"2021-06-09T12:49:27.007018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"final_predictions = final_bert_selection","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:27.008034Z","iopub.status.idle":"2021-06-09T12:49:27.008407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.head()\nsample_submission.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:27.00925Z","iopub.status.idle":"2021-06-09T12:49:27.009709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f_score(TP, FP, FN, beta = 0.5):\n    num = (1 + beta*beta) * TP\n    denom = (1 + beta*beta) * TP + beta*beta * FN + FP\n    return num/denom","metadata":{"execution":{"iopub.status.busy":"2021-06-09T12:49:27.010686Z","iopub.status.idle":"2021-06-09T12:49:27.011087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(final_predictions), len(sample_submission))\n\npred_count = 0\n\nTP = 0\nFP = 0\nFN = 0\n\nfor predictions, truths in zip(final_predictions, sample_submission['cleaned_label']):\n    predictions = [clean_text(pred) for pred in predictions.split('|')]\n    truths = [truth for truth in truths.split('|')]\n\n    # Determine best matches for truths\n    JACCARD_THRESHOLD = 0.5\n    potential_matches = []\n    Match = namedtuple('Match', 'truth_index pred_index score')\n\n    for i, t in enumerate(truths):\n        for j, p in enumerate(predictions):\n            score = jaccard_similarity(t, p)\n            potential_matches.append(Match(i, j, score))\n\n    potential_matches.sort(key=lambda m: m.score, reverse=True)\n\n    matches = []\n    for potential_match in potential_matches:\n        if any(used_match.truth_index == potential_match.truth_index\\\n              or used_match.pred_index == potential_match.pred_index\\\n              for used_match in matches):\n            continue\n        if potential_match.score >= JACCARD_THRESHOLD:\n            matches.append(potential_match)\n\n    n_true_pos = len(matches)\n    n_false_pos = (len(predictions) - len(matches))\n    n_false_neg = len(truths) - len(matches)\n\n    pred_count += len(predictions)\n    TP += n_true_pos\n    FP += n_false_pos\n    FN += n_false_neg\n\nprint(f\"TP: {TP}, FP: {FP}, FN: {FN}\")\nprint(f\"--> sum = {TP+FP+FN}, total_preds = {pred_count}\")\n\nscore = f_score(TP, FP, FN, beta = 0.5)\nprint(f\"Achieved F-micro0.5 score of {score}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}