{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:52:52.745634Z","iopub.execute_input":"2021-06-02T08:52:52.746002Z","iopub.status.idle":"2021-06-02T08:52:52.752644Z","shell.execute_reply.started":"2021-06-02T08:52:52.74597Z","shell.execute_reply":"2021-06-02T08:52:52.75173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:52:52.754725Z","iopub.execute_input":"2021-06-02T08:52:52.755162Z","iopub.status.idle":"2021-06-02T08:54:16.186693Z","shell.execute_reply.started":"2021-06-02T08:52:52.755127Z","shell.execute_reply":"2021-06-02T08:54:16.185788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_colwidth', None)\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-02T08:54:16.190901Z","iopub.execute_input":"2021-06-02T08:54:16.191189Z","iopub.status.idle":"2021-06-02T08:54:16.198784Z","shell.execute_reply.started":"2021-06-02T08:54:16.19116Z","shell.execute_reply":"2021-06-02T08:54:16.19768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\ntest_sample = train.sample(100)\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers_train = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers_train[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:54:16.202003Z","iopub.execute_input":"2021-06-02T08:54:16.202281Z","iopub.status.idle":"2021-06-02T08:54:28.543499Z","shell.execute_reply.started":"2021-06-02T08:54:16.202257Z","shell.execute_reply":"2021-06-02T08:54:28.54272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\n\npapers_test = {}\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers_test[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:54:28.546774Z","iopub.execute_input":"2021-06-02T08:54:28.547048Z","iopub.status.idle":"2021-06-02T08:54:28.566066Z","shell.execute_reply.started":"2021-06-02T08:54:28.547023Z","shell.execute_reply":"2021-06-02T08:54:28.565051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Making validation set","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:54:28.567366Z","iopub.execute_input":"2021-06-02T08:54:28.567713Z","iopub.status.idle":"2021-06-02T08:54:28.573367Z","shell.execute_reply.started":"2021-06-02T08:54:28.567678Z","shell.execute_reply":"2021-06-02T08:54:28.572325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_labels = {}\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels[label_1.lower()] = label_3.lower()\n    all_labels[label_2.lower()] = label_3.lower()\n    all_labels[label_3.lower()] = label_3.lower()\n\nprint(f'No. different labels: {len(all_labels)}')\n\n\nvalidation_labels = ['adni', \"alzheimer's disease neuroimaging initiative (adni)\",\n                     'alzheimers disease neuroimaging initiative', 'alzheimer s disease neuroimaging initiative adni ',\n#                     'rural-urban continuum codes', 'rural urban continuum codes',\n#                     'baccalaureate and beyond', 'baccalaureate and beyond longitudinal study',\n                    ]\nfor val_lab in validation_labels:\n    all_labels.pop(val_lab)\n    \nprint(print(f'No. different labels: {len(all_labels)}'))","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:54:28.574927Z","iopub.execute_input":"2021-06-02T08:54:28.575262Z","iopub.status.idle":"2021-06-02T08:54:28.624967Z","shell.execute_reply.started":"2021-06-02T08:54:28.575227Z","shell.execute_reply":"2021-06-02T08:54:28.624098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers_valid = {}\ncount_valid = 0\nfor i, id, dataset_label, cleaned_label in train[['Id', 'dataset_label', 'cleaned_label']].itertuples(): #test_sample['Id']: sample_submission['Id']\n    paper = papers_train[id]\n    \n    labels = set()\n    \n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    if dataset_label.lower() in validation_labels or cleaned_label.lower() in validation_labels:\n        papers_valid[id] = papers_train[id]\n        count_valid += 1\n    else:\n        for label in validation_labels:\n            if label in text_1 or label in text_2:\n                papers_valid[id] = papers_train[id]\n                count_valid += 1\n                break\n    \n#     if dataset_label.lower() in validation_labels or cleaned_label.lower() in validation_labels:\n#         papers_valid[id] = papers_train[id]\n#         count_valid += 1\n            \nfor paper_id in papers_valid:\n    papers_train.pop(paper_id)\n\nprint(\"Valid label count: \", count_valid)\nprint(len(papers_train))\nprint(len(papers_valid))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:54:28.626207Z","iopub.execute_input":"2021-06-02T08:54:28.626694Z","iopub.status.idle":"2021-06-02T08:56:51.28606Z","shell.execute_reply.started":"2021-06-02T08:54:28.626657Z","shell.execute_reply":"2021-06-02T08:56:51.285099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"# all_labels = {}\n\n# for label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n#     all_labels[label_1.lower()] = label_3.lower()\n#     all_labels[label_2.lower()] = label_3.lower()\n#     all_labels[label_3.lower()] = label_3.lower()\n# #     all_labels.add(str(label_1).lower())\n# #     all_labels.add(str(label_2).lower())\n# #     all_labels.add(str(label_3).lower())\n# # for la in zip(all_labels.keys(), all_labels.values()):\n# #     print(la)\n# print(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:56:51.287417Z","iopub.execute_input":"2021-06-02T08:56:51.287928Z","iopub.status.idle":"2021-06-02T08:56:51.291877Z","shell.execute_reply.started":"2021-06-02T08:56:51.28789Z","shell.execute_reply":"2021-06-02T08:56:51.29096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"# def clean_text(txt):\n#     return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n# def totally_clean_text(txt):\n#     txt = clean_text(txt)\n#     txt = re.sub(' +', ' ', txt)\n#     return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:56:51.293138Z","iopub.execute_input":"2021-06-02T08:56:51.293657Z","iopub.status.idle":"2021-06-02T08:56:51.310357Z","shell.execute_reply.started":"2021-06-02T08:56:51.293621Z","shell.execute_reply":"2021-06-02T08:56:51.309622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# literal_preds = []\n\n# for paper_id in papers_valid.keys(): #test_sample['Id']: sample_submission['Id']\n#     paper = papers_valid[paper_id]\n#     text_1 = '. '.join(section['text'] for section in paper).lower()\n#     text_2 = totally_clean_text(text_1)\n    \n#     labels = set()\n#     for label in all_labels.keys():\n#         if label in text_1 or label in text_2:\n#             labels.add(all_labels[label])\n# #             if label != all_labels[label]:\n# #                 print(f\"{label} -> {all_labels[label]}\")\n# #     print(labels)\n    \n#     labels = sorted(labels, key=len, reverse = True)\n# #     print(labels)\n# #     for idx, lab1 in enumerate(labels):\n# # #         print(lab1)\n# # #         print(\"...\")\n# #         for lab2 in [label for label in labels[idx:] if label != lab1]:\n# # #             print(lab2)\n# #             words1 = set(lab1.split())\n# #             words2 = set(lab2.split())\n# # #             print(f\"{len(words1 & words2)}/{len(words1)}: {len(words1 & words2)/len(words1)}\")\n# #             if len(words1 & words2)/len(words1) > 0.75:\n# #                 labels.remove(lab2)\n        \n# #         print(\"---\")\n#     literal_preds.append('|'.join(labels))\n# #     print(literal_preds[-1])\n# #     print(\"***\")\n    \n# # print(literal_preds)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:56:51.313263Z","iopub.execute_input":"2021-06-02T08:56:51.313517Z","iopub.status.idle":"2021-06-02T08:56:51.322127Z","shell.execute_reply.started":"2021-06-02T08:56:51.313495Z","shell.execute_reply":"2021-06-02T08:56:51.321491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# random.sample(list(zip(test_sample['cleaned_label'], literal_preds)), 8)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:56:51.327246Z","iopub.execute_input":"2021-06-02T08:56:51.327482Z","iopub.status.idle":"2021-06-02T08:56:51.337536Z","shell.execute_reply.started":"2021-06-02T08:56:51.32746Z","shell.execute_reply":"2021-06-02T08:56:51.336803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 48 # max no. words for each sentence.\nOVERLAP = 16 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\nPRETRAINED_PATH = '../input/scibert-validation-set-version/output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = '../input/coleridge-bert-models/train_ner.json'\nVAL_PATH = '../input/coleridge-bert-models/train_ner.json'\n\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:56:51.338921Z","iopub.execute_input":"2021-06-02T08:56:51.339585Z","iopub.status.idle":"2021-06-02T08:56:51.348801Z","shell.execute_reply.started":"2021-06-02T08:56:51.339523Z","shell.execute_reply":"2021-06-02T08:56:51.347939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:56:51.35011Z","iopub.execute_input":"2021-06-02T08:56:51.350455Z","iopub.status.idle":"2021-06-02T08:56:51.703937Z","shell.execute_reply.started":"2021-06-02T08:56:51.350423Z","shell.execute_reply":"2021-06-02T08:56:51.703046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:56:51.705536Z","iopub.execute_input":"2021-06-02T08:56:51.705901Z","iopub.status.idle":"2021-06-02T08:56:51.712524Z","shell.execute_reply.started":"2021-06-02T08:56:51.705865Z","shell.execute_reply":"2021-06-02T08:56:51.711652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_rows = [] # test data in NER format\n# paper_length = [] # store the number of sentences each paper has\n\n# for paper_id in papers_valid.keys():\n#     # load paper\n#     paper = papers_valid[paper_id]\n\n#     # extract sentences\n#     sentences = [clean_training_text(sentence) for section in paper \n#                  for sentence in section['text'].split('.')\n#                 ]\n#     sentences = shorten_sentences(sentences) # make sentences short\n#     sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n#     sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n\n#     # collect all sentences in json\n#     for sentence in sentences:\n#         sentence_words = sentence.split()\n#         dummy_tags = ['O']*len(sentence_words)\n#         test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n\n#     # track which sentence belongs to which data point\n#     paper_length.append(len(sentences))\n\n# print(f'total number of sentences: {len(test_rows)}')\n\n###################################################################################\n\n#Import preselection model\nfrom joblib import dump, load\nclf = load('../input/preselection-model-validation-set/preselection_model.joblib') \n\ntest_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\n\n\nfor paper_id in papers_valid.keys():\n    # load paper\n    paper = papers_valid[paper_id]\n    \n    # extract sentences\n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    predictions = clf.predict(sentences)\n#     print(sum(predictions))\n    sentences = [sentence for idx, sentence in enumerate(sentences) if predictions[idx] > 0.0]\n#     sentences = [sentence for sentence in sentences if re.match( r'(.* has been used.*|.* (dataset|survey|database|study) .*|.*obtained from .*|.*data from .*|.*data used in .*)', sentence.lower())]#any(word in sentence.lower() for word in ['national', 'research', 'data', 'survey', 'science', 'technology', 'development', 'department', 'study', 'statistics','international'])]\n        \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-02T08:56:51.713783Z","iopub.execute_input":"2021-06-02T08:56:51.714254Z","iopub.status.idle":"2021-06-02T09:03:12.198952Z","shell.execute_reply.started":"2021-06-02T08:56:51.714219Z","shell.execute_reply":"2021-06-02T09:03:12.198003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:03:12.202257Z","iopub.execute_input":"2021-06-02T09:03:12.202536Z","iopub.status.idle":"2021-06-02T09:03:12.209583Z","shell.execute_reply.started":"2021-06-02T09:03:12.202508Z","shell.execute_reply":"2021-06-02T09:03:12.208606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:03:12.210969Z","iopub.execute_input":"2021-06-02T09:03:12.21133Z","iopub.status.idle":"2021-06-02T09:03:12.99819Z","shell.execute_reply.started":"2021-06-02T09:03:12.211292Z","shell.execute_reply":"2021-06-02T09:03:12.996937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:03:13.000051Z","iopub.execute_input":"2021-06-02T09:03:13.000409Z","iopub.status.idle":"2021-06-02T09:03:13.007445Z","shell.execute_reply.started":"2021-06-02T09:03:13.000372Z","shell.execute_reply":"2021-06-02T09:03:13.005965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:03:13.009477Z","iopub.execute_input":"2021-06-02T09:03:13.010036Z","iopub.status.idle":"2021-06-02T09:04:29.491591Z","shell.execute_reply.started":"2021-06-02T09:03:13.009999Z","shell.execute_reply":"2021-06-02T09:04:29.490283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.496167Z","iopub.execute_input":"2021-06-02T09:04:29.496519Z","iopub.status.idle":"2021-06-02T09:04:29.50517Z","shell.execute_reply.started":"2021-06-02T09:04:29.496479Z","shell.execute_reply":"2021-06-02T09:04:29.5041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.506418Z","iopub.execute_input":"2021-06-02T09:04:29.506994Z","iopub.status.idle":"2021-06-02T09:04:29.564333Z","shell.execute_reply.started":"2021-06-02T09:04:29.506954Z","shell.execute_reply":"2021-06-02T09:04:29.563317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels[:100]","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.567315Z","iopub.execute_input":"2021-06-02T09:04:29.569771Z","iopub.status.idle":"2021-06-02T09:04:29.580133Z","shell.execute_reply.started":"2021-06-02T09:04:29.56973Z","shell.execute_reply":"2021-06-02T09:04:29.579238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.581678Z","iopub.execute_input":"2021-06-02T09:04:29.582433Z","iopub.status.idle":"2021-06-02T09:04:29.597079Z","shell.execute_reply.started":"2021-06-02T09:04:29.582359Z","shell.execute_reply":"2021-06-02T09:04:29.596114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_bert_labels[:20]\nprint([x for x in filtered_bert_labels if x != ''])","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.598808Z","iopub.execute_input":"2021-06-02T09:04:29.599525Z","iopub.status.idle":"2021-06-02T09:04:29.616334Z","shell.execute_reply.started":"2021-06-02T09:04:29.599486Z","shell.execute_reply":"2021-06-02T09:04:29.613189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"final_predictions = []\nfor bert_pred in filtered_bert_labels:\n    if bert_pred:\n        final_predictions.append(bert_pred)\n    else:\n        final_predictions.append('')","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.617981Z","iopub.execute_input":"2021-06-02T09:04:29.618635Z","iopub.status.idle":"2021-06-02T09:04:29.625585Z","shell.execute_reply.started":"2021-06-02T09:04:29.618576Z","shell.execute_reply":"2021-06-02T09:04:29.624333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(final_predictions)\nprint([x for x in final_predictions if x != ''])","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.627321Z","iopub.execute_input":"2021-06-02T09:04:29.628163Z","iopub.status.idle":"2021-06-02T09:04:29.643816Z","shell.execute_reply.started":"2021-06-02T09:04:29.628125Z","shell.execute_reply":"2021-06-02T09:04:29.642792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission['PredictionString'] = final_predictions\n# sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.646185Z","iopub.execute_input":"2021-06-02T09:04:29.647769Z","iopub.status.idle":"2021-06-02T09:04:29.655023Z","shell.execute_reply.started":"2021-06-02T09:04:29.647732Z","shell.execute_reply":"2021-06-02T09:04:29.653224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T09:04:29.659343Z","iopub.execute_input":"2021-06-02T09:04:29.65993Z","iopub.status.idle":"2021-06-02T09:04:29.669861Z","shell.execute_reply.started":"2021-06-02T09:04:29.659891Z","shell.execute_reply":"2021-06-02T09:04:29.669166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}