{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"LENGTH = 1\nFL_TH = 1.0\nPRETRAINED = '../input/coleridge-bert-ner-external-pseudo-labels'\nHOW = ['W_MLM'] # 'NER_ONLY', 'W_QA', 'W_MLM', 'W_MATCH', 'DOUBLE_BLENDING'\nW_QA_TH = 0.75\n\nADNL_GOVT_LABELS_PATH = '../input/coleridge-additional-datasets/additional_datasets_v6.csv'\n\nMAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\nSEED = 42","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:30:08.913967Z","iopub.execute_input":"2021-06-03T03:30:08.914357Z","iopub.status.idle":"2021-06-03T03:30:08.919663Z","shell.execute_reply.started":"2021-06-03T03:30:08.91432Z","shell.execute_reply":"2021-06-03T03:30:08.918608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setting","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\nimport os\nimport re\nimport json\nimport time\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport tokenizers\nfrom tokenizers import *\nimport transformers\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline\nfrom datasets import load_dataset\n\nfrom functools import partial\nfrom tqdm.notebook import tqdm\nfrom torch.nn import functional as F\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom IPython.display import clear_output\n\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:30:08.924638Z","iopub.execute_input":"2021-06-03T03:30:08.92508Z","iopub.status.idle":"2021-06-03T03:31:36.086968Z","shell.execute_reply.started":"2021-06-03T03:30:08.925015Z","shell.execute_reply":"2021-06-03T03:31:36.086147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://huggingface.co/transformers/_modules/transformers/trainer_utils.html\ndef set_seed(seed: int):\n    \"\"\"\n    Helper function for reproducible behavior to set the seed in ``random``, ``numpy``, ``torch`` and/or ``tf`` (if\n    installed).\n\n    Args:\n        seed (:obj:`int`): The seed to set.\n    \"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    # ^^ safe to call this function even if cuda is not available\n    \n    print(f'Setted Pipeline SEED = {SEED}')\n\n\nset_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:31:36.089128Z","iopub.execute_input":"2021-06-03T03:31:36.089427Z","iopub.status.idle":"2021-06-03T03:31:36.0968Z","shell.execute_reply.started":"2021-06-03T03:31:36.089397Z","shell.execute_reply":"2021-06-03T03:31:36.095933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\npapers = {}\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:31:36.098095Z","iopub.execute_input":"2021-06-03T03:31:36.098392Z","iopub.status.idle":"2021-06-03T03:31:36.15176Z","shell.execute_reply.started":"2021-06-03T03:31:36.098363Z","shell.execute_reply":"2021-06-03T03:31:36.151008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert NER Prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"## Transform data to NER format\nGroup by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\ndef clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:31:36.152959Z","iopub.execute_input":"2021-06-03T03:31:36.153247Z","iopub.status.idle":"2021-06-03T03:31:36.160365Z","shell.execute_reply.started":"2021-06-03T03:31:36.153219Z","shell.execute_reply":"2021-06-03T03:31:36.15939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in sample_submission['Id']:\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > LENGTH] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences w/ length{LENGTH}: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:31:36.163536Z","iopub.execute_input":"2021-06-03T03:31:36.163943Z","iopub.status.idle":"2021-06-03T03:31:36.204494Z","shell.execute_reply.started":"2021-06-03T03:31:36.1639Z","shell.execute_reply":"2021-06-03T03:31:36.2037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Do Predict And Collect Results","metadata":{}},{"cell_type":"code","source":"PRETRAINED_PATH = os.path.join(PRETRAINED, 'output')\nTRAIN_PATH = os.path.join(PRETRAINED, 'train_ner.json')\nVAL_PATH = os.path.join(PRETRAINED, 'train_ner.json')\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'\n\nos.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:31:36.206222Z","iopub.execute_input":"2021-06-03T03:31:36.206496Z","iopub.status.idle":"2021-06-03T03:31:36.21209Z","shell.execute_reply.started":"2021-06-03T03:31:36.206467Z","shell.execute_reply":"2021-06-03T03:31:36.211191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:31:36.213271Z","iopub.execute_input":"2021-06-03T03:31:36.213509Z","iopub.status.idle":"2021-06-03T03:31:36.968589Z","shell.execute_reply.started":"2021-06-03T03:31:36.213485Z","shell.execute_reply":"2021-06-03T03:31:36.967449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 42 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:31:36.972159Z","iopub.execute_input":"2021-06-03T03:31:36.972434Z","iopub.status.idle":"2021-06-03T03:31:36.977673Z","shell.execute_reply.started":"2021-06-03T03:31:36.972408Z","shell.execute_reply":"2021-06-03T03:31:36.976917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_outputs = []\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:31:36.979837Z","iopub.execute_input":"2021-06-03T03:31:36.980311Z","iopub.status.idle":"2021-06-03T03:33:28.034834Z","shell.execute_reply.started":"2021-06-03T03:31:36.980266Z","shell.execute_reply":"2021-06-03T03:33:28.033997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Restore Dataset Labels From Predictions","metadata":{}},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n\ndel test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:33:28.036253Z","iopub.execute_input":"2021-06-03T03:33:28.03653Z","iopub.status.idle":"2021-06-03T03:33:28.041885Z","shell.execute_reply.started":"2021-06-03T03:33:28.036501Z","shell.execute_reply":"2021-06-03T03:33:28.040884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]\n\nbert_dataset_labels[:4]","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:33:28.043249Z","iopub.execute_input":"2021-06-03T03:33:28.043505Z","iopub.status.idle":"2021-06-03T03:33:28.059827Z","shell.execute_reply.started":"2021-06-03T03:33:28.043479Z","shell.execute_reply":"2021-06-03T03:33:28.058812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filter Based On Jaccard Score And Clean","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:33:28.061298Z","iopub.execute_input":"2021-06-03T03:33:28.061883Z","iopub.status.idle":"2021-06-03T03:33:28.073273Z","shell.execute_reply.started":"2021-06-03T03:33:28.061851Z","shell.execute_reply":"2021-06-03T03:33:28.072153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard(label, got_label) < FL_TH for got_label in filtered):\n            filtered.append(label)\n    \n    filtered_bert_labels.append('|'.join(filtered))\n\nprint(f'filtered_bert_labels[:4] w/ FL{FL_TH}:')\nfiltered_bert_labels[:4]","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:33:28.074638Z","iopub.execute_input":"2021-06-03T03:33:28.074988Z","iopub.status.idle":"2021-06-03T03:33:28.088283Z","shell.execute_reply.started":"2021-06-03T03:33:28.074958Z","shell.execute_reply":"2021-06-03T03:33:28.087122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Match","metadata":{}},{"cell_type":"code","source":"def read_json_pub(filename, train_data_path=paper_test_folder, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    \n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data\n    \n    \ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:34:04.735085Z","iopub.execute_input":"2021-06-03T03:34:04.735428Z","iopub.status.idle":"2021-06-03T03:34:04.742817Z","shell.execute_reply.started":"2021-06-03T03:34:04.7354Z","shell.execute_reply":"2021-06-03T03:34:04.742124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'W_MATCH' in HOW :\n    adnl_govt_labels = pd.read_csv(ADNL_GOVT_LABELS_PATH)\n\n    literal_preds = []\n    to_append = []\n    for index, row in tqdm(sample_submission.iterrows()):\n        to_append = [row['Id'],'']\n        large_string = str(read_json_pub(row['Id'], paper_test_folder))\n        clean_string = text_cleaning(large_string)\n        for index, row2 in adnl_govt_labels.iterrows():\n            query_string = str(row2['title'])\n            if query_string in clean_string:\n                if to_append[1] != '' and clean_text(query_string) not in to_append[1]:\n                    to_append[1] = to_append[1] + '|' + clean_text(query_string)\n                if to_append[1] == '':\n                    to_append[1] = clean_text(query_string)\n        literal_preds.append(*to_append[1:])\n\nelse: literal_preds = None\n\nliteral_preds","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:34:04.75194Z","iopub.execute_input":"2021-06-03T03:34:04.752802Z","iopub.status.idle":"2021-06-03T03:34:04.951305Z","shell.execute_reply.started":"2021-06-03T03:34:04.752757Z","shell.execute_reply":"2021-06-03T03:34:04.950498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters Check","metadata":{}},{"cell_type":"code","source":"print(f'LENGTH = {LENGTH}')\nprint(f'FL_TH = {FL_TH}')\nprint(f'PRETRAINED = {PRETRAINED}')\nprint(f'HOW = {HOW}')\nif 'W_MATCH' in HOW: print(f'ADNL_GOVT_LABELS_PATH = {ADNL_GOVT_LABELS_PATH}')\nif 'W_QA' in HOW: print(f'W_QA_TH = {W_QA_TH}')","metadata":{"execution":{"iopub.status.busy":"2021-06-03T03:34:04.992155Z","iopub.execute_input":"2021-06-03T03:34:04.992389Z","iopub.status.idle":"2021-06-03T03:34:04.998789Z","shell.execute_reply.started":"2021-06-03T03:34:04.992366Z","shell.execute_reply":"2021-06-03T03:34:04.997869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"|   | CV | LB |\n| - | -- | -- |\n| NER only |   | 0.382 |\n| NER only FL1.0 |   | 0.388 |\n| NERv1 external+Pseudo length1 only FL1.0 (bug) |   | 0.397 |\n| NERv1 external+Pseudo length1 only FL1.0 (fixed) |   | 0.397 |\n| NERv1 external+Pseudo length1 FL1.0 W_MATCH |   | 0.583 |\n| (NERv1 external+Pseudo length1 FL1.0 + QA 0.995) 0.75 |   | 0.449 |\n| (NERv1 external+Pseudo length1 FL1.0 + QA 0.995) 0.75 WM |   | 0.582 |\n| (NERv2 external+Pseudo length1 FL1.0 + QA 0.995) 0.75 WM DB |   | 0.560 |\n| (NERv2 external+Pseudo length1 FL1.0 + QA 0.995) 0.75 |   | 0.451 |\n| **(NERv2 external+Pseudo length1 FL1.0 + QA 0.995) 0.75 + MLMv1 external-pseudo length1 FOPP FL1.0** |   | **0.453** |","metadata":{}},{"cell_type":"markdown","source":"# Bert QA","metadata":{}},{"cell_type":"code","source":"THRESHOLD = 0.995\nCP_PATH = '../input/bert-for-question-answering-baseline-training' + '/'\n\nSEED = 2020\n\nDATA_PATH = \"../input/coleridgeinitiative-show-us-the-data/\"\nDATA_PATH_TRAIN = DATA_PATH + 'train/'\nDATA_PATH_TEST = DATA_PATH + 'test/'\n\nNUM_WORKERS = 4\n\nVOCABS = {\n    \"bert-base-uncased\": \"../input/vocabs/bert-base-uncased-vocab.txt\",\n}\n\nMODEL_PATHS = {\n    'bert-base-uncased': '../input/bertconfigs/uncased_L-12_H-768_A-12/uncased_L-12_H-768_A-12/',\n    'bert-large-uncased-whole-word-masking-finetuned-squad': '../input/bertconfigs/wwm_uncased_L-24_H-1024_A-16/wwm_uncased_L-24_H-1024_A-16/',\n    'albert-large-v2': '../input/albert-configs/albert-large-v2/albert-large-v2/',\n    'albert-base-v2': '../input/albert-configs/albert-base-v2/albert-base-v2/',\n    'distilbert': '../input/albert-configs/distilbert/distilbert/',\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    # General\n    k = 5\n    seed = 2021\n\n    # Texts\n    max_len = 256\n    \n    # Architecture\n    selected_model = \"bert-base-uncased\"\n    lowercase = True\n    \n    # Training\n    batch_size = 16\n    batch_size_val = batch_size * 2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncodedText:\n    def __init__(self, ids, offsets):\n        self.ids = ids\n        self.offsets = offsets\n\n\ndef create_tokenizer_and_tokens(config):\n    if \"roberta\" in config.selected_model:\n        raise NotImplementedError\n        \n    elif \"albert\" in config.selected_model:\n        raise NotImplementedError\n        \n    else:\n        tokenizer = BertWordPieceTokenizer(\n            MODEL_PATHS[config.selected_model] + 'vocab.txt',\n            lowercase=config.lowercase,\n        )\n\n        tokens = {\n            'cls': tokenizer.token_to_id('[CLS]'),\n            'sep': tokenizer.token_to_id('[SEP]'),\n            'pad': tokenizer.token_to_id('[PAD]'),\n        }\n    \n    return tokenizer, tokens","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_text(id_, root=\"\"):\n    with open(os.path.join(root, id_ + \".json\")) as f:\n        text = json.load(f)\n    return text\n\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef locate_label_string(text, label):\n    \"\"\"\n    Finds the label in the text\n    \"\"\"\n    len_label = len(label) - 1\n\n    candidates_idx = [i for i, e in enumerate(text) if e == label[1]]\n    for idx in candidates_idx:\n        if \" \" + text[idx: idx + len_label] == label:\n            idx_start = idx\n            idx_end = idx + len_label\n            break\n\n    assert (\n        text[idx_start:idx_end] == label[1:]\n    ), f'\"{text[idx_start: idx_end]}\" instead of \"{label}\" in \"{text}\"'\n\n    char_targets = np.zeros(len(text))\n    char_targets[idx_start:idx_end] = 1\n\n    return idx_start, idx_end, char_targets\n\n\ndef locate_label_tokens(offsets, char_targets):\n    \"\"\"\n    Finds the tokens corresponding to the found labels\n    \"\"\"\n    target_idx = []\n    for idx, (offset1, offset2) in enumerate(offsets):\n        if sum(char_targets[offset1:offset2]) > 0:\n            target_idx.append(idx)\n\n    if not len(target_idx):\n        for idx, (offset1, offset2) in enumerate(offsets):\n            if sum(char_targets[offset1:offset2]) > 0:\n                target_idx.append(idx)\n\n    return target_idx[0], target_idx[-1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_data(\n    text,\n    label,\n    tokenizer,\n    tokens,\n    max_len=100,\n    model_name=\"bert\",\n):\n    \"\"\"\n    Prepares the data for the question answering task.\n    Adapted from Abishek's work on the Tweet Sentiment extraction competition, \n    check his work for more details !\n    \"\"\"\n    target_start, target_end = 0, 0\n    text = \" \" + \" \".join(str(text).split())\n    label = \" \" + \" \".join(str(label).split())\n\n    if label != \" \":\n        idx_start, idx_end, char_targets = locate_label_string(\n            text, label\n        )\n\n    tokenized = tokenizer.encode(text)\n    input_ids_text = tokenized.ids[1:-1]\n\n    # print(input_ids_text, len(input_ids_text))\n\n    offsets = tokenized.offsets[1:-1]\n\n    if label != \" \":\n        target_start, target_end = locate_label_tokens(offsets, char_targets)\n\n    if target_end >= max_len - 2:  # target is too far in the sentence, we crop its beginning.\n        n_tok_to_crop = target_start - max_len // 2\n        new_str_start = offsets[n_tok_to_crop][0]\n\n        input_ids_text = input_ids_text[n_tok_to_crop:]\n\n        offsets = [tuple(t) for t in np.array(offsets[n_tok_to_crop:]) - new_str_start]\n        text = text[new_str_start:]\n\n        target_start -= n_tok_to_crop\n        target_end -= n_tok_to_crop\n\n    input_ids = (\n        [tokens[\"cls\"]]\n        + input_ids_text[:max_len - 2]\n        + [tokens[\"sep\"]]\n    )\n\n    if \"roberta\" in model_name:\n        token_type_ids = [0] * len(input_ids)\n    else:\n        token_type_ids = [1] * len(input_ids)\n\n    text_offsets = [(0, 0)] + offsets[:max_len - 2] + [(0, 0)]\n\n    target_start += 1\n    target_end += 1\n\n    # target_end = min(target_end, max_len - 1)\n\n    assert len(input_ids) == len(token_type_ids) and len(input_ids) == len(text_offsets), (len(input_ids), len(text_offsets))  # noqa\n\n    padding_length = max_len - len(input_ids)\n    if padding_length > 0:\n        input_ids = input_ids + ([tokens[\"pad\"]] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n        text_offsets = text_offsets + ([(0, 0)] * padding_length)\n\n    return {\n        \"ids\": input_ids,\n        \"token_type_ids\": token_type_ids,\n        \"targets_start\": target_start,\n        \"targets_end\": target_end,\n        \"text\": text,\n        \"label\": label,\n        \"offsets\": text_offsets,\n    }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset\n\nclass ArticleDataset(Dataset):\n    \"\"\"\n    Dataset for inference. \n    \"\"\"\n    def __init__(\n        self,\n        id_,\n        tokenizer,\n        tokens,\n        max_len=512,\n        words_per_split=300,\n        margin=10,\n        model_name=\"bert\",\n        root=\"\"\n    ):\n        self.tokens = tokens\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n        self.model_name = model_name\n        self.words_per_split = words_per_split\n        self.margin = margin\n\n        self.article = load_text(id_, root=root)\n        \n        self.texts = self.article_to_texts()\n\n    def __len__(self):\n        return len(self.texts)\n    \n    def article_to_texts(self):\n        \"\"\"\n        Each article is divided into sections, \n        and then into subsets of self.words_per_split words\n        \"\"\"\n        texts = []\n        for section in self.article:\n            clean_section = clean_text(section['text']).split(' ')[:5000]  # only keep first 5k words\n            \n            for i in range(len(clean_section) // self.words_per_split + 1):\n                start = max(0, self.words_per_split * i - self.margin)\n                end = self.words_per_split * (i + 1) + self.margin\n                text = \" \".join(clean_section[start: end])\n                texts.append(text)\n            \n        return texts\n\n    def __getitem__(self, idx):\n        data = process_data(\n            self.texts[idx],\n            \"\",\n            self.tokenizer,\n            self.tokens,\n            max_len=self.max_len,\n            model_name=self.model_name,\n        )\n\n        return {\n            \"ids\": torch.tensor(data[\"ids\"], dtype=torch.long),\n            \"token_type_ids\": torch.tensor(data[\"token_type_ids\"], dtype=torch.long),\n            \"target_start\": torch.tensor(data[\"targets_start\"], dtype=torch.long),\n            \"target_end\": torch.tensor(data[\"targets_end\"], dtype=torch.long),\n            \"text\": data[\"text\"],\n            \"label\": data[\"label\"],\n            \"offsets\": torch.tensor(data[\"offsets\"], dtype=torch.long),\n        }","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import BertModel, BertConfig\n\nTRANSFORMERS = {   \n    \"bert-base-uncased\": (BertModel, \"bert-base-uncased\", BertConfig),\n}\n\n\nclass QATransformer(nn.Module):\n    \"\"\"\n    Simple model for Question Answering\n    \"\"\"\n    def __init__(self, model):\n        super().__init__()\n        self.name = model\n\n        self.pad_idx = 1 if \"roberta\" in self.name else 0\n\n        model_class, _, config_class = TRANSFORMERS[model]\n\n        try:\n            config = config_class.from_json_file(MODEL_PATHS[model] + 'bert_config.json')\n        except:\n            config = config_class.from_json_file(MODEL_PATHS[model] + 'config.json')\n        config.output_hidden_states = True\n\n        self.transformer =  model_class(config)\n\n        self.nb_features = self.transformer.pooler.dense.out_features\n\n        self.logits = nn.Sequential(\n            nn.Linear(self.nb_features, self.nb_features),\n            nn.Tanh(),\n            nn.Linear(self.nb_features, 2),\n        )\n\n    def forward(self, tokens, token_type_ids):\n        \"\"\"\n        Usual torch forward function\n\n        Arguments:\n            tokens {torch tensor} -- Sentence tokens\n            token_type_ids {torch tensor} -- Sentence tokens ids\n        \"\"\"\n\n        hidden_states = self.transformer(\n            tokens,\n            attention_mask=(tokens != self.pad_idx).long(),\n            token_type_ids=token_type_ids,\n        )[-1]\n\n        features = hidden_states[-1]\n        logits = self.logits(features)\n\n        start_logits, end_logits = logits[:, :, 0], logits[:, :, 1]\n\n        return start_logits, end_logits","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model_weights(model, filename, verbose=1, cp_folder=\"\"):\n    \"\"\"\n    Loads the weights of a PyTorch model. The exception handles cpu/gpu incompatibilities.\n\n    Args:\n        model (torch model): Model to load the weights to.\n        filename (str): Name of the checkpoint.\n        verbose (int, optional): Whether to display infos. Defaults to 1.\n        cp_folder (str, optional): Folder to load from. Defaults to \"\".\n\n    Returns:\n        torch model: Model with loaded weights.\n    \"\"\"\n\n    if verbose:\n        print(f\"\\n -> Loading weights from {os.path.join(cp_folder,filename)}\\n\")\n    try:\n        model.load_state_dict(os.path.join(cp_folder, filename), strict=True)\n    except BaseException:\n        model.load_state_dict(\n            torch.load(os.path.join(cp_folder, filename), map_location=\"cpu\"),\n            strict=True,\n        )\n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom torch.utils.data import DataLoader\n\n\ndef predict(model, dataset, batch_size=32):\n    \"\"\"\n    Usual predict torch function\n\n    Arguments:\n        model {torch model} -- Model to predict with\n        dataset {torch dataset} -- Dataset to get predictions from\n\n    Keyword Arguments:\n        batch_size {int} -- Batch size (default: {32})\n\n    Returns:\n        numpy array -- Predictions\n    \"\"\"\n\n    model.eval()\n    start_probas = []\n    end_probas = []\n\n    loader = DataLoader(\n        dataset, batch_size=batch_size, shuffle=False, num_workers=NUM_WORKERS\n    )\n\n    with torch.no_grad():\n        for data in loader:\n            ids, token_type_ids = data[\"ids\"], data[\"token_type_ids\"]\n\n            start_logits, end_logits = model(\n                ids.cuda(), token_type_ids.cuda()\n            )\n\n            start_probs = torch.softmax(start_logits, dim=1).cpu().detach().numpy()\n            end_probs = torch.softmax(end_logits, dim=1).cpu().detach().numpy()\n\n            for s, e in zip(start_probs, end_probs):\n                start_probas.append(list(s))\n                end_probas.append(list(e))\n\n    return start_probas, end_probas","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_string_from_idx(text, idx_start, idx_end, offsets):\n    \"\"\"\n    Uses the offsets to retrieve the predicted string based on the start and end indices\n    \"\"\"\n    if idx_end < idx_start:\n        idx_end = idx_start\n\n    predicted_string = \"\"\n    for i in range(idx_start, idx_end + 1):\n        predicted_string += text[offsets[i][0]: offsets[i][1]]\n        if i + 1 < len(offsets) and offsets[i][1] < offsets[i + 1][0]:\n            predicted_string += \" \"\n\n    return predicted_string\n\n\ndef get_pred_from_probas(dataset, start_probas, end_probas, threshold=0.):\n    preds = []\n    for i in range(len(dataset)):\n        if start_probas[i].max() > threshold or end_probas[i].max() > threshold:\n            start_idx = np.argmax(start_probas[i])\n            end_idx = np.argmax(end_probas[i])\n            if start_idx < end_idx and end_idx - start_idx < 10:\n                # print(start_idx, end_idx)\n                data = dataset[i]\n                preds.append(get_string_from_idx(data[\"text\"], start_idx, end_idx, data[\"offsets\"]))\n\n    return preds","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def post_process(preds):\n    \"\"\"\n    Naive processing of prediction : \n    Remove duplicates and convert to expected format.\n    \"\"\"\n    preds = np.unique(preds)\n    return \"|\".join(preds)\n\n\ndef k_fold_inference(config, df, tokenizer, tokens, weights, threshold=0.):\n    models = []\n    for w in weights:\n        model = QATransformer(config.selected_model).cuda()\n        model.zero_grad()\n        load_model_weights(model, w)\n        models.append(model)\n\n    preds = []\n    for text_id in tqdm(df['Id']):\n\n        dataset = ArticleDataset(\n            text_id,\n            tokenizer,\n            tokens,\n            max_len=512,\n            model_name=\"bert\",\n            root=DATA_PATH_TEST\n        )\n\n        start_probas, end_probas = [], []\n        for model in models:\n            start_proba, end_proba = predict(\n                model, \n                dataset, \n                batch_size=config.batch_size_val, \n            )\n            start_probas.append(start_proba)\n            end_probas.append(end_proba)\n\n        start_probas = np.mean(start_probas, 0)\n        end_probas = np.mean(end_probas, 0)\n        \n        # here can do some FOPP\n        pred = get_pred_from_probas(dataset, start_probas, end_probas, threshold=threshold)\n        preds.append(post_process(pred))\n            \n    return preds","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = Config\ndf = pd.read_csv(DATA_PATH + 'sample_submission.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer, tokens = create_tokenizer_and_tokens(config)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = ArticleDataset(\n    df['Id'][0],\n    tokenizer,\n    tokens,\n    max_len=512,\n    model_name=\"bert\",\n    root=DATA_PATH_TEST,\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = [sorted(glob.glob(CP_PATH + \"*.pt\"))[-1]] # -> list(model_paths)\nweights","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"qa_preds_model = k_fold_inference(\n    config,\n    df,\n    tokenizer,\n    tokens,\n    weights,\n    threshold=THRESHOLD,\n)\n\nqa_preds_model[:4]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MLM","metadata":{}},{"cell_type":"code","source":"PRED_TH = 2.0\nFL_TH = 1.0\n\nMODEL_PATH_PREFIX = '../input/coleridge-bert-mlm-external-pseudo-labels-v1'\nMLM_PRETRAINED_PATH = 'mlm-model'\nTOKENIZER = 'model_tokenizer'\n\nLENGTH = 1\nMAX_LENGTH = 64\nOVERLAP = 20\n\nPREDICT_BATCH = 32 # a higher value requires higher GPU memory usage\n\nDATASET_SYMBOL = '$' # this symbol represents a dataset name\nNONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'W_MLM' in HOW:\n    TOKENIZER_PATH = os.path.join(MODEL_PATH_PREFIX, TOKENIZER)\n    PRETRAINED_PATH = os.path.join(MODEL_PATH_PREFIX, MLM_PRETRAINED_PATH)\n    \n    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, use_fast=True)\n    model = AutoModelForMaskedLM.from_pretrained(PRETRAINED_PATH)\n\n    mlm = pipeline(\n        'fill-mask', \n        model=model,\n        tokenizer=tokenizer,\n        device=0 if torch.cuda.is_available() else -1\n    )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset'}\ndef find_mask_candidates(sentence):\n    \"\"\"\n    Extract masking candidates for Masked Dataset Modeling from a given $sentence.\n    A candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence.\n    \"\"\"\n    def candidate_qualified(words):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n            candidates.append((phrase_start, phrase_end))\n    \n    return candidates","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'W_MLM' in HOW:\n    mask = mlm.tokenizer.mask_token\n    all_test_data = []\n    \n    for paper_id in tqdm(sample_submission['Id']):\n        # load paper\n        paper = papers[paper_id]\n\n        # extract sentences\n        sentences = set([clean_paper_sentence(sentence) for section in paper \n                         for sentence in section['text'].split('.')\n                        ])\n        sentences = shorten_sentences(sentences) # make sentences short\n        sentences = [sentence for sentence in sentences if len(sentence) > LENGTH] # only accept sentences with length > 10 chars\n        sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        sentences = [sentence.split() for sentence in sentences] # sentence = list of words\n\n        # mask\n        test_data = []\n        for sentence in sentences:\n            for phrase_start, phrase_end in find_mask_candidates(sentence):\n                dt_point = sentence[:phrase_start] + [mask] + sentence[phrase_end+1:]\n                test_data.append((' '.join(dt_point), ' '.join(sentence[phrase_start:phrase_end+1]))) # (masked text, phrase)\n\n        all_test_data.append(test_data)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if 'W_MLM' in HOW:\n    pred_mlm_labels = []\n\n    for test_data in tqdm(all_test_data):\n        pred_bag = set()\n\n        if len(test_data):\n            texts, phrases = list(zip(*test_data))\n            mlm_pred = []\n            for p_id in range(0, len(texts), PREDICT_BATCH):\n                batch_texts = texts[p_id:p_id+PREDICT_BATCH]\n                batch_pred = mlm(list(batch_texts), targets=[f' {DATASET_SYMBOL}', f' {NONDATA_SYMBOL}'])\n                \n                if len(batch_texts) == 1:\n                    batch_pred = [batch_pred]\n                    \n                mlm_pred.extend(batch_pred)\n\n            for pred_th in np.arange(PRED_TH, 1.0, -0.05): # find-more PP\n                if len(pred_bag) == 0:\n                    for (result1, result2), phrase in zip(mlm_pred, phrases):\n                        if (result1['score'] > result2['score'] * pred_th and result1['token_str'] == DATASET_SYMBOL) or\\\n                           (result2['score'] > result1['score'] * pred_th and result2['token_str'] == NONDATA_SYMBOL):\n                            pred_bag.add(clean_text(phrase))\n                else: break\n\n        # filter labels by jaccard score \n        filtered_labels = []\n        for label in sorted(pred_bag, key=len, reverse=True): # long to short so that we keep the potential best\n            if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < FL_TH for got_label in filtered_labels):\n                filtered_labels.append(label)\n\n        pred_mlm_labels.append('|'.join(filtered_labels))\n        \n    print(f'pred_mlm_labels[:4] w/ PT{PRED_TH} FL{FL_TH}: \\n{pred_mlm_labels[:4]}')\n        \nelse: print('MLM is not used.')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Blending","metadata":{}},{"cell_type":"code","source":"def model_pred_blending(major_pred: list, second_pred: list) -> list:\n    final_predictions = []\n    for pred_ner, perd_qa in tqdm(zip(major_pred, second_pred)):\n        # NER\n        if pred_ner and not perd_qa:\n            final_predictions.append(pred_ner)\n        # QA \n        elif perd_qa and not pred_ner:\n            final_predictions.append(perd_qa)\n        # All\n        elif pred_ner and perd_qa:\n            ner_labels, qa_labels = pred_ner.split('|'), perd_qa.split('|')\n            all_temp = ner_labels.copy()\n            for qa_label in qa_labels:\n                if all( jaccard(ner_label, qa_label) < W_QA_TH for ner_label in ner_labels ):\n                    all_temp.append(qa_label)\n            final_predictions.append('|'.join(all_temp))\n        # None\n        else:\n            final_predictions.append(pred_ner)\n    return final_predictions","metadata":{"execution":{"iopub.status.busy":"2021-06-05T00:24:19.7385Z","iopub.execute_input":"2021-06-05T00:24:19.739158Z","iopub.status.idle":"2021-06-05T00:24:19.754099Z","shell.execute_reply.started":"2021-06-05T00:24:19.739005Z","shell.execute_reply":"2021-06-05T00:24:19.752913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = []\n\nif 'NER_ONLY' in HOW:\n    final_predictions = filtered_bert_labels\n\nelif 'W_QA' in HOW:\n    if 'W_MATCH' not in HOW: literal_preds = [None] * len(filtered_bert_labels)\n    if 'DOUBLE_BLENDING' in HOW:\n        ner_qa_predictions = model_pred_blending(filtered_bert_labels, qa_preds_model)\n        final_predictions = model_pred_blending(literal_preds, ner_qa_predictions)\n    else:\n        for pred_ner, perd_qa, literal_match in tqdm(zip(filtered_bert_labels, qa_preds_model, literal_preds)):\n            # MATCH\n            if literal_match:\n                final_predictions.append(literal_match)\n            # NER\n            elif pred_ner and not perd_qa:\n                final_predictions.append(pred_ner)\n            # QA \n            elif perd_qa and not pred_ner:\n                final_predictions.append(perd_qa)\n            # All\n            elif pred_ner and perd_qa:\n                ner_labels, qa_labels = pred_ner.split('|'), perd_qa.split('|')\n                all_temp = ner_labels.copy()\n                for qa_label in qa_labels:\n                    if all( jaccard(ner_label, qa_label) < W_QA_TH for ner_label in ner_labels ):\n                        all_temp.append(qa_label)\n                final_predictions.append('|'.join(all_temp))\n            # None\n            else:\n                final_predictions.append(pred_ner)\n    \nelif 'W_MATCH' in HOW:\n    for literal_match, ner_pred in zip(literal_preds, filtered_bert_labels):\n        if literal_match:\n            final_predictions.append(literal_match)\n        else:\n            final_predictions.append(ner_pred)\n\nelif 'W_MLM' in HOW:\n    for ner_pred, mlm_pred in zip(filtered_bert_labels, pred_mlm_labels):\n        if ner_pred:\n            final_predictions.append(ner_pred)\n        else:\n            final_predictions.append(mlm_pred)\n            \n# if 'W_MLM' in HOW:\n#     for index, (ner_qa_pred, mlm_pred) in enumerate( zip(final_predictions, pred_mlm_labels) ):\n#         if not ner_qa_pred:\n#             final_predictions[index] = mlm_pred\n\nfinal_predictions[:4]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.to_csv(f'submission.csv', index=False)\nsample_submission.head()","metadata":{},"execution_count":null,"outputs":[]}]}