{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is a template on using Literal Matching + Masked Language Modeling to identify datasets in papers.\n\nThe training of the Bert model was done in another notebook: [\\[Coleridge\\] BERT - Masked Dataset Modeling.](https://www.kaggle.com/tungmphung/coleridge-bert-masked-dataset-modeling)\n\nThe approach is:\n- Locate all the sequences of capitalized words (these sequences may contain some stopwords),\n- Replace each sequence with one of 2 special symbols (e.g. $ and #), implying if that sequence represents a dataset name or not.\n- Have the model learn the MLM task.","metadata":{}},{"cell_type":"markdown","source":"# Install packages","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-20T09:30:31.542864Z","iopub.execute_input":"2021-05-20T09:30:31.543211Z","iopub.status.idle":"2021-05-20T09:31:59.504081Z","shell.execute_reply.started":"2021-05-20T09:30:31.543142Z","shell.execute_reply":"2021-05-20T09:31:59.503091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\nfrom functools import partial\nimport string\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline, AutoModelForSequenceClassification\n\nsns.set()\nrandom.seed(123)\nnp.random.seed(456)\ntorch.manual_seed(2021)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\nfrom nltk.corpus import stopwords\nfrom unidecode import unidecode\n\nSTOPWORDS = set(stopwords.words('english'))\nREMOVE_MATCH = False\nMASKEDLM = True","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:31:59.50591Z","iopub.execute_input":"2021-05-20T09:31:59.506162Z","iopub.status.idle":"2021-05-20T09:32:07.860783Z","shell.execute_reply.started":"2021-05-20T09:31:59.506139Z","shell.execute_reply":"2021-05-20T09:32:07.859799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntrain = pd.read_csv(train_path)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:07.862094Z","iopub.execute_input":"2021-05-20T09:32:07.862431Z","iopub.status.idle":"2021-05-20T09:32:07.987127Z","shell.execute_reply.started":"2021-05-20T09:32:07.862402Z","shell.execute_reply":"2021-05-20T09:32:07.986155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if True:\n    sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\n    sample_submission = pd.read_csv(sample_submission_path)\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\nelse:\n    sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\n    sample_submission = pd.read_csv(sample_submission_path)[-1500:-1000]\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:07.988168Z","iopub.execute_input":"2021-05-20T09:32:07.98845Z","iopub.status.idle":"2021-05-20T09:32:08.03299Z","shell.execute_reply.started":"2021-05-20T09:32:07.988426Z","shell.execute_reply":"2021-05-20T09:32:08.032263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal Matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"existing_labels = set(np.load('../input/showdata-labels1/existing_labels.npy', allow_pickle = True).tolist())\ndf2=pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\ntrain = pd.read_csv('../input/showdata-labels1/cleaned_train.csv')\n    \nprint(f'No. different labels: {len(existing_labels)}')\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:08.035852Z","iopub.execute_input":"2021-05-20T09:32:08.036114Z","iopub.status.idle":"2021-05-20T09:32:25.211832Z","shell.execute_reply.started":"2021-05-20T09:32:08.03609Z","shell.execute_reply":"2021-05-20T09:32:25.211147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"existing_labels","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:25.213587Z","iopub.execute_input":"2021-05-20T09:32:25.214034Z","iopub.status.idle":"2021-05-20T09:32:25.221312Z","shell.execute_reply.started":"2021-05-20T09:32:25.214004Z","shell.execute_reply":"2021-05-20T09:32:25.220723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\n## Functions for literal matching\n\ndef json_to_text(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data\n    \ndef text_cleaning(text):\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\ntqdm.pandas()\nsample_submission['text'] = sample_submission['Id'].progress_apply(partial(json_to_text, train_files_path=paper_test_folder))\n\ndef read_json_pub(filename, train_data_path=paper_test_folder, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:25.222358Z","iopub.execute_input":"2021-05-20T09:32:25.22278Z","iopub.status.idle":"2021-05-20T09:32:25.248582Z","shell.execute_reply.started":"2021-05-20T09:32:25.222752Z","shell.execute_reply":"2021-05-20T09:32:25.247123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not REMOVE_MATCH:\n    import gc\n    column_names = [\"Id\", \"PredictionString\"]\n\n    submission = pd.DataFrame(columns = column_names)\n\n    to_append=[]\n    for index, row in sample_submission.iterrows():\n        to_append=[row['Id'],'']\n        large_string = str(read_json_pub(row['Id'],paper_test_folder))\n        clean_string=text_cleaning(large_string)\n        for index, row2 in df2.iterrows():\n            query_string = str(row2['title'])\n            if query_string in clean_string:\n                if to_append[1]!='' and clean_text(query_string) not in to_append[1]:\n                    to_append[1]=to_append[1]+'|'+clean_text(query_string)\n                if to_append[1]=='':\n                    to_append[1]=clean_text(query_string)\n        df_length = len(submission)\n        submission.loc[df_length] = to_append\n    submission.to_csv('submission.csv', index = False)\n    literal_preds = submission['PredictionString'].tolist()\n    gc.collect()\nelse:\n    literal_preds = [[] for _ in range(len(sample_submission))]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:25.249761Z","iopub.execute_input":"2021-05-20T09:32:25.25003Z","iopub.status.idle":"2021-05-20T09:32:26.644236Z","shell.execute_reply.started":"2021-05-20T09:32:25.250005Z","shell.execute_reply":"2021-05-20T09:32:26.643253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# literal_preds = []\n\ndef predict(sample_sub):\n    id_list = []\n    labels_list = []\n    for index, row in tqdm(sample_sub.iterrows()):\n        sample_text = row['text']\n        row_id = row['Id']\n        if not REMOVE_MATCH:\n            clean_string = text_cleaning(sample_text)\n            temp_df = train[train['text'] == clean_string]\n            cleaned_labels = temp_df['cleaned_label'].to_list()\n            #for known_label in existing_labels:\n            #    if known_label in sample_text.lower():\n            #        if known_label not in STOPWORDS:\n            #            if len(known_label)>1:\n            #                cleaned_labels.append(clean_text(known_label))\n            for index, row2 in df2.iterrows():\n                query_string = str(row2['title'])\n                if query_string in clean_string:\n                    cleaned_labels.append(clean_text(query_string))\n            cleaned_labels = [clean_text(x) for x in cleaned_labels]\n            cleaned_labels = set(cleaned_labels)\n        else:\n            cleaned_labels = []\n        labels_list.append('|'.join(cleaned_labels))\n        id_list.append(row_id)\n    return (id_list,labels_list)\n\n# Z=predict(sample_submission)\n# literal_preds = Z[1]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:26.64539Z","iopub.execute_input":"2021-05-20T09:32:26.645708Z","iopub.status.idle":"2021-05-20T09:32:26.652757Z","shell.execute_reply.started":"2021-05-20T09:32:26.645679Z","shell.execute_reply":"2021-05-20T09:32:26.651944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Masked Dataset Modeling","metadata":{}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"#PRETRAINED_PATH = '../input/coleridgemlmwithrep1/mlm-model'\n#TOKENIZER_PATH = '../input/coleridgemlmwithrep1/model_tokenizer'\nPRETRAINED_PATH = '../input/coleridgemlmdeberta/mlm-model'\nTOKENIZER_PATH = '../input/coleridgemlmdeberta/model_tokenizer'\n\nMAX_LENGTH = 64\nOVERLAP = 20\n\nPREDICT_BATCH = 32 # a higher value requires higher GPU memory usage\n\nDATASET_SYMBOL = '$' # this symbol represents a dataset name\nNONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:26.65361Z","iopub.execute_input":"2021-05-20T09:32:26.654086Z","iopub.status.idle":"2021-05-20T09:32:26.668424Z","shell.execute_reply.started":"2021-05-20T09:32:26.65406Z","shell.execute_reply":"2021-05-20T09:32:26.667674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data to MLM format","metadata":{}},{"cell_type":"markdown","source":"### Load model and tokenizer","metadata":{}},{"cell_type":"code","source":"# During training, the suspect phrases (i.e. uppercase/ connection words) are replaced with either \n# $ denoting dataset or # denoting not dataset, and the phrases are randomly masked - the model is\n# then tasked to predict the masked words. In the inference phase, the suspect phrases (this time # \n# or $ is unknown) are masked and the model infers either # or $. \n\n# The fill-mask pipeline has an argument target_tokens when called which indicates what the valid \n# tokens for prediction are.\n\ntokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, use_fast=True)\n\n\nif MASKEDLM:\n    model = AutoModelForMaskedLM.from_pretrained(PRETRAINED_PATH)\n    mlm = pipeline(\n        'fill-mask', \n        model=model,\n        tokenizer=tokenizer,\n        device=0 if torch.cuda.is_available() else -1\n        )\nelse:\n    model = AutoModelForSequenceClassification.from_pretrained(PRETRAINED_PATH)\n    snt = pipeline(\n        'sentiment-analysis', \n        model=model,\n        tokenizer=tokenizer,\n        device=0 if torch.cuda.is_available() else -1\n        )","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:26.66949Z","iopub.execute_input":"2021-05-20T09:32:26.669764Z","iopub.status.idle":"2021-05-20T09:32:38.024621Z","shell.execute_reply.started":"2021-05-20T09:32:26.669731Z","shell.execute_reply":"2021-05-20T09:32:38.023825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Auxiliary functions","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\n# connection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset'}\ndef find_mask_candidates_old(sentence):\n    \"\"\"\n    Extract masking candidates for Masked Dataset Modeling from a given $sentence.\n    A candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence.\n    \"\"\"\n    def candidate_qualified(words):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n            candidates.append((phrase_start, phrase_end))\n    \n    return candidates\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset', 'survey', 'study','sequence'}\nprep_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'this', 'we', 'their', 'it', 'to'}\ndef find_mask_candidates(sentence):\n    \"\"\"\n    Extract negative samples for Masked Dataset Modeling from a given $sentence.\n    A negative candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence. Lastly, the sequence must be quite different to any of the \n    ground truth labels (measured by Jaccard similarity).\n    \"\"\"\n    def candidate_qualified(words):\n        # remove beginning words that are connection_tokens except data/dataset\n        startIdx = 0\n        endIdx = 0\n        while len(words) and words[0].lower() in prep_tokens:\n            words = words[1:]\n            startIdx +=1\n        # remove ending words that are connection_tokens\n        while len(words) and words[-1].lower() in prep_tokens:\n            words = words[:-1]\n            endIdx+=1\n        # comparison without connection_tokens\n        ### NOTE SCHANGE TO 2 FROM 3, i.e. 3 word labels accepted\n        if (len(words) <= 3 or \\\n            sum([1 for word in words if not word.isnumeric()])<=2):\n            return False, []\n        elif len(words)==4 and words[-1].isnumeric() and words[1] == 'and':\n            # to get rid of references, e.g. Johnson and Johnson 2018\n            return False, []\n        else:\n            return True, [startIdx, endIdx]\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        # if word is captial or connection token\n        # if word[0].isupper() or word in connection_tokens:\n        if word[0].isupper() or (word[0].isnumeric() and len(word)>2):\n            # set as phrase start if phrase start doesn't exist, if not set as end\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        elif word not in connection_tokens:\n            # if current phrase fulfils dissimilarity requirement, reset phrase_start\n            if phrase_start != -1:\n                qualified, tmpidxs = candidate_qualified(sentence[phrase_start:phrase_end+1])\n                if qualified:\n                    candidates.append((phrase_start+tmpidxs[0], phrase_end-tmpidxs[1]))\n                phrase_start = phrase_end = -1\n    \n    # to deal with case where phrase end is last word\n    if phrase_start != -1:\n        qualified, tmpidxs = candidate_qualified(sentence[phrase_start:phrase_end+1])\n        if qualified:\n            candidates.append((phrase_start+tmpidxs[0], phrase_end-tmpidxs[1]))\n    \n    return candidates\n\ndef pre_tokenize(sentence):\n    try:\n        sentence = sentence.split()\n    except:\n        pass\n    wordlist = ['university', 'initiative','international','information']\n    for i in range(len(sentence)):\n        word  = sentence[i]\n        if word.isupper():\n            sentence[i] = '#'\n        elif word[0].isupper() and len(word)>8 and word.lower() not in wordlist:\n            sentence[i] = '$'\n    return sentence\n\ndef pre_tokenize_mask(sentence):\n    try:\n        sentence = sentence.split()\n    except:\n        pass\n    wordlist = ['university', 'initiative','international','information']\n    for i in range(len(sentence)):\n        word  = sentence[i]\n        if word.isupper() and not word == '[MASK]':\n            sentence[i] = 'XXXX'\n        elif word[0].isupper() and len(word)>8 and word.lower() not in wordlist:\n            sentence[i] = 'ZZZZ'\n    return sentence","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:38.026247Z","iopub.execute_input":"2021-05-20T09:32:38.026637Z","iopub.status.idle":"2021-05-20T09:32:38.057159Z","shell.execute_reply.started":"2021-05-20T09:32:38.026601Z","shell.execute_reply":"2021-05-20T09:32:38.055786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform","metadata":{}},{"cell_type":"code","source":"if MASKEDLM:\n    mask = mlm.tokenizer.mask_token","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:38.058696Z","iopub.execute_input":"2021-05-20T09:32:38.059239Z","iopub.status.idle":"2021-05-20T09:32:38.07232Z","shell.execute_reply.started":"2021-05-20T09:32:38.059195Z","shell.execute_reply":"2021-05-20T09:32:38.071423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_test_data = []\ndN = 5\nfor paper_id in sample_submission['Id']:\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = set([clean_paper_sentence(sentence) for section in paper \n                     for sentence in section['text'].split('.')\n                    ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n    sentences = [sentence.split() for sentence in sentences] # sentence = list of words\n    \n    # mask\n    test_data = []\n    for sentence in sentences:\n        for phrase_start, phrase_end in find_mask_candidates(sentence):\n            if MASKEDLM:\n                dt_point = sentence[:phrase_start] + ['@'] + sentence[phrase_start:phrase_end+1] + [mask] + sentence[phrase_end+1:]\n                dt_point = pre_tokenize_mask(dt_point)\n                # dt_point = sentence[:phrase_start] + [mask] + sentence[phrase_end+1:]\n                test_data.append((' '.join(dt_point), ' '.join(sentence[phrase_start:phrase_end+1]))) # (masked text, phrase)\n            else:\n                dt_point = pre_tokenize(sentence[max(phrase_start-dN,0):phrase_end+1+dN])\n                test_data.append((' '.join(dt_point), sentence[phrase_start:phrase_end+1]))\n    \n    all_test_data.append(test_data)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:32:38.074044Z","iopub.execute_input":"2021-05-20T09:32:38.07432Z","iopub.status.idle":"2021-05-20T09:32:38.153789Z","shell.execute_reply.started":"2021-05-20T09:32:38.074295Z","shell.execute_reply":"2021-05-20T09:32:38.153071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predict","metadata":{}},{"cell_type":"code","source":"pred_labels = []\n\npbar = tqdm(total = len(all_test_data))\n# each iteration is one ID, test_data contains a list of tuples, 1st member of tuple\n# is the masked sentence, second member is the phrase\nfor i,test_data in enumerate(all_test_data):\n    pred_bag = set()\n    \n    if len(test_data):\n        texts, phrases = list(zip(*test_data))\n        preds = []\n        # iterate through sentences for that ID\n        for p_id in range(0, len(texts), PREDICT_BATCH):\n            batch_texts = texts[p_id:p_id+PREDICT_BATCH]\n            # since there are two target tokens, batch_pred is a list of two dicts, each\n            # containing the token identity and the corresponding prob score\n            if MASKEDLM:\n                batch_pred = mlm(list(batch_texts), targets=[f' {DATASET_SYMBOL}', f' {NONDATA_SYMBOL}',])\n            else:\n                batch_pred = snt(list(batch_texts))\n            \n            if len(batch_texts) == 1 and MASKEDLM:\n                batch_pred = [batch_pred]\n            \n            preds.extend(batch_pred)\n        \n        if MASKEDLM:\n            # append phrase to pred_bag if probability score corresponding to DATASET_SYMBOL is\n            # sufficiently high - pred_bag contains all predictions for a particular ID\n            for (result1, result2), phrase in zip(preds, phrases):\n                # print(result1)\n                if (result1['score'] > result2['score']*2 and any(result1['token_str'] == x for x in [DATASET_SYMBOL,'(#!'])) or\\\n                   (result2['score'] > result1['score']*2 and any(result2['token_str'] == x for x in [DATASET_SYMBOL,'(#!'])):\n                    pred_bag.add(clean_text(phrase))\n        else:\n            for lbl, phrase in zip(preds,phrases):\n                if lbl['label']=='LABEL_1':\n                    pred_bag.add(clean_text(phrase))\n    # filter labels by jaccard score \n    filtered_labels = []\n    # starting with longest labels, label is accepted only if there are no previously\n    # accepted labels or if label is sufficiently dissimilar to other accepted labels\n    for label in sorted(pred_bag, key=len, reverse=True):\n        if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered_labels):\n            if any(jaccard_similarity(label, got_label) > 0.5 for got_label in existing_labels) and REMOVE_MATCH:\n                continue\n            filtered_labels.append(label)\n            \n    pred_labels.append('|'.join(filtered_labels))\n    pbar.update(1)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-05-20T09:32:38.1549Z","iopub.execute_input":"2021-05-20T09:32:38.155323Z","iopub.status.idle":"2021-05-20T09:33:00.904145Z","shell.execute_reply.started":"2021-05-20T09:32:38.155293Z","shell.execute_reply":"2021-05-20T09:33:00.901231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_pred","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:00.906971Z","iopub.execute_input":"2021-05-20T09:33:00.907743Z","iopub.status.idle":"2021-05-20T09:33:00.921893Z","shell.execute_reply.started":"2021-05-20T09:33:00.9077Z","shell.execute_reply":"2021-05-20T09:33:00.919854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if MASKEDLM:\n    preds\nelse:\n    for lbl, phrase in zip(preds,phrases):\n        print('Label: '+ str(lbl['label'])+ '-> '+str(lbl['score']) +'; string:'+ clean_text(phrase))","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:00.92385Z","iopub.execute_input":"2021-05-20T09:33:00.924379Z","iopub.status.idle":"2021-05-20T09:33:00.936879Z","shell.execute_reply.started":"2021-05-20T09:33:00.924352Z","shell.execute_reply":"2021-05-20T09:33:00.934692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:00.938858Z","iopub.execute_input":"2021-05-20T09:33:00.939263Z","iopub.status.idle":"2021-05-20T09:33:00.957554Z","shell.execute_reply.started":"2021-05-20T09:33:00.939209Z","shell.execute_reply":"2021-05-20T09:33:00.955382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"def combPred(match, modPred):\n    pred = match.split('|')\n    modPred = modPred.split('|')\n    match = match.split('|')\n    for p in modPred:\n        if all(jaccard_similarity(p, lbl) < 0.5 for lbl in match):\n            if any(x in p for x in ['noaa ', 'nsf ', 'genome sequence', 'rsna']) or ('survey' in p.split()[-2:]):\n                pred.append(p)\n    return '|'.join(pred)\n\ndef filterPred(modPred):\n    modPred = modPred.split('|')\n    out = []\n    for p in modPred:\n        if len(p.split())>10:\n            continue\n        elif any(x in p for x in ['table', ' all ', 'study group','data center', 'fig', ' s1', ' s2', ' s3' ' s4']):\n            continue\n        elif any((x.isnumeric() and (len(x)>4 or len(x)==3)) for x in p.split()):\n            continue\n        elif len(p)>0 and max(map(len,p.split()))<=2:\n            continue\n        else:\n            out.append(p)\n    out = '|'.join(out)\n    return out\n\nfinal_predictions = []\npred_labels_filt = [filterPred(x) for x in pred_labels]\nfor literal_match, mlm_pred in zip(literal_preds, pred_labels_filt):    \n    if literal_match and not REMOVE_MATCH:\n        if True:\n            literal_match = combPred(literal_match, mlm_pred)\n        final_predictions.append(literal_match)\n    else:\n        final_predictions.append(mlm_pred)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:00.959876Z","iopub.execute_input":"2021-05-20T09:33:00.960363Z","iopub.status.idle":"2021-05-20T09:33:00.988466Z","shell.execute_reply.started":"2021-05-20T09:33:00.960305Z","shell.execute_reply":"2021-05-20T09:33:00.9865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [x for x in pred_labels_filt if x]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:00.990876Z","iopub.execute_input":"2021-05-20T09:33:00.991475Z","iopub.status.idle":"2021-05-20T09:33:01.004496Z","shell.execute_reply.started":"2021-05-20T09:33:00.991397Z","shell.execute_reply":"2021-05-20T09:33:01.00258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission = sample_submission[['Id','PredictionString']]\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:01.007883Z","iopub.execute_input":"2021-05-20T09:33:01.008294Z","iopub.status.idle":"2021-05-20T09:33:01.028697Z","shell.execute_reply.started":"2021-05-20T09:33:01.008246Z","shell.execute_reply":"2021-05-20T09:33:01.026347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:01.030546Z","iopub.execute_input":"2021-05-20T09:33:01.030937Z","iopub.status.idle":"2021-05-20T09:33:01.048721Z","shell.execute_reply.started":"2021-05-20T09:33:01.030909Z","shell.execute_reply":"2021-05-20T09:33:01.047616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmpDf = pd.read_csv('../input/training-set-found-labels/FNE_plusAbbrevs.csv')\ntmpDf.loc[tmpDf['Dataset']==1]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:01.049873Z","iopub.execute_input":"2021-05-20T09:33:01.050187Z","iopub.status.idle":"2021-05-20T09:33:01.36424Z","shell.execute_reply.started":"2021-05-20T09:33:01.050158Z","shell.execute_reply":"2021-05-20T09:33:01.363045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[x for x in final_predictions if len(x)>0]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:01.366586Z","iopub.execute_input":"2021-05-20T09:33:01.367291Z","iopub.status.idle":"2021-05-20T09:33:01.374576Z","shell.execute_reply.started":"2021-05-20T09:33:01.367247Z","shell.execute_reply":"2021-05-20T09:33:01.373814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[x for x in pred_labels_filt if len(x)>0]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:01.375724Z","iopub.execute_input":"2021-05-20T09:33:01.376198Z","iopub.status.idle":"2021-05-20T09:33:01.388255Z","shell.execute_reply.started":"2021-05-20T09:33:01.376167Z","shell.execute_reply":"2021-05-20T09:33:01.386945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:33:01.389713Z","iopub.execute_input":"2021-05-20T09:33:01.390134Z","iopub.status.idle":"2021-05-20T09:33:01.402593Z","shell.execute_reply.started":"2021-05-20T09:33:01.390106Z","shell.execute_reply":"2021-05-20T09:33:01.401521Z"},"trusted":true},"execution_count":null,"outputs":[]}]}