{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is a template on using Literal Matching + Masked Language Modeling to identify datasets in papers.\n\nThe training of the Bert model was done in another notebook: [\\[Coleridge\\] BERT - Masked Dataset Modeling.](https://www.kaggle.com/tungmphung/coleridge-bert-masked-dataset-modeling)\n\nThe approach is:\n- Locate all the sequences of capitalized words (these sequences may contain some stopwords),\n- Replace each sequence with one of 2 special symbols (e.g. $ and #), implying if that sequence represents a dataset name or not.\n- Have the model learn the MLM task.","metadata":{"papermill":{"duration":0.018346,"end_time":"2021-04-11T04:38:09.491631","exception":false,"start_time":"2021-04-11T04:38:09.473285","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Install packages","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.016779,"end_time":"2021-04-11T04:38:09.567513","exception":false,"start_time":"2021-04-11T04:38:09.550734","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"_kg_hide-output":true,"papermill":{"duration":88.046883,"end_time":"2021-04-11T04:39:37.631399","exception":false,"start_time":"2021-04-11T04:38:09.584516","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{"papermill":{"duration":0.023417,"end_time":"2021-04-11T04:39:37.67913","exception":false,"start_time":"2021-04-11T04:39:37.655713","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline\n\nsns.set()\nrandom.seed(123)\nnp.random.seed(456)\ntorch.manual_seed(2021)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","metadata":{"papermill":{"duration":7.880855,"end_time":"2021-04-11T04:39:45.583436","exception":false,"start_time":"2021-04-11T04:39:37.702581","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{"papermill":{"duration":0.023324,"end_time":"2021-04-11T04:39:45.63063","exception":false,"start_time":"2021-04-11T04:39:45.607306","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)","metadata":{"papermill":{"duration":62.388997,"end_time":"2021-04-11T04:40:48.04337","exception":false,"start_time":"2021-04-11T04:39:45.654373","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\npapers = {}\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"papermill":{"duration":0.076843,"end_time":"2021-04-11T04:40:48.164636","exception":false,"start_time":"2021-04-11T04:40:48.087793","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal Matching","metadata":{"papermill":{"duration":0.046257,"end_time":"2021-04-11T04:40:48.253856","exception":false,"start_time":"2021-04-11T04:40:48.207599","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{"papermill":{"duration":0.040419,"end_time":"2021-04-11T04:40:48.339913","exception":false,"start_time":"2021-04-11T04:40:48.299494","status":"completed"},"tags":[]}},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"papermill":{"duration":0.339058,"end_time":"2021-04-11T04:40:48.715976","exception":false,"start_time":"2021-04-11T04:40:48.376918","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{"papermill":{"duration":0.026781,"end_time":"2021-04-11T04:40:48.7672","exception":false,"start_time":"2021-04-11T04:40:48.740419","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"papermill":{"duration":0.049515,"end_time":"2021-04-11T04:40:48.850503","exception":false,"start_time":"2021-04-11T04:40:48.800988","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds = []\n\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if label in text_1 or label in text_2:\n            labels.add(clean_text(label))\n    \n    literal_preds.append('|'.join(labels))","metadata":{"papermill":{"duration":0.208434,"end_time":"2021-04-11T04:40:49.096091","exception":false,"start_time":"2021-04-11T04:40:48.887657","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds[:5]","metadata":{"papermill":{"duration":0.034763,"end_time":"2021-04-11T04:40:49.155544","exception":false,"start_time":"2021-04-11T04:40:49.120781","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Masked Dataset Modeling","metadata":{"papermill":{"duration":0.024532,"end_time":"2021-04-11T04:40:49.205679","exception":false,"start_time":"2021-04-11T04:40:49.181147","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{"papermill":{"duration":0.024189,"end_time":"2021-04-11T04:40:49.254345","exception":false,"start_time":"2021-04-11T04:40:49.230156","status":"completed"},"tags":[]}},{"cell_type":"code","source":"PRETRAINED_PATH = '../input/coleridge-mlm-model/mlm-model'\nTOKENIZER_PATH = '../input/coleridge-mlm-model/model_tokenizer'\n\nMAX_LENGTH = 64\nOVERLAP = 20\n\nPREDICT_BATCH = 32 # a higher value requires higher GPU memory usage\n\nDATASET_SYMBOL = '$' # this symbol represents a dataset name\nNONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{"papermill":{"duration":0.031765,"end_time":"2021-04-11T04:40:49.31041","exception":false,"start_time":"2021-04-11T04:40:49.278645","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data to MLM format","metadata":{"papermill":{"duration":0.024232,"end_time":"2021-04-11T04:40:49.359019","exception":false,"start_time":"2021-04-11T04:40:49.334787","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Load model and tokenizer","metadata":{"papermill":{"duration":0.025745,"end_time":"2021-04-11T04:40:49.409385","exception":false,"start_time":"2021-04-11T04:40:49.38364","status":"completed"},"tags":[]}},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, use_fast=True)\nmodel = AutoModelForMaskedLM.from_pretrained(PRETRAINED_PATH)\n\nmlm = pipeline(\n    'fill-mask', \n    model=model,\n    tokenizer=tokenizer,\n    device=0 if torch.cuda.is_available() else -1\n)","metadata":{"papermill":{"duration":15.616478,"end_time":"2021-04-11T04:41:05.050779","exception":false,"start_time":"2021-04-11T04:40:49.434301","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Auxiliary functions","metadata":{"papermill":{"duration":0.025081,"end_time":"2021-04-11T04:41:05.102658","exception":false,"start_time":"2021-04-11T04:41:05.077577","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset'}\ndef find_mask_candidates(sentence):\n    \"\"\"\n    Extract masking candidates for Masked Dataset Modeling from a given $sentence.\n    A candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence.\n    \"\"\"\n    def candidate_qualified(words):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n            candidates.append((phrase_start, phrase_end))\n    \n    return candidates","metadata":{"papermill":{"duration":0.041024,"end_time":"2021-04-11T04:41:05.168795","exception":false,"start_time":"2021-04-11T04:41:05.127771","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform","metadata":{"papermill":{"duration":0.024622,"end_time":"2021-04-11T04:41:05.21842","exception":false,"start_time":"2021-04-11T04:41:05.193798","status":"completed"},"tags":[]}},{"cell_type":"code","source":"mask = mlm.tokenizer.mask_token","metadata":{"papermill":{"duration":0.030696,"end_time":"2021-04-11T04:41:05.273944","exception":false,"start_time":"2021-04-11T04:41:05.243248","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_test_data = []\n\n# pbar = tqdm(total = len(sample_submission))\nfor paper_id in sample_submission['Id']:\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = set([clean_paper_sentence(sentence) for section in paper \n                     for sentence in section['text'].split('.')\n                    ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n    sentences = [sentence.split() for sentence in sentences] # sentence = list of words\n    \n    # mask\n    test_data = []\n    for sentence in sentences:\n        for phrase_start, phrase_end in find_mask_candidates(sentence):\n            dt_point = sentence[:phrase_start] + [mask] + sentence[phrase_end+1:]\n            test_data.append((' '.join(dt_point), ' '.join(sentence[phrase_start:phrase_end+1]))) # (masked text, phrase)\n    \n    all_test_data.append(test_data)\n    \n    # process bar\n#     pbar.update(1)","metadata":{"papermill":{"duration":0.09657,"end_time":"2021-04-11T04:41:05.395301","exception":false,"start_time":"2021-04-11T04:41:05.298731","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predict","metadata":{"papermill":{"duration":0.025167,"end_time":"2021-04-11T04:41:05.445399","exception":false,"start_time":"2021-04-11T04:41:05.420232","status":"completed"},"tags":[]}},{"cell_type":"code","source":"pred_mlm_labels = []\n\npbar = tqdm(total = len(all_test_data))\nfor test_data in all_test_data:\n    pred_bag = set()\n    \n    if len(test_data):\n        texts, phrases = list(zip(*test_data))\n        mlm_pred = []\n        for p_id in range(0, len(texts), PREDICT_BATCH):\n            batch_texts = texts[p_id:p_id+PREDICT_BATCH]\n            batch_pred = mlm(list(batch_texts), targets=[f' {DATASET_SYMBOL}', f' {NONDATA_SYMBOL}'])\n            \n            if len(batch_texts) == 1:\n                batch_pred = [batch_pred]\n            \n            mlm_pred.extend(batch_pred)\n        \n        for (result1, result2), phrase in zip(mlm_pred, phrases):\n            if (result1['score'] > result2['score']*3 and result1['token_str'] == DATASET_SYMBOL) or\\\n               (result2['score'] > result1['score']*5 and result2['token_str'] == NONDATA_SYMBOL):\n                pred_bag.add(clean_text(phrase))\n    \n    # filter labels by jaccard score \n    filtered_labels = []\n    \n    for label in sorted(pred_bag, key=len, reverse=True):\n        if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered_labels):\n            filtered_labels.append(label)\n            \n    pred_mlm_labels.append('|'.join(filtered_labels))\n    pbar.update(1)","metadata":{"papermill":{"duration":2.752323,"end_time":"2021-04-11T04:41:08.222784","exception":false,"start_time":"2021-04-11T04:41:05.470461","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_mlm_labels[:5]","metadata":{"papermill":{"duration":0.034555,"end_time":"2021-04-11T04:41:08.28519","exception":false,"start_time":"2021-04-11T04:41:08.250635","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregate final predictions and write submission file","metadata":{"papermill":{"duration":0.027011,"end_time":"2021-04-11T04:41:08.339056","exception":false,"start_time":"2021-04-11T04:41:08.312045","status":"completed"},"tags":[]}},{"cell_type":"code","source":"final_predictions = []\nfor literal_match, mlm_pred in zip(literal_preds, pred_mlm_labels):\n    if literal_match:\n        final_predictions.append(literal_match)\n    else:\n        final_predictions.append(mlm_pred)","metadata":{"papermill":{"duration":0.033861,"end_time":"2021-04-11T04:41:08.400168","exception":false,"start_time":"2021-04-11T04:41:08.366307","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":0.036764,"end_time":"2021-04-11T04:41:08.464088","exception":false,"start_time":"2021-04-11T04:41:08.427324","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"papermill":{"duration":0.039858,"end_time":"2021-04-11T04:41:08.531064","exception":false,"start_time":"2021-04-11T04:41:08.491206","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.027932,"end_time":"2021-04-11T04:41:08.587169","exception":false,"start_time":"2021-04-11T04:41:08.559237","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}