{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook illustrates how to use Masked Language Modeling for this competition.\n\nObservation: most of the dataset names consist of only words with uppercased-first-letter and some stopwords like `on`, `in`, `and` (e.g. `Early Childhood Longitudinal Study`, `Trends in International Mathematics and Science Study`). \n\nThus, one approach to find the datasets is: \n- Locate all the sequences of capitalized words (these sequences may contain some stopwords), \n- Replace each sequence with one of 2 special symbols (e.g. `$` and `#`), implying if that sequence represents a dataset name or not.\n- Have the model learn the MLM task.","metadata":{}},{"cell_type":"markdown","source":"The code below shows how to train a model for that purpose with the help of the `huggingface`.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number (e.g. 50) for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:45:45.029852Z","iopub.execute_input":"2021-05-24T10:45:45.030432Z","iopub.status.idle":"2021-05-24T10:45:45.036885Z","shell.execute_reply.started":"2021-05-24T10:45:45.030301Z","shell.execute_reply":"2021-05-24T10:45:45.035859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-05-24T10:45:45.043667Z","iopub.execute_input":"2021-05-24T10:45:45.044154Z","iopub.status.idle":"2021-05-24T10:46:17.684957Z","shell.execute_reply.started":"2021-05-24T10:45:45.044114Z","shell.execute_reply":"2021-05-24T10:46:17.683648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline, AutoConfig, \\\nAutoModelForSequenceClassification, DataCollatorWithPadding\n\nsns.set()\nrandom.seed(123)\nnp.random.seed(456)\n\nMASKEDLM = True\nNER = False","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:17.687378Z","iopub.execute_input":"2021-05-24T10:46:17.687743Z","iopub.status.idle":"2021-05-24T10:46:27.170043Z","shell.execute_reply.started":"2021-05-24T10:46:17.6877Z","shell.execute_reply":"2021-05-24T10:46:27.168365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_label(label_list):\n    out = []\n    prep_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'this', 'we', 'their', 'it', 'to'}\n    for label in label_list:\n        if len(label.split())==1:\n            continue\n        elif len(label.split())==2 and not all(word.isupper() for word in label.split()):\n            continue\n        elif any(word[0].islower() for word in label.split()):\n            continue\n        elif '\\xad' in label:\n            continue\n        out.append(label)\n    return out\n\nimport itertools\nrccDf = pd.read_json('../input/rccfull/train_test/data_set_citations.json')[['publication_id','mention_list']]\nrccDf['mention_list'] = rccDf['mention_list'].map(filter_label)\nnewLabels=list(set(list(itertools.chain(*rccDf['mention_list'].tolist()))))","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:27.171573Z","iopub.execute_input":"2021-05-24T10:46:27.171922Z","iopub.status.idle":"2021-05-24T10:46:27.381958Z","shell.execute_reply.started":"2021-05-24T10:46:27.171881Z","shell.execute_reply":"2021-05-24T10:46:27.380797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef clean_text_multiple(line):\n    return '|'.join(clean_text(txt) for txt in line.split('|'))\n    \n\nrccDf = pd.read_json('../input/rccfull/train_test/data_set_citations.json')[['publication_id','mention_list']]\nrccDf['mention_list'] = rccDf['mention_list'].map(filter_label)\nrccDf['mention_list'] = rccDf['mention_list'].map('|'.join)\nrccDf = rccDf.rename(columns = {'publication_id':'Id', 'mention_list':'dataset_label'})\nrccDf['cleaned_label'] = rccDf['dataset_label'].map(clean_text_multiple)\nrccDf['dataset_title'] = rccDf['dataset_label']\nrccDf['pub_title'] = ''\nrccDf = rccDf.drop([i for i,j in enumerate(rccDf['dataset_label']=='') if j],axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:27.383447Z","iopub.execute_input":"2021-05-24T10:46:27.383817Z","iopub.status.idle":"2021-05-24T10:46:27.5239Z","shell.execute_reply.started":"2021-05-24T10:46:27.383781Z","shell.execute_reply":"2021-05-24T10:46:27.522762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rccDf","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:27.527163Z","iopub.execute_input":"2021-05-24T10:46:27.527639Z","iopub.status.idle":"2021-05-24T10:46:27.554946Z","shell.execute_reply.started":"2021-05-24T10:46:27.52759Z","shell.execute_reply":"2021-05-24T10:46:27.554128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_LENGTH = 64\nOVERLAP = 20\n\nDATASET_SYMBOL = '$' # this symbol represents a dataset name\nNONDATA_SYMBOL = '#' # this symbol represents a non-dataset name\n\nVAL = 1","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:27.556196Z","iopub.execute_input":"2021-05-24T10:46:27.556609Z","iopub.status.idle":"2021-05-24T10:46:27.56125Z","shell.execute_reply.started":"2021-05-24T10:46:27.556579Z","shell.execute_reply":"2021-05-24T10:46:27.559866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"# train\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\nrccDf = rccDf[:MAX_SAMPLE]\ntrain = pd.concat([rccDf, train], ignore_index=True)\n\n\n# Group by publication, training labels should have the same form as expected output.\ntrain = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()    \n\nprint('train size: ', len(train))\n\nexisting_labels = set(np.load('../input/showdata-labels1/existing_labels.npy', allow_pickle = True).tolist())\nexisting_labels_list = [lbl.split() for lbl in existing_labels]\nexisting_labels_list = existing_labels_list+[lbl.split() for lbl in newLabels]","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:27.562651Z","iopub.execute_input":"2021-05-24T10:46:27.562998Z","iopub.status.idle":"2021-05-24T10:46:28.595844Z","shell.execute_reply.started":"2021-05-24T10:46:27.562967Z","shell.execute_reply":"2021-05-24T10:46:28.594538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:28.597759Z","iopub.execute_input":"2021-05-24T10:46:28.598193Z","iopub.status.idle":"2021-05-24T10:46:28.620899Z","shell.execute_reply.started":"2021-05-24T10:46:28.598145Z","shell.execute_reply":"2021-05-24T10:46:28.619395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extDf = pd.read_csv('../input/training-set-found-labels/FNE_plusAbbrevs.csv')\nextDf.loc[extDf['Dataset']==1].sample(30)\nextDatasets = extDf.loc[extDf['Dataset']==1,'Text'].tolist()\nextDatasets = [dat for dat in extDatasets if len(dat.split())>1]\nextDatasets = list(set(extDatasets + newLabels + list(existing_labels)))","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:28.622771Z","iopub.execute_input":"2021-05-24T10:46:28.623263Z","iopub.status.idle":"2021-05-24T10:46:28.751646Z","shell.execute_reply.started":"2021-05-24T10:46:28.623211Z","shell.execute_reply":"2021-05-24T10:46:28.750311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(extDatasets)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:28.753865Z","iopub.execute_input":"2021-05-24T10:46:28.754372Z","iopub.status.idle":"2021-05-24T10:46:28.762534Z","shell.execute_reply.started":"2021-05-24T10:46:28.754322Z","shell.execute_reply":"2021-05-24T10:46:28.761325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data for train MLM","metadata":{}},{"cell_type":"markdown","source":"### Auxiliary functions","metadata":{}},{"cell_type":"code","source":"def clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    \"\"\"\n    find all positions of $small_list in $big_list.\n    \"\"\"\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef jaccard_similarity_list(l1, l2):\n    \"\"\"\n    Return the Jaccard Similarity score of 2 lists.\n    \"\"\"\n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset', 'survey', 'study','sequence'}\nprep_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'this', 'we', 'their', 'it', 'to'}\nfullLabels = list(set(list(existing_labels)+list(extDatasets)))\nabbreviations = [re.sub('[()]','',words.split()[-1]) for words in fullLabels if re.sub('[()]','',words.split()[-1]).isupper()]\nabbreviations = list(set([word for word in abbreviations if len(word)>2]))\ndef find_negative_candidates(sentence, labels, misc_candidates = []):\n    \"\"\"\n    Extract negative samples for Masked Dataset Modeling from a given $sentence.\n    A negative candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence. Lastly, the sequence must be quite different to any of the \n    ground truth labels (measured by Jaccard similarity).\n    \"\"\"\n    def candidate_qualified(words, labels):\n        # remove beginning words that are connection_tokens except data/dataset\n        startIdx = 0\n        endIdx = 0\n        while len(words) and words[0].lower() in prep_tokens:\n            words = words[1:]\n            startIdx +=1\n        # remove ending words that are connection_tokens\n        while len(words) and words[-1].lower() in prep_tokens:\n            words = words[:-1]\n            endIdx+=1\n        # comparison without connection_tokens\n        # possible change to 2\n        if len(words)==1 and np.random.rand()<0.0:\n            if (not words[0].isnumeric()) and words[0].isupper() and len(words[0])>=3 and all(words[0] not in label for label in labels) and all(words[0].lower() not in label for label in labels):\n                if any(char.isdigit() for char in words[0]):\n                    return False, []\n                elif words[0] not in abbreviations:\n                    return True, [startIdx, endIdx]\n                else:\n                    return False, []\n            else:\n                return False, []\n        elif len(words)==2:\n            if any((word[0].islower() or word.isnumeric() or len(word)<=2) for word in words):\n                return False, []\n            elif any(word.lower() in prep_tokens for word in words):\n                return False,[]\n            elif any(' '.join(words) in ' '.join(label) for label in labels):\n                return False, []\n            else:\n                return True, [startIdx, endIdx]\n        elif (len(words) <= 2 or \\\n            any(jaccard_similarity_list(words, label) >= 0.75 for label in labels) or \\\n            sum([1 for word in words if not word.isnumeric()])<=2):\n            return False, []\n        elif len(words)==3 and (words[1] == 'and' or all(word.isupper() for word in words)):\n            return False, []\n        elif any([word.lower() in ['dataset', 'data', 'survey', 'study'] for word in words]):\n            return False, []\n        elif any(jaccard_similarity_list([word.lower() for word in words], label) >= 0.5 for label in existing_labels_list):\n            return False, []\n        elif any(jaccard_similarity_list([word.lower() for word in words[0:4]], label) >= 0.5 for label in existing_labels_list):\n            return False, []\n        elif len(words)==4 and words[-1].isnumeric() and words[1] == 'and':\n            # to get rid of references, e.g. Johnson and Johnson 2018\n            return False, []\n        else:\n            return True, [startIdx, endIdx]\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        # if word is captial or connection token\n        # if word[0].isupper() or word in connection_tokens:\n        if word[0].isupper() or (word[0].isnumeric() and len(word)>2):\n            # set as phrase start if phrase start doesn't exist, if not set as end\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        elif word not in connection_tokens:\n            # if current phrase fulfils dissimilarity requirement, reset phrase_start\n            if phrase_start != -1:\n                qualified, tmpidxs = candidate_qualified(sentence[phrase_start:phrase_end+1], labels)\n                if qualified:\n                    candidates.append((phrase_start+tmpidxs[0], phrase_end-tmpidxs[1]))\n                phrase_start = phrase_end = -1\n    \n    # to deal with case where phrase end is last word\n    if phrase_start != -1:\n        qualified, tmpidxs = candidate_qualified(sentence[phrase_start:phrase_end+1], labels)\n        if qualified:\n            candidates.append((phrase_start+tmpidxs[0], phrase_end-tmpidxs[1]))\n            \n    for cand in misc_candidates:\n        words = cand.split()\n        if any(jaccard_similarity_list([word.lower() for word in words], label) >= 0.5 for label in existing_labels_list):\n            continue\n        elif any(jaccard_similarity_list([word.lower() for word in words], label) >= 0.5 for label in labels):\n            continue\n        elif any(jaccard_similarity_list([word.lower() for word in words], label) >= 0.5 for label in candidates):\n            continue\n        else:\n            phrase_start = sentence.index(words[0])\n            candidates.append((phrase_start,phrase_start+len(words)-1))\n    return candidates\n\ndef pre_tokenize(sentence):\n    try:\n        sentence = sentence.split()\n    except:\n        pass\n    wordlist = ['university', 'initiative','international','information']\n    for i in range(len(sentence)):\n        word  = sentence[i]\n        if word.isupper():\n            sentence[i] = '#'\n        elif word[0].isupper() and len(word)>8 and word.lower() not in wordlist:\n            sentence[i] = '$'\n    return sentence\n\ndef pre_tokenize_mask(sentence):\n    try:\n        sentence = sentence.split()\n    except:\n        pass\n    wordlist = ['university', 'initiative','international','information']\n    for i in range(len(sentence)):\n        word  = sentence[i]\n        if word.isupper():\n            sentence[i] = 'XXXX'\n        elif word[0].isupper() and len(word)>8 and word.lower() not in wordlist:\n            sentence[i] = 'ZZZZ'\n    return sentence\n\ndef replaceWithExt(phrase, prob = 0.8):\n    if np.random.rand()<prob:\n        if len(phrase)>1:\n            phrase = np.random.choice(extDatasets).split()\n            return phrase\n        else:\n            return phrase\n    else:\n        return phrase","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:28.765012Z","iopub.execute_input":"2021-05-24T10:46:28.765497Z","iopub.status.idle":"2021-05-24T10:46:28.826471Z","shell.execute_reply.started":"2021-05-24T10:46:28.76545Z","shell.execute_reply":"2021-05-24T10:46:28.825232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NER Encoding","metadata":{}},{"cell_type":"code","source":"if NER:\n    #from transformers import pipeline\n    #from transformers import AutoModelForTokenClassification, AutoTokenizer\n    #modelstrNER = 'squeezebert/squeezebert-mnli'\n    #modelNER = AutoModelForTokenClassification.from_pretrained(modelstrNER)\n    #tokenizerNER = AutoTokenizer.from_pretrained(modelstrNER)\n    #nerPipe = pipeline('ner', grouped_entities = True, device = 0, use_fast = True, model = modelNER, tokenizer = tokenizerNER)\n    import spacy\n    nerPipe = spacy.load('en_core_web_sm', pipeline=[\"ner\"])","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:28.828354Z","iopub.execute_input":"2021-05-24T10:46:28.828803Z","iopub.status.idle":"2021-05-24T10:46:28.835545Z","shell.execute_reply.started":"2021-05-24T10:46:28.828747Z","shell.execute_reply":"2021-05-24T10:46:28.834037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extract positive and negative samples","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\n\ncorpus = []\ncnt_pos = 0\ncnt_neg = 0\nneg_phrase = []\npos_phrase = []\nclassLabels = []\nNERDict = {}\ndN = 5\n\n\npbar = tqdm(total = len(train))\nfor paper_id, dataset_labels in train[['Id', 'dataset_label']].itertuples(index=False):\n    labels = [clean_paper_sentence(label) for label in dataset_labels.split('|')]\n    labels = list(set(labels))\n    labels = [label.split() for label in labels]\n    if isinstance(paper_id, str):\n        with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n            paper = json.load(f)\n        content = '. '.join(section['text'] for section in paper)\n    else:\n        with open('../input/rccfull/train_test/files/text/'+str(paper_id)+'.txt','r') as f:\n            content = f.read()\n    sentences = set([clean_paper_sentence(sentence) for sentence in content.split('.')])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    text = '.'.join(sentences)\n    labels_ = [' '.join(label) for label in labels]\n    tmpLabels = [label for label in fullLabels if label in text]\n    \n    \n    if NER:\n        try:\n            nerEnc = NERDict[str(paper_id)]\n        except:\n            nerEnc = [nerPipe(sentence).ents for sentence in sentences]\n            #nerEnc = nerPipe(sentences)\n            NERDict[str(paper_id)] = nerEnc\n        #for i_sent, sentence in enumerate(sentences):\n        #    replace_words = [word for word in nerEnc[i_sent] if \n        #                     (any(word['entity_group']==entgrp for entgrp in ['PER', 'LOC']) \n        #                      and '#' not in word['word'] and word['score']>0.7)]\n        #    for word in replace_words:\n        #        sentences[i_sent] = sentence.replace(word['word'],'John' if word['entity_group']=='PER' else 'Chicago')\n            \n    \n    sentences = [sentence.split() for sentence in sentences] \n    # positive samples\n    for sentence in sentences:\n        sentence_str = ' '.join(sentence)\n        for label in tmpLabels:\n            if label in labels_:\n                continue\n            elif label not in sentence_str:\n                continue\n            elif any(label in label2 for label2 in labels_):\n                continue\n            else:\n                labels.append(label.split())\n                labels_ = [' '.join(label) for label in labels]\n        for label in (labels):\n            for pos in find_sublist(sentence, label):\n                pos_phrase.append(sentence[max(pos-dN,0):pos+len(label)+dN])\n                if MASKEDLM:\n                    #dt_point = sentence[:pos] + [DATASET_SYMBOL] + sentence[pos+len(label):]\n                    dt_point = sentence[:pos] + ['@'] + replaceWithExt(sentence[pos:pos+len(label)]) + [DATASET_SYMBOL] + sentence[pos+len(label):]\n                    dt_point = pre_tokenize_mask(dt_point)\n                    corpus.append(' '.join(dt_point))\n                    cnt_pos += 1\n                else:\n                    # dt_point = sentence[:pos] + [DATASET_SYMBOL] + sentence[pos+len(label):]\n                    # tmpPhrase = sentence[max(pos-dN,0):pos+len(label)+dN]\n                    tmpPhrase = sentence[max(pos-dN,0):pos] + replaceWithExt(sentence[pos:pos+len(label)]) + sentence[pos+len(label):pos+len(label)+dN]\n                    dt_point = pre_tokenize(tmpPhrase)\n                    corpus.append(' '.join(dt_point))\n                    classLabels.append(1)\n                    cnt_pos += 1\n    \n    # negative samples\n    for ii,sentence in enumerate(sentences):\n        if NER:\n            # misc = [word['word'] for word in nerEnc[ii] if word['entity_group'] == 'MIS']\n            misc = [word.text for word in nerEnc[ii] if word.label_ in ['GPE', 'ORG', 'NORD']]\n        else:\n            misc = []\n        sentence_str = ' '.join(sentence)\n        if all(w not in sentence_str for w in {'data', 'study', 'survey'}):\n            continue\n        for phrase_start, phrase_end in find_negative_candidates(sentence, labels, misc_candidates = misc):\n            neg_phrase.append(sentence[phrase_start:phrase_end+1])\n            if MASKEDLM:\n                # dt_point = sentence[:phrase_start] + [NONDATA_SYMBOL] + sentence[phrase_end+1:]\n                dt_point = sentence[:phrase_start] + ['@'] + sentence[phrase_start:phrase_end+1] + [NONDATA_SYMBOL] + sentence[phrase_end+1:]\n                dt_point = pre_tokenize_mask(dt_point)\n                corpus.append(' '.join(dt_point))\n                cnt_neg += 1\n            else:\n                # dt_point = sentence[:phrase_start] + [DATASET_SYMBOL] + sentence[phrase_end+1:]\n                dt_point = pre_tokenize(sentence[max(phrase_start-dN,0):phrase_end+1+dN])\n                corpus.append(' '.join(dt_point))\n                cnt_neg += 1\n                classLabels.append(0)\n    \n    # process bar\n    pbar.update(1)\n    pbar.set_description(f'Training data size: {cnt_pos} postives + {cnt_neg} negatives')\n\nif NER:\n    import pickle\n    with open('nerEnc.pickle', 'wb') as handle:\n        pickle.dump(NERDict, handle)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:46:28.837596Z","iopub.execute_input":"2021-05-24T10:46:28.838083Z","iopub.status.idle":"2021-05-24T10:51:58.786596Z","shell.execute_reply.started":"2021-05-24T10:46:28.838036Z","shell.execute_reply":"2021-05-24T10:51:58.784876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[line for line in corpus[:] if 'NLSF' in line]","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:28:48.867657Z","iopub.execute_input":"2021-05-24T10:28:48.868019Z","iopub.status.idle":"2021-05-24T10:28:48.873446Z","shell.execute_reply.started":"2021-05-24T10:28:48.867989Z","shell.execute_reply":"2021-05-24T10:28:48.872767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[x for x in neg_phrase[5000:5100] if len(x)==2]","metadata":{"execution":{"iopub.status.busy":"2021-05-24T10:42:31.804857Z","iopub.execute_input":"2021-05-24T10:42:31.80521Z","iopub.status.idle":"2021-05-24T10:42:31.813905Z","shell.execute_reply.started":"2021-05-24T10:42:31.805181Z","shell.execute_reply":"2021-05-24T10:42:31.813041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"existing_labels","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.801793Z","iopub.status.idle":"2021-05-23T09:15:26.802174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus[2200:2220]","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.803039Z","iopub.status.idle":"2021-05-23T09:15:26.803425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save data to a file","metadata":{}},{"cell_type":"code","source":"with open('train_mlm.json', 'w') as f:\n    if not MASKEDLM:\n        for sentence, lbl in zip(corpus, classLabels):\n            row_json = {'text':sentence, 'label':lbl}\n            json.dump(row_json, f)\n            f.write('\\n')\n    else:\n        for sentence in corpus:\n            row_json = {'text':sentence}\n            json.dump(row_json, f)\n            f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.804268Z","iopub.status.idle":"2021-05-23T09:15:26.804722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune the Transformer","metadata":{}},{"cell_type":"code","source":"len(corpus)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.805512Z","iopub.status.idle":"2021-05-23T09:15:26.806069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if VAL:\n    datasets = load_dataset('json',\n                data_files={'train' : 'train_mlm.json'},\n                split = 'train[:80%]')\n    val_datasets = load_dataset('json',\n                    data_files={'train' : 'train_mlm.json'},\n                    split = 'train[80%:]')\nelse:\n    datasets = load_dataset('json',\n                data_files={'train' : 'train_mlm.json'},)\n\n\ndatasets","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.807016Z","iopub.status.idle":"2021-05-23T09:15:26.807424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenize and collate data","metadata":{}},{"cell_type":"code","source":"# model_checkpoint = \"bert-base-cased\"\n# model_checkpoint = \"roberta-base\"\nmodel_checkpoint = 'microsoft/deberta-base'\n# model_checkpoint = 'bert-large-cased'","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.808473Z","iopub.status.idle":"2021-05-23T09:15:26.808913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.809985Z","iopub.status.idle":"2021-05-23T09:15:26.810378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not 'berta' in model_checkpoint:\n    tokens_to_mask = [tokenizer.encode(sym)[1] for sym in [DATASET_SYMBOL,NONDATA_SYMBOL]]\nelse:\n    tokens_to_mask = [tokenizer.encode(sym)[1] for sym in [DATASET_SYMBOL,NONDATA_SYMBOL,' '+DATASET_SYMBOL,' '+NONDATA_SYMBOL]]","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.81118Z","iopub.status.idle":"2021-05-23T09:15:26.811565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.encode('John')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.812365Z","iopub.status.idle":"2021-05-23T09:15:26.812772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize_function(examples):\n    examples[\"text\"] = tokenizer(examples[\"text\"])\n    return examples[\"text\"]\n\ntokenized_datasets = datasets.map(tokenize_function, batched=True, num_proc=1, remove_columns=[\"text\"])\nval_tokenized_datasets = val_datasets.map(tokenize_function, batched=True, num_proc=1, remove_columns=[\"text\"])","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.813676Z","iopub.status.idle":"2021-05-23T09:15:26.814044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenized_datasets","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.814858Z","iopub.status.idle":"2021-05-23T09:15:26.815245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.encode('English')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.816017Z","iopub.status.idle":"2021-05-23T09:15:26.816456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokens_to_mask","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.817286Z","iopub.status.idle":"2021-05-23T09:15:26.817713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.nn.utils.rnn import pad_sequence\nfrom typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union\n\nclass  DataCollator(DataCollatorForLanguageModeling):\n    def __init__(self,tokenizer, tokens_to_mask, mlm_probability=0.00):\n        super(DataCollator, self).__init__(tokenizer=tokenizer, mlm_probability=0.00)\n        self.tokens_to_mask = tokens_to_mask\n    \n    def mask_tokens(\n        self, inputs: torch.Tensor, special_tokens_mask: Optional[torch.Tensor] = None\n    ) -> Tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"\n        Prepare masked tokens inputs/labels for masked language modeling: 80% MASK, 10% random, 10% original.\n        \"\"\"\n        labels = inputs.clone()\n        # We sample a few tokens in each sequence for MLM training (with probability `self.mlm_probability`)\n        probability_matrix = torch.full(labels.shape, self.mlm_probability)\n        if special_tokens_mask is None:\n            special_tokens_mask = [\n                self.tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) for val in labels.tolist()\n            ]\n            special_tokens_mask = torch.tensor(special_tokens_mask, dtype=torch.bool)\n        else:\n            special_tokens_mask = special_tokens_mask.bool()\n\n        probability_matrix.masked_fill_(special_tokens_mask, value=0.0)\n        for tok in self.tokens_to_mask:\n            probability_matrix.masked_fill_(torch.eq(inputs, tok), value=0.99)\n        masked_indices = torch.bernoulli(probability_matrix).bool()\n        labels[~masked_indices] = -100  # We only compute loss on masked tokens\n\n        # 80% of the time, we replace masked input tokens with tokenizer.mask_token ([MASK])\n        indices_replaced = torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices\n        inputs[indices_replaced] = self.tokenizer.convert_tokens_to_ids(self.tokenizer.mask_token)\n\n        # 10% of the time, we replace masked input tokens with random word\n        indices_random = torch.bernoulli(torch.full(labels.shape, 0.5)).bool() & masked_indices & ~indices_replaced\n        random_words = torch.randint(len(self.tokenizer), labels.shape, dtype=torch.long)\n        inputs[indices_random] = random_words[indices_random]\n\n        # The rest of the time (10% of the time) we keep the masked input tokens unchanged\n        return inputs, labels","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.818573Z","iopub.status.idle":"2021-05-23T09:15:26.81897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load pre-trained model and fine-tune","metadata":{}},{"cell_type":"code","source":"if MASKEDLM:\n    data_collator = DataCollator(tokenizer, tokens_to_mask, mlm_probability=0.0)\n    # data_collator = DataCollatorForLanguageModeling(tokenizer)\n    model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)\nelse:\n    data_collator = DataCollatorWithPadding(tokenizer)\n    model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.819712Z","iopub.status.idle":"2021-05-23T09:15:26.820087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Trainer, TrainingArguments\n\ntraining_args = TrainingArguments(\n    output_dir=\"output-mlm\" if MASKEDLM else \"output-seqClass\",\n    evaluation_strategy = \"steps\",\n    warmup_steps = 2000,\n    learning_rate=2e-5,\n    weight_decay=0.01,\n    save_steps=2000,\n    num_train_epochs=2.5,\n    report_to=\"none\",\n    logging_steps = 500,\n    save_total_limit = 1,\n    per_device_train_batch_size = 16,\n    per_device_eval_batch_size = 32,\n    eval_steps = 1000\n)\n\nif VAL:\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=tokenized_datasets,\n        eval_dataset=val_tokenized_datasets,\n        data_collator=data_collator,\n    )\nelse:\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=tokenized_datasets[\"train\"],\n        data_collator=data_collator,\n    )","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.820988Z","iopub.status.idle":"2021-05-23T09:15:26.82137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.822241Z","iopub.status.idle":"2021-05-23T09:15:26.822627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save model","metadata":{}},{"cell_type":"code","source":"if MASKEDLM:\n    trainer.model.save_pretrained('mlm-model')\nelse:\n    trainer.model.save_pretrained('seqClass-model')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.823551Z","iopub.status.idle":"2021-05-23T09:15:26.823955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save tokenizer","metadata":{}},{"cell_type":"code","source":"config = AutoConfig.from_pretrained(model_checkpoint)\n\ntokenizer.save_pretrained('model_tokenizer')\nconfig.save_pretrained('model_tokenizer')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T09:15:26.824827Z","iopub.status.idle":"2021-05-23T09:15:26.825216Z"},"trusted":true},"execution_count":null,"outputs":[]}]}