{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook illustrates how to use Masked Language Modeling for this competition.\n\nObservation: most of the dataset names consist of only words with uppercased-first-letter and some stopwords like `on`, `in`, `and` (e.g. `Early Childhood Longitudinal Study`, `Trends in International Mathematics and Science Study`). \n\nThus, one approach to find the datasets is: \n- Locate all the sequences of capitalized words (these sequences may contain some stopwords), \n- Replace each sequence with one of 2 special symbols (e.g. `$` and `#`), implying if that sequence represents a dataset name or not.\n- Have the model learn the MLM task.","metadata":{}},{"cell_type":"markdown","source":"The code below shows how to train a model for that purpose with the help of the `huggingface`.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number (e.g. 50) for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:32:30.985718Z","iopub.execute_input":"2021-06-21T04:32:30.986106Z","iopub.status.idle":"2021-06-21T04:32:30.991864Z","shell.execute_reply.started":"2021-06-21T04:32:30.986003Z","shell.execute_reply":"2021-06-21T04:32:30.990898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-21T04:32:30.994648Z","iopub.execute_input":"2021-06-21T04:32:30.995375Z","iopub.status.idle":"2021-06-21T04:32:57.973833Z","shell.execute_reply.started":"2021-06-21T04:32:30.995299Z","shell.execute_reply":"2021-06-21T04:32:57.972799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline, AutoConfig\n\nsns.set()\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:32:57.977626Z","iopub.execute_input":"2021-06-21T04:32:57.977905Z","iopub.status.idle":"2021-06-21T04:33:05.519158Z","shell.execute_reply.started":"2021-06-21T04:32:57.977877Z","shell.execute_reply":"2021-06-21T04:33:05.518219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_checkpoint = \"bert-base-cased\"\n\nMAX_LENGTH = 64\nOVERLAP = 20\n\nDATASET_SYMBOL = '$' # this symbol represents a dataset name\nNONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:33:05.523532Z","iopub.execute_input":"2021-06-21T04:33:05.525578Z","iopub.status.idle":"2021-06-21T04:33:05.531513Z","shell.execute_reply.started":"2021-06-21T04:33:05.525534Z","shell.execute_reply":"2021-06-21T04:33:05.530694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"# train\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\n# Group by publication, training labels should have the same form as expected output.\ntrain = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()    \n\nprint('train size: ', len(train))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:33:05.536301Z","iopub.execute_input":"2021-06-21T04:33:05.538816Z","iopub.status.idle":"2021-06-21T04:33:06.224659Z","shell.execute_reply.started":"2021-06-21T04:33:05.538777Z","shell.execute_reply":"2021-06-21T04:33:06.223691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data for train MLM","metadata":{}},{"cell_type":"markdown","source":"### Auxiliary functions","metadata":{}},{"cell_type":"code","source":"def clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    \"\"\"\n    find all positions of $small_list in $big_list.\n    \"\"\"\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef jaccard_similarity_list(l1, l2):\n    \"\"\"\n    Return the Jaccard Similarity score of 2 lists.\n    \"\"\"\n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset'}\ndef find_negative_candidates(sentence, labels):\n    \"\"\"\n    Extract negative samples for Masked Dataset Modeling from a given $sentence.\n    A negative candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence. Lastly, the sequence must be quite different to any of the \n    ground truth labels (measured by Jaccard similarity).\n    \"\"\"\n    def candidate_qualified(words, labels):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2 and \\\n               all(jaccard_similarity_list(words, label) < 0.75 for label in labels)\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1], labels):\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1], labels):\n            candidates.append((phrase_start, phrase_end))\n    \n    return candidates","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:33:06.227621Z","iopub.execute_input":"2021-06-21T04:33:06.227903Z","iopub.status.idle":"2021-06-21T04:33:06.242967Z","shell.execute_reply.started":"2021-06-21T04:33:06.227877Z","shell.execute_reply":"2021-06-21T04:33:06.241873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extract positive and negative samples","metadata":{}},{"cell_type":"code","source":"corpus = []\ncnt_pos = 0\ncnt_neg = 0\n\npbar = tqdm(total = len(train))\nfor paper_id, dataset_labels in train[['Id', 'dataset_label']].itertuples(index=False):\n    labels = [clean_paper_sentence(label).split() for label in dataset_labels.split('|')]\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n    content = '. '.join(section['text'] for section in paper)\n    sentences = set([clean_paper_sentence(sentence) for sentence in content.split('.')])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    sentences = [sentence.split() for sentence in sentences]\n    \n    # positive samples\n    for sentence in sentences:\n        for label in labels:\n            for pos in find_sublist(sentence, label):\n                dt_point = sentence[:pos] + [DATASET_SYMBOL] + sentence[pos+len(label):]\n                corpus.append(' '.join(dt_point))\n                cnt_pos += 1\n    \n    # negative samples\n    for sentence in sentences:\n        sentence_str = ' '.join(sentence)\n        if all(w not in sentence_str for w in {'data', 'study'}):\n            continue\n        for phrase_start, phrase_end in find_negative_candidates(sentence, labels):\n            dt_point = sentence[:phrase_start] + [NONDATA_SYMBOL] + sentence[phrase_end+1:]\n            corpus.append(' '.join(dt_point))\n            cnt_neg += 1\n    \n    # process bar\n    pbar.update(1)\n    pbar.set_description(f'Training data size: {cnt_pos} postives + {cnt_neg} negatives')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:33:06.245221Z","iopub.execute_input":"2021-06-21T04:33:06.245617Z","iopub.status.idle":"2021-06-21T04:37:09.950625Z","shell.execute_reply.started":"2021-06-21T04:33:06.245569Z","shell.execute_reply":"2021-06-21T04:37:09.94997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save data to a file","metadata":{}},{"cell_type":"code","source":"with open('train_mlm.json', 'w') as f:\n    for sentence in corpus:\n        row_json = {'text':sentence}\n        json.dump(row_json, f)\n        f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:37:09.952628Z","iopub.execute_input":"2021-06-21T04:37:09.952971Z","iopub.status.idle":"2021-06-21T04:37:12.496233Z","shell.execute_reply.started":"2021-06-21T04:37:09.952935Z","shell.execute_reply":"2021-06-21T04:37:12.495315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune the Transformer","metadata":{}},{"cell_type":"code","source":"datasets = load_dataset('json',\n            data_files={'train' : 'train_mlm.json'},\n            )\n\ndatasets[\"train\"][:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:37:12.497408Z","iopub.execute_input":"2021-06-21T04:37:12.497736Z","iopub.status.idle":"2021-06-21T04:37:13.257609Z","shell.execute_reply.started":"2021-06-21T04:37:12.497701Z","shell.execute_reply":"2021-06-21T04:37:13.256848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tokenize and collate data","metadata":{}},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:37:13.2588Z","iopub.execute_input":"2021-06-21T04:37:13.259157Z","iopub.status.idle":"2021-06-21T04:37:14.185054Z","shell.execute_reply.started":"2021-06-21T04:37:13.259131Z","shell.execute_reply":"2021-06-21T04:37:14.184116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize_function(examples):\n    return tokenizer(examples[\"text\"])\n\ntokenized_datasets = datasets.map(tokenize_function, batched=True, num_proc=1, remove_columns=[\"text\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:37:14.186442Z","iopub.execute_input":"2021-06-21T04:37:14.186793Z","iopub.status.idle":"2021-06-21T04:37:40.065747Z","shell.execute_reply.started":"2021-06-21T04:37:14.186755Z","shell.execute_reply":"2021-06-21T04:37:40.064801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:37:40.067177Z","iopub.execute_input":"2021-06-21T04:37:40.067523Z","iopub.status.idle":"2021-06-21T04:37:40.074155Z","shell.execute_reply.started":"2021-06-21T04:37:40.067485Z","shell.execute_reply":"2021-06-21T04:37:40.071111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load pre-trained model and fine-tune","metadata":{}},{"cell_type":"code","source":"model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:37:40.075598Z","iopub.execute_input":"2021-06-21T04:37:40.075953Z","iopub.status.idle":"2021-06-21T04:37:48.100397Z","shell.execute_reply.started":"2021-06-21T04:37:40.075917Z","shell.execute_reply":"2021-06-21T04:37:48.099403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Trainer, TrainingArguments\n\ntraining_args = TrainingArguments(\n    output_dir=\"output-mlm\",\n    evaluation_strategy = \"no\",\n    learning_rate=1e-4,\n    weight_decay=0.05,\n    save_steps=12000,\n    num_train_epochs=4,\n    report_to=\"none\",\n)\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n    data_collator=data_collator,\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:37:48.101721Z","iopub.execute_input":"2021-06-21T04:37:48.102071Z","iopub.status.idle":"2021-06-21T04:37:54.346135Z","shell.execute_reply.started":"2021-06-21T04:37:48.102035Z","shell.execute_reply":"2021-06-21T04:37:54.345322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:37:54.347356Z","iopub.execute_input":"2021-06-21T04:37:54.347686Z","iopub.status.idle":"2021-06-21T04:48:21.381134Z","shell.execute_reply.started":"2021-06-21T04:37:54.347652Z","shell.execute_reply":"2021-06-21T04:48:21.377844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save model","metadata":{}},{"cell_type":"code","source":"trainer.model.save_pretrained('mlm-model')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:48:21.383341Z","iopub.status.idle":"2021-06-21T04:48:21.383948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save tokenizer","metadata":{}},{"cell_type":"code","source":"config = AutoConfig.from_pretrained(model_checkpoint)\n\ntokenizer.save_pretrained('model_tokenizer')\nconfig.save_pretrained('model_tokenizer')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T04:48:21.386699Z","iopub.status.idle":"2021-06-21T04:48:21.387407Z"},"trusted":true},"execution_count":null,"outputs":[]}]}