{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook shows how to fine-tune a BERT model (from huggingface) for our dataset recognition task.\n\nNote that internet is needed during the training phase (for downloading the bert-base-cased model). Internet can be turned off during prediction.","metadata":{"editable":false}},{"cell_type":"markdown","source":"## Install packages","metadata":{"editable":false}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{"editable":false}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyper-parameters","metadata":{"editable":false}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nMAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{"editable":false}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\nprint(f'No. raw training rows: {len(train)}')","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{"editable":false}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data to NER format","metadata":{"editable":false}},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef get_words(idx, sentences, length):\n    words = []\n    # Keep taking sentences with words untill full\n    for sentence in sentences[(idx+1):]:\n        words.extend(sentence.split())\n        words.append(\"[SEP]\")\n\n        # Combining these would result in a too large entry\n        if length-len(words) < 0:\n            break\n\n    if length-len(words) > 0:\n        for sentence in sentences[:idx]:\n            words.extend(sentence.split())\n            words.append(\"[SEP]\")\n\n            # Combining these would result in a too large entry\n            if length-len(words) < 0:\n                break\n    \n#     # DOC is shorter than maximum length\n#     if length-len(words) > 0:\n#         words.append(\"[PAD]\")\n\n    return words[:length]\n        \ndef get_sentences_before(sentences, idx, jdx, cur_length):\n    reverse_words = []\n    for sentence in sentences[max(idx-jdx, 0):idx][::-1]:\n        words = sentence.split()\n        words.append(\"[SEP]\")\n        words = words[::-1]\n        # If added sentences are too much, then only add last words of a sentence\n        # this indicates this iteration is the last, since no further 'before-sentences' can be added\n        if len(words)+len(reverse_words)+cur_length > MAX_LENGTH-1:\n            reverse_words.extend(words[:(MAX_LENGTH-len(reverse_words)-cur_length-1)])\n            break\n        else:\n            reverse_words.extend(words)\n    \n    if idx-jdx < 0:\n        for sentence in sentences[idx-jdx:][::-1]:\n            words = sentence.split()\n            words.append(\"[SEP]\")\n            words = words[::-1]\n            # If added sentences are too much, then only add last words of a sentence\n            # this indicates this iteration is the last, since no further 'before-sentences' can be added\n            if len(words)+len(reverse_words)+cur_length > MAX_LENGTH-1:\n                reverse_words.extend(words[:(MAX_LENGTH-len(reverse_words)-cur_length-1)])\n                break\n            else:\n                reverse_words.extend(words)\n    \n    return reverse_words[::-1]\n\ndef shorten_sentences_with_context(sentences):\n    sentences = list(sentences)\n    short_sentences = []\n    for idx, val in enumerate(sentences):\n        val = val.split()\n        \n        # If this sentence is too long already:\n        if len(val) > (MAX_LENGTH - 2): # We append [CLS], [SEP] and [PAD] tokens\n            for p in range(0, len(val), MAX_LENGTH-1):\n                temp = val[p:p+MAX_LENGTH-1]\n\n                # Append this sentence to the list if it is of max length\n                # If this is the final part of the sentence that fits with the tokens\n                if len(temp) == MAX_LENGTH-2:\n                    short_sentences.append(\"[CLS] \" + ' '.join(temp) + \" [SEP]\")\n                        \n                # This is the final part of the sentence, but does not fit [SEP] token\n                elif len(temp) == MAX_LENGTH-1:\n                    short_sentences.append(\"[CLS] \" + ' '.join(temp))\n\n                # Fill it up and continue if not full\n                else:\n                    temp.append(\"[SEP]\")\n                    temp.extend(get_words(idx, sentences, MAX_LENGTH-len(temp)-1))\n                    short_sentences.append(\"[CLS] \" + ' '.join(temp))\n\n            continue\n\n\n        # For each sentence there are multiple instances, using a larger set of sentences before each time\n        val.append(\"[SEP]\")\n        for jdx in range(len(sentences)):#idx+1):\n            reverse_words = get_sentences_before(sentences, idx, jdx, len(val))\n            \n            # Combine 'before-senteces' and this sentence\n            tot_words = reverse_words + val\n            \n            # If this is full, then we can't append any more before this part, \n            # so this loop is done, go to next sentence\n            if len(tot_words) == MAX_LENGTH-1:\n                short_sentences.append(\"[CLS] \" + ' '.join(tot_words))\n                break\n            \n            # Else append next sentences, untill full\n            tot_words.extend(get_words(idx, sentences, MAX_LENGTH-len(tot_words)-1))\n            short_sentences.append(\"[CLS] \" + ' '.join(tot_words))\n\n    return list(set(short_sentences))\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, list(zip(sentence_words, nes))","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # TEST for each case in context function\n# MAX_LENGTH = 6 # lower max_length for testing\n\n# def gen_sen(n):\n#     return ' '.join([str(i) for i in range(n)])\n# # T1: Sentence too long\n# t1 = [gen_sen(2), gen_sen(10), gen_sen(4)[-3:]]\n# r1 = shorten_sentences_with_context(t1)\n# print(\"Test 1\")\n# print([len(i.split()) for i in t1])\n# print([len(i.split()) for i in r1])\n# print(t1)\n# print(r1)\n# # T2: Add multiple sentences\n# t2 = [gen_sen(2), gen_sen(1), gen_sen(4)[-1:]]\n# r2 = shorten_sentences_with_context(t2)\n# print(\"Test 2\")\n# print([len(i.split()) for i in t2])\n# print([len(i.split()) for i in r2])\n# print(t2)\n# print(r2)\n# MAX_LENGTH = 64 # Reset max_length","metadata":{"_kg_hide-input":true,"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nkeep_percentage = 0.2 # Percentage to keep a subset of the negative samples\nner_data = []\n\npbar = tqdm(total=len(train))\nfor i, id, dataset_label in train[['Id', 'dataset_label']].itertuples():\n    # paper\n    paper = papers[id]\n    \n    # labels\n    labels = dataset_label.split('|')\n    labels = [clean_training_text(label) for label in labels]\n    \n    # sentences\n    sentences = set([clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.') \n                ])\n    sentences = shorten_sentences_with_context(sentences) #shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    # positive sample\n    for sentence in sentences:\n        is_positive, tags = tag_sentence(sentence, labels)\n        if is_positive:\n            cnt_pos += 1\n            ner_data.append(tags)\n        elif any(word in sentence.lower() for word in ['data', 'study']):\n            if random.random() < keep_percentage:\n                ner_data.append(tags)\n                cnt_neg += 1\n    \n    # process bar\n    pbar.update(1)\n    pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\n\n# shuffling\nrandom.shuffle(ner_data)","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"write data to file.","metadata":{"editable":false}},{"cell_type":"code","source":"with open('train_ner.json', 'w') as f:\n    for row in ner_data:\n        words, nes = list(zip(*row))\n        row_json = {'tokens' : words, 'tags' : nes}\n        json.dump(row_json, f)\n        f.write('\\n')","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune a BERT model for NER","metadata":{"editable":false}},{"cell_type":"code","source":"!python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n--model_name_or_path 'bert-base-cased' \\\n--train_file './train_ner.json' \\\n--validation_file './train_ner.json' \\\n--num_train_epochs 1 \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--save_steps 15000 \\\n--output_dir './output' \\\n--report_to 'none' \\\n--seed 123 \\\n--do_train ","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After the tuning finishes, we should find our model in './output'.","metadata":{"editable":false}}]}