{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook shows how to fine-tune a BERT model (from huggingface) for our dataset recognition task.\n\nNote that internet is needed during the training phase (for downloading the bert-base-cased model). Internet can be turned off during prediction.","metadata":{}},{"cell_type":"markdown","source":"## Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-04T13:13:23.725752Z","iopub.execute_input":"2021-06-04T13:13:23.726135Z","iopub.status.idle":"2021-06-04T13:13:51.61559Z","shell.execute_reply.started":"2021-06-04T13:13:23.726051Z","shell.execute_reply":"2021-06-04T13:13:51.614458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\nfrom sklearn.model_selection import train_test_split\nfrom random import sample\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\n\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T20:17:19.145221Z","iopub.execute_input":"2021-06-07T20:17:19.145521Z","iopub.status.idle":"2021-06-07T20:17:20.050966Z","shell.execute_reply.started":"2021-06-07T20:17:19.14546Z","shell.execute_reply":"2021-06-07T20:17:20.050176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./","metadata":{"execution":{"iopub.status.busy":"2021-06-07T20:18:22.179129Z","iopub.execute_input":"2021-06-07T20:18:22.179493Z","iopub.status.idle":"2021-06-07T20:18:22.954645Z","shell.execute_reply.started":"2021-06-07T20:18:22.179462Z","shell.execute_reply":"2021-06-07T20:18:22.953705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyper-parameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\ndata_augmentation = True\n\n# Default training data size for validation1 settings with 4000 validation size: 38441 positives + 446696 negatives\nPOS_SAMPLE_SIZE = 38441\nNEG_SAMPLE_SIZE = 192205\n\nVALIDATION_SIZE = 4000\n\nMAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-07T20:18:24.906979Z","iopub.execute_input":"2021-06-07T20:18:24.907377Z","iopub.status.idle":"2021-06-07T20:18:24.912674Z","shell.execute_reply.started":"2021-06-07T20:18:24.907329Z","shell.execute_reply":"2021-06-07T20:18:24.911563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\n\ntrain = pd.read_csv(train_path)\n#train, test = train_test_split(temp, test_size = VALIDATION_SIZE, random_state=1337)\ntrain = train[:MAX_SAMPLE]\nprint(f'No. raw training rows: {len(train)}')\nprint(f'No. raw training rows: {len(test)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T20:18:26.661113Z","iopub.execute_input":"2021-06-07T20:18:26.661465Z","iopub.status.idle":"2021-06-07T20:18:26.771748Z","shell.execute_reply.started":"2021-06-07T20:18:26.661434Z","shell.execute_reply":"2021-06-07T20:18:26.770793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T20:18:29.431323Z","iopub.execute_input":"2021-06-07T20:18:29.431643Z","iopub.status.idle":"2021-06-07T20:18:29.745315Z","shell.execute_reply.started":"2021-06-07T20:18:29.431615Z","shell.execute_reply":"2021-06-07T20:18:29.74441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-07T20:18:30.635124Z","iopub.execute_input":"2021-06-07T20:18:30.635513Z","iopub.status.idle":"2021-06-07T20:19:16.270578Z","shell.execute_reply.started":"2021-06-07T20:18:30.635482Z","shell.execute_reply":"2021-06-07T20:19:16.269566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data to NER format","metadata":{}},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, list(zip(sentence_words, nes))","metadata":{"execution":{"iopub.status.busy":"2021-06-07T20:19:16.275527Z","iopub.execute_input":"2021-06-07T20:19:16.275871Z","iopub.status.idle":"2021-06-07T20:19:16.294896Z","shell.execute_reply.started":"2021-06-07T20:19:16.275839Z","shell.execute_reply":"2021-06-07T20:19:16.293934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nner_data = []\npos_indexes = []\nneg_indexes = []\n\nalllabels = []\nfor i, id, dataset_label in train[['Id', 'dataset_label']].itertuples():\n    labels = dataset_label.split('|')\n    labels = [clean_training_text(label) for label in labels]\n    alllabels += labels\nalllabels = list(set(alllabels))\n\npbar = tqdm(total=len(train))\nfor i, id, dataset_label in train[['Id', 'dataset_label']].itertuples():\n    # paper\n    paper = papers[id]\n    \n    # labels\n    labels = dataset_label.split('|')\n    labels = [clean_training_text(label) for label in labels]\n    \n    # sentences\n    sentences = set([clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.') \n                ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    \n    #for sentence in sentences:\n    #    chars = 0\n    #    for token in sentence:\n    #        chars += len(token)\n    #    inputCharLengths.append(chars)\n    #    inputChars.append(sentence)\n    #sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n    # remove sentences with > 500 characters\n    sentences = [sentence for sentence in sentences if sum(len(i) for i in sentence) < 500]\n    \n    \n    # positive sample\n    for sentence in sentences:\n        is_positive, tags = tag_sentence(sentence, labels)\n        if is_positive:\n            cnt_pos += 1\n            if data_augmentation and (random.random() < 0.05):\n                start_index = 0\n                end_index = 0\n                for i in range(len(tags)-1):\n                    currentt, ctag = tags[i]\n                    nextt, ntag = tags[i+1]\n                    if ctag == 'O' and ntag == 'B':\n                        start_index = i+1\n                    elif ctag == 'B' and ntag == 'O':\n                        end_index =  i+1\n                    elif ctag == 'I' and ntag == 'O':\n                        end_index = i+1\n                sbegin = tags[:start_index]\n                send = tags[end_index:]\n                newlabel = random.choice(alllabels).split()\n                bi = []\n                for i in range(len(newlabel)):\n                    if i == 0:\n                        bi.append((newlabel[i], 'B'))\n                    else:\n                        bi.append((newlabel[i], 'I'))\n                tags = sbegin + bi + send\n                print(tags)\n            ner_data.append(tags)\n            pos_indexes.append(len(ner_data) - 1)\n        elif any(word in sentence.lower() for word in ['data', 'study']): #'statistics', 'compilation', 'dossier', 'dataset', 'reports', 'studies', 'measurements', 'file', 'archive', 'set', 'public', 'toy', 'synthetic' \n            ner_data.append(tags)\n            cnt_neg += 1\n            neg_indexes.append(len(ner_data) - 1)\n    \n    # process bar\n    pbar.update(1)\n    pbar.set_description(f\"Training data size before balance: {cnt_pos} positives + {cnt_neg} negatives\")\n\n    \nprint(\"posOrigLen:\", len(pos_indexes))\nprint(\"negOrigLen:\", len(neg_indexes))\n\n# adjust sample balance\npos_indexes_after_sample = sample(pos_indexes,POS_SAMPLE_SIZE)\nneg_indexes_after_sample = sample(neg_indexes,NEG_SAMPLE_SIZE)\n\npos_to_remove = set(pos_indexes) - set(pos_indexes_after_sample)\nneg_to_remove = set(neg_indexes) - set(neg_indexes_after_sample)\nto_remove = pos_to_remove.union(neg_to_remove)\n\nprint(\"Before:\", len(ner_data))\nner_data = [v for i, v in enumerate(ner_data) if i not in to_remove]\nprint(\"After:\", len(ner_data))\n\n    \n# shuffling\nrandom.shuffle(ner_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T21:28:51.765096Z","iopub.execute_input":"2021-06-07T21:28:51.765578Z","iopub.status.idle":"2021-06-07T21:31:39.172014Z","shell.execute_reply.started":"2021-06-07T21:28:51.765541Z","shell.execute_reply":"2021-06-07T21:31:39.167185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.plot(inputCharLengths)\n#plt.show()\n\n#largest = np.argmax(inputCharLengths)\n#print(inputChars[largest])\n#print(largest)","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:17:35.094046Z","iopub.execute_input":"2021-06-04T13:17:35.094297Z","iopub.status.idle":"2021-06-04T13:17:35.101523Z","shell.execute_reply.started":"2021-06-04T13:17:35.094272Z","shell.execute_reply":"2021-06-04T13:17:35.100521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"write data to file.","metadata":{}},{"cell_type":"code","source":"# sorting the list\n#list2 = inputCharLengths.copy()\n#list2.sort()\n  \n# printing the last element\n#print(\"Largest length is:\", list2[-1])\n\n#plt.plot(list2)\n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:17:35.103455Z","iopub.execute_input":"2021-06-04T13:17:35.103732Z","iopub.status.idle":"2021-06-04T13:17:35.110709Z","shell.execute_reply.started":"2021-06-04T13:17:35.103703Z","shell.execute_reply":"2021-06-04T13:17:35.109926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('train_ner.json', 'w') as f:\n    for row in ner_data:\n        words, nes = list(zip(*row))\n        row_json = {'tokens' : words, 'tags' : nes}\n        json.dump(row_json, f)\n        f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:17:35.111868Z","iopub.execute_input":"2021-06-04T13:17:35.112231Z","iopub.status.idle":"2021-06-04T13:17:39.03151Z","shell.execute_reply.started":"2021-06-04T13:17:35.112194Z","shell.execute_reply":"2021-06-04T13:17:39.030547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tune a BERT model for NER","metadata":{}},{"cell_type":"code","source":"!python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n--model_name_or_path 'allenai/scibert_scivocab_cased' \\\n--train_file './train_ner.json' \\\n--validation_file './train_ner.json' \\\n--num_train_epochs 1 \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--save_steps 60000 \\\n--output_dir './output' \\\n--report_to 'none' \\\n--seed 123 \\\n--do_train ","metadata":{"execution":{"iopub.status.busy":"2021-06-04T13:17:39.032878Z","iopub.execute_input":"2021-06-04T13:17:39.033346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After the tuning finishes, we should find our model in './output'.","metadata":{}}]}