{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Install packages","metadata":{}},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:20.301514Z","iopub.execute_input":"2021-06-19T13:53:20.301918Z","iopub.status.idle":"2021-06-19T13:53:21.182922Z","shell.execute_reply.started":"2021-06-19T13:53:20.301830Z","shell.execute_reply":"2021-06-19T13:53:21.182145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyper-parameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nMAX_SAMPLE = None # set a small number for experimentation, set None for production.\n\n# MAX_SAMPLE=5000","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:21.184333Z","iopub.execute_input":"2021-06-19T13:53:21.184700Z","iopub.status.idle":"2021-06-19T13:53:21.188277Z","shell.execute_reply.started":"2021-06-19T13:53:21.184674Z","shell.execute_reply":"2021-06-19T13:53:21.187483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\ntrain = pd.read_csv(train_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:21.189402Z","iopub.execute_input":"2021-06-19T13:53:21.189828Z","iopub.status.idle":"2021-06-19T13:53:21.342554Z","shell.execute_reply.started":"2021-06-19T13:53:21.189795Z","shell.execute_reply":"2021-06-19T13:53:21.341505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:21.344116Z","iopub.execute_input":"2021-06-19T13:53:21.344541Z","iopub.status.idle":"2021-06-19T13:53:21.352913Z","shell.execute_reply.started":"2021-06-19T13:53:21.344499Z","shell.execute_reply":"2021-06-19T13:53:21.351881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[:MAX_SAMPLE]\nprint(f'No. raw training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:21.355730Z","iopub.execute_input":"2021-06-19T13:53:21.356050Z","iopub.status.idle":"2021-06-19T13:53:21.370682Z","shell.execute_reply.started":"2021-06-19T13:53:21.356020Z","shell.execute_reply":"2021-06-19T13:53:21.369560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:21.372204Z","iopub.execute_input":"2021-06-19T13:53:21.372542Z","iopub.status.idle":"2021-06-19T13:53:21.450465Z","shell.execute_reply.started":"2021-06-19T13:53:21.372513Z","shell.execute_reply":"2021-06-19T13:53:21.449724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:21.451550Z","iopub.execute_input":"2021-06-19T13:53:21.451971Z","iopub.status.idle":"2021-06-19T13:53:21.469943Z","shell.execute_reply.started":"2021-06-19T13:53:21.451941Z","shell.execute_reply":"2021-06-19T13:53:21.468946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:21.471603Z","iopub.execute_input":"2021-06-19T13:53:21.471881Z","iopub.status.idle":"2021-06-19T13:53:22.128239Z","shell.execute_reply.started":"2021-06-19T13:53:21.471854Z","shell.execute_reply":"2021-06-19T13:53:22.127481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"papers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:53:22.129202Z","iopub.execute_input":"2021-06-19T13:53:22.129608Z","iopub.status.idle":"2021-06-19T13:54:22.437746Z","shell.execute_reply.started":"2021-06-19T13:53:22.129570Z","shell.execute_reply":"2021-06-19T13:54:22.436704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform data to NER format","metadata":{}},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n#             print(label_words)\n            all_pos = find_sublist(sentence_words, label_words)\n#             print(all_pos)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n        \n#         print(nes)\n        return True, [sentence_words, nes]\n#         return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, [sentence_words, nes]\n#         return False, list(zip(sentence_words, nes))","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:54:22.439027Z","iopub.execute_input":"2021-06-19T13:54:22.439342Z","iopub.status.idle":"2021-06-19T13:54:22.451948Z","shell.execute_reply.started":"2021-06-19T13:54:22.439312Z","shell.execute_reply":"2021-06-19T13:54:22.450762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nner_data = []\ns=False\npbar = tqdm(total=len(train))\nfor i, id, dataset_label in train[['Id', 'dataset_label']].itertuples():\n    # paper\n    paper = papers[id]\n    \n    # labels\n    labels = dataset_label.split('|')\n    labels = [clean_training_text(label) for label in labels]\n    \n    # sentences\n    sentences = set([clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.') \n                ])\n    sentences = shorten_sentences(sentences) # make sentences short\n    sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n    \n#     print(sentences)\n    \n    # positive sample\n    for sentence in sentences:\n        is_positive, tags = tag_sentence(sentence, labels)\n        if is_positive:\n#             print(tags)\n\n            cnt_pos += 1\n            ner_data.append(tags)\n#             s=True\n#             break\n        elif any(word in sentence.lower() for word in ['data', 'study']): \n            ner_data.append(tags)\n            cnt_neg += 1\n#     if s==True:\n#         break\n    # process bar\n    pbar.update(1)\n    pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\n\n# shuffling\nrandom.shuffle(ner_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:54:22.453558Z","iopub.execute_input":"2021-06-19T13:54:22.454231Z","iopub.status.idle":"2021-06-19T13:57:15.068741Z","shell.execute_reply.started":"2021-06-19T13:54:22.453986Z","shell.execute_reply":"2021-06-19T13:57:15.067670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ner_dataset=pd.DataFrame(ner_data,columns=['sentence','tag'])","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:58:29.443138Z","iopub.execute_input":"2021-06-19T13:58:29.443491Z","iopub.status.idle":"2021-06-19T13:58:30.068506Z","shell.execute_reply.started":"2021-06-19T13:58:29.443463Z","shell.execute_reply":"2021-06-19T13:58:30.067354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ner_dataset.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:58:30.070472Z","iopub.execute_input":"2021-06-19T13:58:30.070922Z","iopub.status.idle":"2021-06-19T13:58:30.088687Z","shell.execute_reply.started":"2021-06-19T13:58:30.070874Z","shell.execute_reply":"2021-06-19T13:58:30.087592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ner_dataset.to_csv(\"ner_dataset.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-19T13:59:40.351111Z","iopub.execute_input":"2021-06-19T13:59:40.351636Z","iopub.status.idle":"2021-06-19T13:59:52.303805Z","shell.execute_reply.started":"2021-06-19T13:59:40.351594Z","shell.execute_reply":"2021-06-19T13:59:52.302962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with open('train_ner.json', 'w') as f:\n#     for row in ner_data:\n#         words, nes = list(zip(*row))\n#         row_json = {'tokens' : words, 'tags' : nes}\n#         json.dump(row_json, f)\n#         f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-06-19T14:01:36.529304Z","iopub.execute_input":"2021-06-19T14:01:36.529825Z","iopub.status.idle":"2021-06-19T14:01:36.533391Z","shell.execute_reply.started":"2021-06-19T14:01:36.529782Z","shell.execute_reply":"2021-06-19T14:01:36.532501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import joblib\n# import torch\n# import torch.nn as nn\n# import transformers\n\n# import numpy as np\n# import pandas as pd\n\n# from sklearn import preprocessing\n# from sklearn import model_selection\n\n# from tqdm import tqdm\n# from transformers import AdamW\n# from transformers import get_linear_schedule_with_warmup","metadata":{"execution":{"iopub.status.busy":"2021-06-19T14:01:36.703798Z","iopub.execute_input":"2021-06-19T14:01:36.704314Z","iopub.status.idle":"2021-06-19T14:01:36.708854Z","shell.execute_reply.started":"2021-06-19T14:01:36.704269Z","shell.execute_reply":"2021-06-19T14:01:36.707707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class config:\n#     MAX_LEN = 128\n#     TRAIN_BATCH_SIZE = 32\n#     VALID_BATCH_SIZE = 8\n#     EPOCHS = 3\n#     BASE_MODEL_PATH = \"../input/bert-base-uncased/\"\n#     MODEL_PATH = \"model.bin\"\n#     TRAINING_FILE = \"../input/entity-annotated-corpus/ner_dataset.csv\"\n#     TOKENIZER = transformers.BertTokenizer.from_pretrained(\n#         'bert-base-uncased',\n#         do_lower_case=True\n#     )","metadata":{"execution":{"iopub.status.busy":"2021-06-19T14:01:36.861216Z","iopub.execute_input":"2021-06-19T14:01:36.861781Z","iopub.status.idle":"2021-06-19T14:01:36.866421Z","shell.execute_reply.started":"2021-06-19T14:01:36.861734Z","shell.execute_reply":"2021-06-19T14:01:36.865584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class EntityDataset:\n#     def __init__(self, texts, pos, tags):\n#         self.texts = texts\n#         self.pos = pos\n#         self.tags = tags\n    \n#     def __len__(self):\n#         return len(self.texts)\n    \n#     def __getitem__(self, item):\n#         text = self.texts[item]\n#         pos = self.pos[item]\n#         tags = self.tags[item]\n\n#         ids = []\n#         target_pos = []\n#         target_tag =[]\n\n#         for i, s in enumerate(text):\n#             inputs = config.TOKENIZER.encode(\n#                 s,\n#                 add_special_tokens=False\n#             )\n#             # abhishek: ab ##hi ##sh ##ek\n#             input_len = len(inputs)\n#             ids.extend(inputs)\n#             target_pos.extend([pos[i]] * input_len)\n#             target_tag.extend([tags[i]] * input_len)\n\n#         ids = ids[:config.MAX_LEN - 2]\n#         target_pos = target_pos[:config.MAX_LEN - 2]\n#         target_tag = target_tag[:config.MAX_LEN - 2]\n\n#         ids = [101] + ids + [102]\n#         target_pos = [0] + target_pos + [0]\n#         target_tag = [0] + target_tag + [0]\n\n#         mask = [1] * len(ids)\n#         token_type_ids = [0] * len(ids)\n\n#         padding_len = config.MAX_LEN - len(ids)\n\n#         ids = ids + ([0] * padding_len)\n#         mask = mask + ([0] * padding_len)\n#         token_type_ids = token_type_ids + ([0] * padding_len)\n#         target_pos = target_pos + ([0] * padding_len)\n#         target_tag = target_tag + ([0] * padding_len)\n\n#         return {\n#             \"ids\": torch.tensor(ids, dtype=torch.long),\n#             \"mask\": torch.tensor(mask, dtype=torch.long),\n#             \"token_type_ids\": torch.tensor(token_type_ids, dtype=torch.long),\n#             \"target_pos\": torch.tensor(target_pos, dtype=torch.long),\n#             \"target_tag\": torch.tensor(target_tag, dtype=torch.long),\n#         }","metadata":{"execution":{"iopub.status.busy":"2021-06-19T14:01:37.292549Z","iopub.execute_input":"2021-06-19T14:01:37.293111Z","iopub.status.idle":"2021-06-19T14:01:37.298980Z","shell.execute_reply.started":"2021-06-19T14:01:37.293052Z","shell.execute_reply":"2021-06-19T14:01:37.298203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}