{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)\n\nMAX_LENGTH = 512 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nMAX_SAMPLE = None # set a small number for experimentation, set None for production.\n\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\nextra_path = '../input/coleridge-additional-gov-datasets-22000popular/data_set_800_with22000popular.csv'\n\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\nextra = pd.read_csv(extra_path)\nprint(f'No. raw training rows: {len(train)}')\nprint(f'No. extra rows: {len(extra)}')\n\ntrain = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')\n\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper\n        \ndef clean_training_text(text):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    #return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n    #text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text)).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                    u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                    u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                    u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                    u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                    \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\ndef combine_sentences(sentences):\n    num=0\n    com_sentences=[]\n    lenth=0\n    for sb in sentences:\n        lenth=lenth+len(sb.split(' '))\n    while num <= len(sentences)-1:\n        temp_sentence=sentences[num]\n        while (num+1)<=(len(sentences)-1):\n            #if len((temp_sentence+'. '+sentences[num+1]).split(' '))<=(512/((lenth/512)+1)):\n            if len((temp_sentence+'. '+sentences[num+1]).split(' '))<=512:\n                temp_sentence = temp_sentence+'. '+sentences[num+1]\n                num = num+1\n            else:\n                break\n        num = num+1\n        com_sentences.append(temp_sentence)\n    return com_sentences\n\n \ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        #words = sentence.split()\n        if len(sentence.split(' ')) > MAX_LENGTH:\n            for p in range(0, len(sentence), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(sentence[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\n\ndef find_sublist(big_list, small_list):\n    all_positions = []\n    for i in range(len(big_list) - len(small_list) + 1):\n        if small_list == big_list[i:i+len(small_list)]:\n            all_positions.append(i)\n    \n    return all_positions\n\ndef tag_sentence(sentence, labels): # requirement: both sentence and labels are already cleaned\n    sentence_words = sentence.split()\n    sentence_words_1 = sentence.lower().split()\n    \n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence.lower())\n                                  for label in labels): # positive sample\n        nes = ['O'] * len(sentence_words)\n        for label in labels:\n            label_words = label.split()\n\n            all_pos = find_sublist(sentence_words_1, label_words)\n            for pos in all_pos:\n                nes[pos] = 'B'\n                for i in range(pos+1, pos+len(label_words)):\n                    nes[i] = 'I'\n\n        return True, list(zip(sentence_words, nes))\n        \n    else: # negative sample\n        nes = ['O'] * len(sentence_words)\n        return False, list(zip(sentence_words, nes))\n    \ndef label_sentences(sentence,labels):\n    sentence_words = sentence.split()\n    nes='Dataset'\n    if labels is not None and any(re.findall(f'\\\\b{label}\\\\b', sentence)\n                                  for label in labels): # positive sample\n        return True,(sentence, nes)\n    else:\n        return False,(sentence, nes)\n    \nextra_label = set()\nfor i, el in extra[['title']].itertuples():\n    extra_label.add(clean_training_text(el))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-29T16:19:23.663563Z","iopub.execute_input":"2021-05-29T16:19:23.663974Z","iopub.status.idle":"2021-05-29T16:20:26.352353Z","shell.execute_reply.started":"2021-05-29T16:19:23.663891Z","shell.execute_reply":"2021-05-29T16:20:26.351453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#1-3701，900\nMAX_NUMBER=2101\nMIN_NUMBER=1751","metadata":{"execution":{"iopub.status.busy":"2021-05-29T16:20:26.35386Z","iopub.execute_input":"2021-05-29T16:20:26.354138Z","iopub.status.idle":"2021-05-29T16:20:26.359406Z","shell.execute_reply.started":"2021-05-29T16:20:26.354111Z","shell.execute_reply":"2021-05-29T16:20:26.358223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_pos, cnt_neg = 0, 0 # number of sentences that contain/not contain labels\nner_data = []\ncount = 0\npbar = tqdm(total=(MAX_NUMBER - MIN_NUMBER))\nfor i, id, dataset_label in train[['Id', 'dataset_label']].itertuples():\n    count = count+1\n    if MAX_NUMBER > count >= MIN_NUMBER:\n        #papers\n        paper = papers[id]\n    \n        # labels\n        labels = dataset_label.lower().split('|')\n        labels = [clean_training_text(label) for label in labels]\n        for el in extra_label:\n            labels.append(el)\n    \n        # sentences\n        sentences = [clean_training_text(sentence) for section in paper \n                     for sentence in section['text'].split('.') \n                    ]\n        sentences = combine_sentences(sentences) #make short sentences upward to 512\n        sentences = shorten_sentences(sentences) # make long sentences downward to 512\n        sentences = [sentence for sentence in sentences if len(sentence.split(' ')) > 10] # only accept sentences with length > 10 chars\n    \n    # positive sample\n        for sentence in sentences:\n            is_positive, tags = tag_sentence(sentence, labels)\n            if is_positive:\n                cnt_pos += 1\n                ner_data.append(tags)\n    else:\n        continue\n    # process bar\n    pbar.update(1)\n    #pbar.set_description(f\"Training data size: {cnt_pos} positives + {cnt_neg} negatives\")\n   # pbar.set_description(f\"Training data size: {cnt_pos} positives\")\n\n# shuffling\nrandom.shuffle(ner_data)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T16:20:26.361523Z","iopub.execute_input":"2021-05-29T16:20:26.361956Z","iopub.status.idle":"2021-05-29T23:00:49.71719Z","shell.execute_reply.started":"2021-05-29T16:20:26.361901Z","shell.execute_reply":"2021-05-29T23:00:49.715885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('train_ner_pos_512_10p_U_F6.json', 'w') as f:\n    for row in ner_data:\n        words, nes = list(zip(*row))\n        row_json = {'tokens' : words, 'tags' : nes}\n        json.dump(row_json, f)\n        f.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-05-29T23:00:49.719547Z","iopub.execute_input":"2021-05-29T23:00:49.720029Z","iopub.status.idle":"2021-05-29T23:00:51.129311Z","shell.execute_reply.started":"2021-05-29T23:00:49.719966Z","shell.execute_reply":"2021-05-29T23:00:51.128441Z"},"trusted":true},"execution_count":null,"outputs":[]}]}