{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:39:16.901332Z","iopub.execute_input":"2021-05-23T03:39:16.901854Z","iopub.status.idle":"2021-05-23T03:39:16.905505Z","shell.execute_reply.started":"2021-05-23T03:39:16.901788Z","shell.execute_reply":"2021-05-23T03:39:16.904605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"!cp /root/.cache/pip/wheels/d1/30/66/3b27c9cec890a8550e25bf266efeb2c78836fcb5331c0c1e5c/transformers-4.7.0.dev0-py3-none-any.whl ./","metadata":{}},{"cell_type":"markdown","source":"!cp /root/.cache/pip/wheels/d1/30/66/3b27c9cec890a8550e25bf266efeb2c78836fcb5331c0c1e5c/transformers-4.7.0.dev0-py3-none-any.whl ./","metadata":{}},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"#!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n#!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n#!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n#!pip install ../input/transformer470/heel-dir/transformers-4.7.0.dev0-py3-none-any.whl\n#!conda install ../input/bigbird12820epoch/transformers-master/transformers-master","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:38:41.323277Z","iopub.execute_input":"2021-05-21T03:38:41.323763Z","iopub.status.idle":"2021-05-21T03:39:44.22025Z","shell.execute_reply.started":"2021-05-21T03:38:41.323732Z","shell.execute_reply":"2021-05-21T03:39:44.218913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install transformers --no-index --find-links=file:///kaggle/input/transformer470/heel-dir","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:39:44.222772Z","iopub.execute_input":"2021-05-21T03:39:44.223231Z","iopub.status.idle":"2021-05-21T03:39:54.403615Z","shell.execute_reply.started":"2021-05-21T03:39:44.223187Z","shell.execute_reply":"2021-05-21T03:39:54.402429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n#import datasets\n\n#import torch\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n#import torch\n#from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline, TFAutoModelForSequenceClassification\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-23T03:24:07.392589Z","iopub.execute_input":"2021-05-23T03:24:07.39296Z","iopub.status.idle":"2021-05-23T03:24:08.218438Z","shell.execute_reply.started":"2021-05-23T03:24:07.392926Z","shell.execute_reply":"2021-05-23T03:24:08.217618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\nextra_path = '../input/bigger-govt-dataset-list/data_set_800.csv'\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\n#for paper_id in train['Id'].unique():\n #   with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n  #      paper = json.load(f)\n   #     papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:39:19.154557Z","iopub.execute_input":"2021-05-23T03:39:19.154917Z","iopub.status.idle":"2021-05-23T03:40:16.628435Z","shell.execute_reply.started":"2021-05-23T03:39:19.154887Z","shell.execute_reply":"2021-05-23T03:40:16.627457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\nextra = pd.read_csv(extra_path)\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-05-21T18:05:38.302358Z","iopub.execute_input":"2021-05-21T18:05:38.302768Z","iopub.status.idle":"2021-05-21T18:05:38.327204Z","shell.execute_reply.started":"2021-05-21T18:05:38.302732Z","shell.execute_reply":"2021-05-21T18:05:38.326294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-05-21T18:05:40.276137Z","iopub.execute_input":"2021-05-21T18:05:40.276511Z","iopub.status.idle":"2021-05-21T18:05:40.339002Z","shell.execute_reply.started":"2021-05-21T18:05:40.276478Z","shell.execute_reply":"2021-05-21T18:05:40.337789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#'international data base',\n                #'homescan and ces'\n                 #'online education database',\n    #'a (A)'+A=0.593\n    #'a'+A=0.597\n    #a (A) = ?\nupgrade_set = {\n                'Program for International Student Assessment',\n                'PISA',\n                'Progress in International reading Literacy Study',\n                'PIRLS',\n                #'Overland Surges from Hurricanes SLOSH basin models',\n                #'Hurricanes SLOSH basin models',\n                #'noaa ocm slr viewer',\n                'North Carolina Floodplain Mapping Program',\n                'North Carolina Emergency Management Spatial Data Download portal',\n                #'NC Floodplain Mapping Program',\n                'NCFMP',\n                'NCFMP program',\n                'NCFMP FFE data',\n                #'NPS',\n                'NPS building data',\n                'Organization for Economic Cooperation and Development',\n                #'OECD'\n                'TIMSS',\n                'Trends in International Mathematics and Science Study',\n                'FoodAPS',\n                #'Information Resource Incorporated (IRI) Consumer Network Panel (CNP) courtesy of the USDA Economic Research Service (ERS)',\n                #'Consumer Expenditure Survey (CES) of the US Bureau of Labor Statistics',\n                'Consumer Expenditure Survey (CES)',\n                'Information Resource Incorporated (IRI)'\n              }","metadata":{"execution":{"iopub.status.busy":"2021-05-22T07:46:53.347945Z","iopub.execute_input":"2021-05-22T07:46:53.348437Z","iopub.status.idle":"2021-05-22T07:46:53.354227Z","shell.execute_reply.started":"2021-05-22T07:46:53.348362Z","shell.execute_reply":"2021-05-22T07:46:53.352851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#all_labels=set()\nfor up in upgrade_set:\n    all_labels.add(up.lower())","metadata":{"execution":{"iopub.status.busy":"2021-05-21T18:05:53.151529Z","iopub.execute_input":"2021-05-21T18:05:53.151924Z","iopub.status.idle":"2021-05-21T18:05:53.156369Z","shell.execute_reply.started":"2021-05-21T18:05:53.151892Z","shell.execute_reply":"2021-05-21T18:05:53.1552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_labels","metadata":{"execution":{"iopub.status.busy":"2021-05-21T18:05:55.064929Z","iopub.execute_input":"2021-05-21T18:05:55.065301Z","iopub.status.idle":"2021-05-21T18:05:55.074602Z","shell.execute_reply.started":"2021-05-21T18:05:55.065266Z","shell.execute_reply":"2021-05-21T18:05:55.073386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(text):\n    #return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\ndef clean_training_text(text):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    #return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n    #text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text)).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                    u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                    u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                    u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                    u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                    \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-05-23T03:23:33.986232Z","iopub.execute_input":"2021-05-23T03:23:33.986575Z","iopub.status.idle":"2021-05-23T03:23:33.996046Z","shell.execute_reply.started":"2021-05-23T03:23:33.986547Z","shell.execute_reply":"2021-05-23T03:23:33.99519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#extra_label = set()\nfor i, el in extra[['title']].itertuples():\n    all_labels.add(clean_text(el))","metadata":{"execution":{"iopub.status.busy":"2021-05-21T18:06:29.248188Z","iopub.execute_input":"2021-05-21T18:06:29.24853Z","iopub.status.idle":"2021-05-21T18:06:29.297985Z","shell.execute_reply.started":"2021-05-21T18:06:29.248502Z","shell.execute_reply":"2021-05-21T18:06:29.296883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds = []\nliteral_compare=[]\ntest_match=[]\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if len(label.split(' ')) == 1:\n            sents = [clean_training_text(sentence).lower() for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n            #sents = [sentence for section in paper \n             #    for sentence in section['text'].split('.')\n              #  ]\n            for sent in sents:\n                for tk in sent.split(' ') :\n                    if label == tk :\n                    #if label == clean_training_text(tk).lower() :\n                        labels.add(clean_text(label))\n                        #test_match.append(sent)\n                        break\n        elif len(label.split(' ')) > 1:\n            if label in text_1 or label in text_2:\n            \n            #sents = [clean_training_text(sentence).lower() for section in paper \n             #    for sentence in section['text'].split('.')\n              #  ]\n            #sents = [sentence for section in paper \n             #    for sentence in section['text'].split('.')\n              #  ]\n            #for sent in sents:\n             #   if label in sent:\n                #if label in clean_training_text(sent).lower():\n                    labels.add(clean_text(label))\n                    #test_match.append(sent)\n        else:\n            continue\n    for what in labels:\n        literal_compare.append(what)\n    \n    literal_preds.append('|'.join(labels))\n#如果label为一个token，则按照token进行比对，如果是一个句子，则按照sublist比对","metadata":{"execution":{"iopub.status.busy":"2021-05-21T18:09:50.906153Z","iopub.execute_input":"2021-05-21T18:09:50.906509Z","iopub.status.idle":"2021-05-21T18:09:53.457127Z","shell.execute_reply.started":"2021-05-21T18:09:50.906466Z","shell.execute_reply":"2021-05-21T18:09:53.456215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds[:5]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T18:10:00.720975Z","iopub.execute_input":"2021-05-21T18:10:00.721341Z","iopub.status.idle":"2021-05-21T18:10:00.727641Z","shell.execute_reply.started":"2021-05-21T18:10:00.721311Z","shell.execute_reply":"2021-05-21T18:10:00.726809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_match","metadata":{"execution":{"iopub.status.busy":"2021-05-21T18:09:59.418769Z","iopub.execute_input":"2021-05-21T18:09:59.419132Z","iopub.status.idle":"2021-05-21T18:09:59.424933Z","shell.execute_reply.started":"2021-05-21T18:09:59.419102Z","shell.execute_reply":"2021-05-21T18:09:59.423909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MultiModel prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"MAX_LENGTH = 512 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\nTOKENIZER_PATH = '../input/mnt-model/xlnet-base-cased/xlnet-base-cased'\nPRETRAINED_PATH = '../input/mnt-model/xlnet-base-cased/xlnet-base-cased'\n\nMPNETPRETRAINED_PATH = '../input/mnt-model/MP_512_10_po_45/MP_512_10_po_15'\nBERTPRETRAINED_PATH='../input/bert64poner15epoch/64po_ner_bert_15p'\nBDPRETRAINED_PATH='../input/bigbird12820epoch/bigbird-128-20epoch/bigbird-128-20epoch'\n\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\n\nTRAIN_PATH = '../input/data-ner-51210p/train_ner_pos_512_10p.json'\nVAL_PATH = '../input/data-ner-51210p/train_ner_pos_512_10p.json'\nTRAIN_PATH_BD = '../input/train-ner-pos400-100/train_ner_pos400_100.json'\nVAL_PATH_BD = '../input/train-ner-pos400-100/train_ner_pos400_100.json'\n\nPREDICTION_SAVE_PATH = './pred'\nBERTPREDICTION_FILE = 'test_predictions.txt'\nMPNETPREDICTION_FILE = 'test_predictions.txt'\nBDPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:05.413841Z","iopub.execute_input":"2021-05-21T03:40:05.414204Z","iopub.status.idle":"2021-05-21T03:40:05.427272Z","shell.execute_reply.started":"2021-05-21T03:40:05.414163Z","shell.execute_reply":"2021-05-21T03:40:05.42593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir ./pred","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:05.428734Z","iopub.execute_input":"2021-05-21T03:40:05.429049Z","iopub.status.idle":"2021-05-21T03:40:06.178357Z","shell.execute_reply.started":"2021-05-21T03:40:05.429018Z","shell.execute_reply":"2021-05-21T03:40:06.177148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"train = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:06.180376Z","iopub.execute_input":"2021-05-21T03:40:06.180858Z","iopub.status.idle":"2021-05-21T03:40:06.864963Z","shell.execute_reply.started":"2021-05-21T03:40:06.180791Z","shell.execute_reply":"2021-05-21T03:40:06.864184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        #words = sentence.split()\n        if len(sentence.split(' ')) > MAX_LENGTH:\n            for p in range(0, len(sentence), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(sentence[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n    \ndef combine_sentences(sentences):\n    num=0\n    com_sentences=set()\n    lenth=0\n    for sb in sentences:\n        lenth=lenth+len(sb.split(' '))\n    while num <= len(sentences)-1:\n        temp_sentence=sentences[num]\n        while (num+1)<=(len(sentences)-1):\n            if len((temp_sentence+'. '+sentences[num+1]).split(' '))<=(512/((lenth/512)+1)):\n                temp_sentence = temp_sentence+'. '+sentences[num+1]\n                num = num+1\n            else:\n                break\n        num = num+1\n        com_sentences.add(temp_sentence)\n    return com_sentences\n\ndef shorten_128sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > 400:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:06.866209Z","iopub.execute_input":"2021-05-21T03:40:06.86669Z","iopub.status.idle":"2021-05-21T03:40:06.874648Z","shell.execute_reply.started":"2021-05-21T03:40:06.866657Z","shell.execute_reply":"2021-05-21T03:40:06.873258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pipline for TC","metadata":{}},{"cell_type":"markdown","source":"tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, use_fast=True)\nmodel = TFAutoModelForSequenceClassification.from_pretrained(PRETRAINED_PATH)\n\nTC = pipeline(\n    'text-classification', \n    model=model,\n    tokenizer=tokenizer,\n    device=0 if torch.cuda.is_available() else -1\n)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T14:23:55.888341Z","iopub.execute_input":"2021-05-20T14:23:55.888683Z","iopub.status.idle":"2021-05-20T14:24:04.836187Z","shell.execute_reply.started":"2021-05-20T14:23:55.888654Z","shell.execute_reply":"2021-05-20T14:24:04.835102Z"}}},{"cell_type":"markdown","source":"### Test rows for mp &berd","metadata":{}},{"cell_type":"markdown","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in sample_submission['Id']:\n    # load paper\n    paper = papers[paper_id]\n    \n    # extract sentences\n    sentences = [clean_training_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = combine_sentences(sentences)\n    sentences = shorten_sentences(sentences) # make sentences short\n    #sentences = [sentence for sentence in sentences if 10 < len(sentence.split(' ')) <= 512] # only accept sentences with length > 10 chars\n    #sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n    #sentences = [sentence for sentence in sentences if TC(sentence)[0]['label']=='Dataset' and 0.3<=TC(sentence)[0]['score']<0.4]\n        \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words)\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:32:38.954925Z","iopub.execute_input":"2021-05-20T15:32:38.955301Z","iopub.status.idle":"2021-05-20T15:35:34.80173Z","shell.execute_reply.started":"2021-05-20T15:32:38.955268Z","shell.execute_reply":"2021-05-20T15:35:34.80061Z"}}},{"cell_type":"code","source":"#!empty_cache()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:06.876018Z","iopub.execute_input":"2021-05-21T03:40:06.876291Z","iopub.status.idle":"2021-05-21T03:40:06.897134Z","shell.execute_reply.started":"2021-05-21T03:40:06.876264Z","shell.execute_reply":"2021-05-21T03:40:06.895864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"os.environ[\"BERTMODEL_PATH\"] = f\"{BERTPRETRAINED_PATH}\"\nos.environ[\"MPNETMODEL_PATH\"] = f\"{MPNETPRETRAINED_PATH}\"\nos.environ[\"BDMODEL_PATH\"] = f\"{BDPRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TRAIN_FILE_BD\"] = f\"{TRAIN_PATH_BD}\"\nos.environ[\"VALIDATION_FILE_BD\"] = f\"{VAL_PATH_BD}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:06.898576Z","iopub.execute_input":"2021-05-21T03:40:06.898882Z","iopub.status.idle":"2021-05-21T03:40:06.913609Z","shell.execute_reply.started":"2021-05-21T03:40:06.898852Z","shell.execute_reply":"2021-05-21T03:40:06.912404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessart directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:06.915573Z","iopub.execute_input":"2021-05-21T03:40:06.915931Z","iopub.status.idle":"2021-05-21T03:40:07.674807Z","shell.execute_reply.started":"2021-05-21T03:40:06.915897Z","shell.execute_reply":"2021-05-21T03:40:07.673455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"tokenizer = AutoTokenizer.from_pretrained('../input/mpnet-64po-ner-15epoch/output', use_fast=True)\nmodel = AutoModelForTokenClassification.from_pretrained('../input/mpnet-64po-ner-15epoch/output')\n\nmlm = pipeline(\n    'ner', \n    model=model,\n    tokenizer=tokenizer,\n    device=0 if torch.cuda.is_available() else -1\n)\n","metadata":{}},{"cell_type":"markdown","source":"def Mpnet_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MPNETMODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:35:35.557774Z","iopub.execute_input":"2021-05-20T15:35:35.558116Z","iopub.status.idle":"2021-05-20T15:35:35.563519Z","shell.execute_reply.started":"2021-05-20T15:35:35.558085Z","shell.execute_reply":"2021-05-20T15:35:35.562725Z"}}},{"cell_type":"markdown","source":"def bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$BERTMODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{}},{"cell_type":"markdown","source":"bd_outputs = []\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)\nfor batch_begin in range(0, len(bd_test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in bd_test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bd_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{BDPREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bd_outputs += [pred.split() for pred in this_preds]","metadata":{}},{"cell_type":"markdown","source":"mpnet_outputs = []\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    Mpnet_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{MPNETPREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        mpnet_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:35:35.564557Z","iopub.execute_input":"2021-05-20T15:35:35.564825Z","iopub.status.idle":"2021-05-20T15:36:36.438283Z","shell.execute_reply.started":"2021-05-20T15:35:35.5648Z","shell.execute_reply":"2021-05-20T15:36:36.437139Z"}}},{"cell_type":"markdown","source":"bert_outputs = []\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{BERTPREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{}},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"markdown","source":"# get test sentences\ntest_sentences_1 = [row['tokens'] for row in test_rows]\n#test_sentences_2 = [row['tokens'] for row in test_rows]\n#test_sentences_3 = [row['tokens'] for row in bd_test_rows]\n\n#del test_rows, bd_test_row","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:36:36.440497Z","iopub.execute_input":"2021-05-20T15:36:36.440796Z","iopub.status.idle":"2021-05-20T15:36:36.446146Z","shell.execute_reply.started":"2021-05-20T15:36:36.440767Z","shell.execute_reply":"2021-05-20T15:36:36.445368Z"}}},{"cell_type":"markdown","source":"bd_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences_3[:length], bd_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bd_dataset_labels.append(labels)\n    \n    del test_sentences_3[:length], bd_outputs[:length]","metadata":{}},{"cell_type":"markdown","source":"mpnet_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences_1[:length], mpnet_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    mpnet_dataset_labels.append(labels)\n    \n    del test_sentences_1[:length], mpnet_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:36:36.447496Z","iopub.execute_input":"2021-05-20T15:36:36.447939Z","iopub.status.idle":"2021-05-20T15:36:36.466327Z","shell.execute_reply.started":"2021-05-20T15:36:36.4479Z","shell.execute_reply":"2021-05-20T15:36:36.465344Z"}}},{"cell_type":"markdown","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences_2[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences_2[:length], bert_outputs[:length]","metadata":{}},{"cell_type":"code","source":"#bert_dataset_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.676656Z","iopub.execute_input":"2021-05-21T03:40:07.676989Z","iopub.status.idle":"2021-05-21T03:40:07.681407Z","shell.execute_reply.started":"2021-05-21T03:40:07.676944Z","shell.execute_reply":"2021-05-21T03:40:07.680072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#mpnet_dataset_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.682623Z","iopub.execute_input":"2021-05-21T03:40:07.683023Z","iopub.status.idle":"2021-05-21T03:40:07.696361Z","shell.execute_reply.started":"2021-05-21T03:40:07.682986Z","shell.execute_reply":"2021-05-21T03:40:07.695324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bd_dataset_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.698025Z","iopub.execute_input":"2021-05-21T03:40:07.698341Z","iopub.status.idle":"2021-05-21T03:40:07.709608Z","shell.execute_reply.started":"2021-05-21T03:40:07.698313Z","shell.execute_reply":"2021-05-21T03:40:07.708594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"markdown","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\nfiltered_mpnet_labels = []\nfiltered_bd_labels = []\n\n#for labels in bert_dataset_labels:\n #   filtered = []\n    \n  #  for label in sorted(labels, key=len):\n   #     label = clean_text(label)\n    #    if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.5 for got_label in filtered):\n     #       if all(jaccard_similarity(label,get_label)<0.5 for get_label in literal_compare):\n      #          filtered.append(label)\n    #filtered_bert_labels.append(filtered)\n                \nfor labels in mpnet_dataset_labels:\n    filtered = set()\n    \n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.5 for got_label in filtered):\n            if all(jaccard_similarity(label,get_label)<0.5 for get_label in literal_compare):\n                filtered.add(label)\n    #filtered_mpnet_labels.append('|'.join(filtered))\n    filtered_mpnet_labels.append(filtered)\n                \n#for labels in bd_dataset_labels:\n #   filtered = []\n    \n  #  for label in sorted(labels, key=len):\n   #     label = clean_text(label)\n    #    if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.5 for got_label in filtered):\n     #       if all(jaccard_similarity(label,get_label)<0.5 for get_label in literal_compare) and label not in ['each']:\n      #          filtered.append(label)\n    \n   # filtered_bd_labels.append(filtered)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T15:27:44.694063Z","iopub.execute_input":"2021-05-20T15:27:44.694561Z","iopub.status.idle":"2021-05-20T15:27:44.708464Z","shell.execute_reply.started":"2021-05-20T15:27:44.69452Z","shell.execute_reply":"2021-05-20T15:27:44.707651Z"}}},{"cell_type":"code","source":"#前者减去后者\ndef cut(mpnet_dataset_labels,bert_dataset_labels):\n    combined_labels=[]\n    for mlabels,blabels in zip(mpnet_dataset_labels,bert_dataset_labels):\n        filtered = []  \n        for mlabel in sorted(mlabels, key=len):\n            #与前面已知的相似度小于0.75\n            if len(filtered) == 0 or all(jaccard_similarity(mlabel, got_label) < 0.75 for got_label in filtered):\n                #先加入与blabels的所有标签都不同的\n                if all(jaccard_similarity(mlabel, blabel) < 0.75 for blabel in blabels):\n                    filtered.append(mlabel)\n\n        combined_labels.append(filtered)\n    return combined_labels            \n                                      \n","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.711189Z","iopub.execute_input":"2021-05-21T03:40:07.711535Z","iopub.status.idle":"2021-05-21T03:40:07.723861Z","shell.execute_reply.started":"2021-05-21T03:40:07.711502Z","shell.execute_reply":"2021-05-21T03:40:07.722694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#取并集\ndef combine(literal_preds, filtered_bert_labels):\n    combine_predictions = []\n    k=set()\n    for literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n        k=set()\n        if literal_match and bert_pred :\n            k.update(literal_match)\n            k.update(bert_pred)\n            combine_predictions.append('|'.join(k))\n        else:\n            if literal_match:\n                combine_predictions.append('|'.join(literal_match))\n            else :\n                combine_predictions.append('|'.join(bert_pred))\n    return combine_predictions\n","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.725067Z","iopub.execute_input":"2021-05-21T03:40:07.725344Z","iopub.status.idle":"2021-05-21T03:40:07.742733Z","shell.execute_reply.started":"2021-05-21T03:40:07.725317Z","shell.execute_reply":"2021-05-21T03:40:07.741595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filtered_bert_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.744044Z","iopub.execute_input":"2021-05-21T03:40:07.744343Z","iopub.status.idle":"2021-05-21T03:40:07.755135Z","shell.execute_reply.started":"2021-05-21T03:40:07.744316Z","shell.execute_reply":"2021-05-21T03:40:07.75425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" #filtered_mpnet_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.756747Z","iopub.execute_input":"2021-05-21T03:40:07.757079Z","iopub.status.idle":"2021-05-21T03:40:07.771001Z","shell.execute_reply.started":"2021-05-21T03:40:07.757047Z","shell.execute_reply":"2021-05-21T03:40:07.770087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filtered_bd_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.771958Z","iopub.execute_input":"2021-05-21T03:40:07.772248Z","iopub.status.idle":"2021-05-21T03:40:07.78457Z","shell.execute_reply.started":"2021-05-21T03:40:07.772222Z","shell.execute_reply":"2021-05-21T03:40:07.783651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#filtered_bd_labels","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.785547Z","iopub.execute_input":"2021-05-21T03:40:07.785809Z","iopub.status.idle":"2021-05-21T03:40:07.798517Z","shell.execute_reply.started":"2021-05-21T03:40:07.785783Z","shell.execute_reply":"2021-05-21T03:40:07.797597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Combine based on Jaccard score","metadata":{}},{"cell_type":"code","source":"#mpnet_bert = cut(filtered_mpnet_labels,filtered_bert_labels)\n#bd_bert = cut(filtered_bd_labels,filtered_bert_labels)\n#mpnet_bd_bert = cut(mpn#et_bert,bd_bert)\n#mp_bd_add = combine(mpnet_bd_bert,bd_bert)\n#final = combine(literal_preds,filtered_mpnet_labels)\n#final = combine(filtered_mpnet_labels, literal_preds)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.799497Z","iopub.execute_input":"2021-05-21T03:40:07.799753Z","iopub.status.idle":"2021-05-21T03:40:07.812345Z","shell.execute_reply.started":"2021-05-21T03:40:07.799729Z","shell.execute_reply":"2021-05-21T03:40:07.811319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#final=filtered_mpnet_labels","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.813923Z","iopub.execute_input":"2021-05-21T03:40:07.814283Z","iopub.status.idle":"2021-05-21T03:40:07.825806Z","shell.execute_reply.started":"2021-05-21T03:40:07.81425Z","shell.execute_reply":"2021-05-21T03:40:07.824595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"markdown","source":"final_predictions = []\nk=[]\nfor literal_match, model_pred in zip(literal_preds, final):\n    k=[]\n    if literal_match and model_pred :\n        k.append(literal_match)\n        k.append(model_pred)\n        final_predictions.append('|'.join(k))\n    else:\n        if literal_match:\n            final_predictions.append(literal_match)\n        else :\n            final_predictions.append(model_pred)\nfinal_predictions","metadata":{"execution":{"iopub.status.busy":"2021-05-20T01:24:44.178582Z","iopub.execute_input":"2021-05-20T01:24:44.178863Z","iopub.status.idle":"2021-05-20T01:24:44.186745Z","shell.execute_reply.started":"2021-05-20T01:24:44.178837Z","shell.execute_reply":"2021-05-20T01:24:44.185734Z"}}},{"cell_type":"code","source":"#sample_submission['PredictionString'] = final_predictions\nsample_submission['PredictionString'] = literal_preds\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.826916Z","iopub.execute_input":"2021-05-21T03:40:07.827213Z","iopub.status.idle":"2021-05-21T03:40:07.852638Z","shell.execute_reply.started":"2021-05-21T03:40:07.827185Z","shell.execute_reply":"2021-05-21T03:40:07.851438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-21T03:40:07.857365Z","iopub.execute_input":"2021-05-21T03:40:07.857675Z","iopub.status.idle":"2021-05-21T03:40:07.866937Z","shell.execute_reply.started":"2021-05-21T03:40:07.857648Z","shell.execute_reply":"2021-05-21T03:40:07.865736Z"},"trusted":true},"execution_count":null,"outputs":[]}]}