{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:43:29.230111Z","iopub.execute_input":"2021-06-08T10:43:29.230897Z","iopub.status.idle":"2021-06-08T10:43:29.243994Z","shell.execute_reply.started":"2021-06-08T10:43:29.230694Z","shell.execute_reply":"2021-06-08T10:43:29.24285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n!pip install ../input/recordlinkage/jellyfish-0.8.2-cp37-cp37m-manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:43:29.246763Z","iopub.execute_input":"2021-06-08T10:43:29.24712Z","iopub.status.idle":"2021-06-08T10:45:29.90647Z","shell.execute_reply.started":"2021-06-08T10:43:29.24709Z","shell.execute_reply":"2021-06-08T10:45:29.90476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom scipy.spatial import distance\n\nrandom.seed(123)\nnp.random.seed(456)\n\nimport jellyfish as jf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-08T10:45:29.908779Z","iopub.execute_input":"2021-06-08T10:45:29.909257Z","iopub.status.idle":"2021-06-08T10:45:31.038282Z","shell.execute_reply.started":"2021-06-08T10:45:29.909206Z","shell.execute_reply":"2021-06-08T10:45:31.036921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper\n        \n#papers","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:45:31.040303Z","iopub.execute_input":"2021-06-08T10:45:31.040701Z","iopub.status.idle":"2021-06-08T10:46:41.966241Z","shell.execute_reply.started":"2021-06-08T10:45:31.040665Z","shell.execute_reply":"2021-06-08T10:46:41.96469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:41.969758Z","iopub.execute_input":"2021-06-08T10:46:41.970138Z","iopub.status.idle":"2021-06-08T10:46:42.010078Z","shell.execute_reply.started":"2021-06-08T10:46:41.970105Z","shell.execute_reply":"2021-06-08T10:46:42.008997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:42.013997Z","iopub.execute_input":"2021-06-08T10:46:42.014482Z","iopub.status.idle":"2021-06-08T10:46:42.041986Z","shell.execute_reply.started":"2021-06-08T10:46:42.014433Z","shell.execute_reply":"2021-06-08T10:46:42.040887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:42.043951Z","iopub.execute_input":"2021-06-08T10:46:42.044392Z","iopub.status.idle":"2021-06-08T10:46:42.076476Z","shell.execute_reply.started":"2021-06-08T10:46:42.044346Z","shell.execute_reply":"2021-06-08T10:46:42.074782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# finding unique values in each column\n[print(f\"{col}:{len(train[col].unique())}\") for col in train.columns]","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:42.078242Z","iopub.execute_input":"2021-06-08T10:46:42.078826Z","iopub.status.idle":"2021-06-08T10:46:42.111579Z","shell.execute_reply.started":"2021-06-08T10:46:42.078786Z","shell.execute_reply":"2021-06-08T10:46:42.110644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Visualisation","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nfrom nltk.probability import FreqDist\n\nwords = list(train['cleaned_label'].values)\nstopwords=['ourselves', 'hers','the','of','and','in', 'between', 'yourself', 'but', 'again','of', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than']\nsplit_words=[]\nfor word in words:\n    lo_w=[]\n    list_of_words=str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:42.113172Z","iopub.execute_input":"2021-06-08T10:46:42.113526Z","iopub.status.idle":"2021-06-08T10:46:43.364267Z","shell.execute_reply.started":"2021-06-08T10:46:42.113492Z","shell.execute_reply":"2021-06-08T10:46:43.362838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostcommon = FreqDist(allwords).most_common(100)\nwordcloud = WordCloud(width=1600, height=800, background_color='white', stopwords=STOPWORDS).generate(str(mostcommon))\nfig = plt.figure(figsize=(30,10), facecolor='white')\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis('off')\nplt.title('Top 100 Most Common Words in cleaned_label', fontsize=50)\nplt.tight_layout(pad=0)\nplt.show()\n\nmostcommon_small = FreqDist(allwords).most_common(25)\nx, y = zip(*mostcommon_small)\nplt.figure(figsize=(50,30))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Words', fontsize=50)\nplt.ylabel('Frequency of Words', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40)\nplt.tight_layout(pad=0)\nplt.title('Freq of 25 Most Common Words in cleaned_label', fontsize=60)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:43.366009Z","iopub.execute_input":"2021-06-08T10:46:43.366488Z","iopub.status.idle":"2021-06-08T10:46:46.658654Z","shell.execute_reply.started":"2021-06-08T10:46:43.36644Z","shell.execute_reply":"2021-06-08T10:46:46.65766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.659722Z","iopub.execute_input":"2021-06-08T10:46:46.660023Z","iopub.status.idle":"2021-06-08T10:46:46.725328Z","shell.execute_reply.started":"2021-06-08T10:46:46.659994Z","shell.execute_reply":"2021-06-08T10:46:46.724391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add extra data","metadata":{}},{"cell_type":"code","source":"# extDf = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\n# extDf = pd.read_csv('../input/bigger-govt-dataset-list/data_set_26897.csv')\n\n# extDf = pd.read_csv('../input/filtered-bigger-govt-dataset/ExtraLabelsCleaned.txt')\n# extDf = extDf.rename(columns={'Label': 'title'})\n# extDf = extDf.drop(' Hits',axis='columns')\n\n# extDf = pd.read_csv('../input/coleridge-additional-gov-datasets-22000popular/additional_gov_datasets_22000popular.csv')\n# extDf = pd.read_csv('../input/coleridge-additional-gov-datasets-22000popular/data_set_800_with8000popular.csv')\n\n# print(len(extDf))\n\n# extDf.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.726423Z","iopub.execute_input":"2021-06-08T10:46:46.726766Z","iopub.status.idle":"2021-06-08T10:46:46.733814Z","shell.execute_reply.started":"2021-06-08T10:46:46.726732Z","shell.execute_reply":"2021-06-08T10:46:46.732219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# added this in version 13\n# for l in extDf.title:\n#     all_labels.add(l)\n    \n# all_labels = set(all_labels)\n# print(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.735537Z","iopub.execute_input":"2021-06-08T10:46:46.735851Z","iopub.status.idle":"2021-06-08T10:46:46.746882Z","shell.execute_reply.started":"2021-06-08T10:46:46.735822Z","shell.execute_reply":"2021-06-08T10:46:46.745788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.748193Z","iopub.execute_input":"2021-06-08T10:46:46.74851Z","iopub.status.idle":"2021-06-08T10:46:46.761613Z","shell.execute_reply.started":"2021-06-08T10:46:46.74848Z","shell.execute_reply":"2021-06-08T10:46:46.760206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds = []\n\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if label in text_1 or label in text_2:\n            labels.add(clean_text(label))\n    \n    literal_preds.append('|'.join(labels))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.766492Z","iopub.execute_input":"2021-06-08T10:46:46.766848Z","iopub.status.idle":"2021-06-08T10:46:46.899757Z","shell.execute_reply.started":"2021-06-08T10:46:46.766815Z","shell.execute_reply":"2021-06-08T10:46:46.898435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def read_json_pub(filename, train_data_path=paper_train_folder, output='text'):\n#     json_path = os.path.join(train_data_path, (filename+'.json'))\n#     headings = []\n#     contents = []\n#     combined = []\n#     with open(json_path, 'r') as f:\n#         json_decode = json.load(f)\n#         for data in json_decode:\n#             headings.append(data.get('section_title'))\n#             contents.append(data.get('text'))\n#             combined.append(data.get('section_title'))\n#             combined.append(data.get('text'))\n    \n#     all_headings = ' '.join(headings)\n#     all_contents = ' '.join(contents)\n#     all_data = '. '.join(combined)\n    \n#     if output == 'text':\n#         return all_contents\n#     elif output == 'head':\n#         return all_headings\n#     else:\n#         return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.903511Z","iopub.execute_input":"2021-06-08T10:46:46.903839Z","iopub.status.idle":"2021-06-08T10:46:46.909261Z","shell.execute_reply.started":"2021-06-08T10:46:46.90381Z","shell.execute_reply":"2021-06-08T10:46:46.907675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # https://www.kaggle.com/chienhsianghung/external-datasets-matching-mlm\n# # https://www.kaggle.com/mlconsult/isin-big-dataset\n\n# def text_cleaning(text):\n#     '''\n#     Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n#     text - Sentence that needs to be cleaned\n#     '''\n#     text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n#     text = re.sub(' +', ' ', text)\n#     emoji_pattern = re.compile(\"[\"\n#                                u\"\\U0001F600-\\U0001F64F\"  # emoticons\n#                                u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n#                                u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n#                                u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n#                                \"]+\", flags=re.UNICODE)\n#     text = emoji_pattern.sub(r'', text)\n#     return text\n\n# literal_preds = []\n# to_append = []\n\n# for index, row in sample_submission.iterrows():\n#     to_append = [row['Id'],'']\n#     large_string = str(read_json_pub(row['Id'], paper_test_folder))\n#     clean_string = text_cleaning(large_string)\n    \n#     for index, row2 in extDf.iterrows():\n#         query_string = str(row2['title'])\n#         if query_string in clean_string:\n#             if to_append[1] != '' and clean_text(query_string) not in to_append[1]:\n#                 to_append[1] = to_append[1] + '|' + clean_text(query_string)\n#             if to_append[1] == '':\n#                 to_append[1] = clean_text(query_string)\n#     literal_preds.append(*to_append[1:])","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.911303Z","iopub.execute_input":"2021-06-08T10:46:46.911751Z","iopub.status.idle":"2021-06-08T10:46:46.923181Z","shell.execute_reply.started":"2021-06-08T10:46:46.911708Z","shell.execute_reply":"2021-06-08T10:46:46.922163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.924702Z","iopub.execute_input":"2021-06-08T10:46:46.925001Z","iopub.status.idle":"2021-06-08T10:46:46.946073Z","shell.execute_reply.started":"2021-06-08T10:46:46.924971Z","shell.execute_reply":"2021-06-08T10:46:46.94464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bert prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"# MAX_LENGTH = 64 # max no. words for each sentence.\n# OVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\n# PREDICT_BATCH = 64000 \n\n# PRETRAINED_PATH = '../input/coleridge-bert-models/output'\n# TEST_INPUT_SAVE_PATH = './input_data'\n# TEST_NER_DATA_FILE = 'test_ner_input.json'\n# TRAIN_PATH = '../input/coleridge-bert-models/train_ner.json'\n# VAL_PATH = '../input/coleridge-bert-models/train_ner.json'\n\n# PREDICTION_SAVE_PATH = './pred'\n# PREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.948194Z","iopub.execute_input":"2021-06-08T10:46:46.948692Z","iopub.status.idle":"2021-06-08T10:46:46.958304Z","shell.execute_reply.started":"2021-06-08T10:46:46.948644Z","shell.execute_reply":"2021-06-08T10:46:46.95672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"# train = train.groupby('Id').agg({\n#     'pub_title': 'first',\n#     'dataset_title': '|'.join,\n#     'dataset_label': '|'.join,\n#     'cleaned_label': '|'.join\n# }).reset_index()\n\n# print(f'No. grouped training rows: {len(train)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:46.960262Z","iopub.execute_input":"2021-06-08T10:46:46.960895Z","iopub.status.idle":"2021-06-08T10:46:47.65152Z","shell.execute_reply.started":"2021-06-08T10:46:46.960838Z","shell.execute_reply":"2021-06-08T10:46:47.650044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def clean_training_text(txt):\n#     \"\"\"\n#     similar to the default clean_text function but without lowercasing.\n#     \"\"\"\n#     return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\n# def shorten_sentences(sentences):\n#     short_sentences = []\n#     for sentence in sentences:\n#         words = sentence.split()\n#         if len(words) > MAX_LENGTH:\n#             for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n#                 short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n#         else:\n#             short_sentences.append(sentence)\n#     return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:47.65316Z","iopub.execute_input":"2021-06-08T10:46:47.653627Z","iopub.status.idle":"2021-06-08T10:46:47.661791Z","shell.execute_reply.started":"2021-06-08T10:46:47.653583Z","shell.execute_reply":"2021-06-08T10:46:47.660503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_rows = [] # test data in NER format\n# paper_length = [] # store the number of sentences each paper has\n\n# for paper_id in sample_submission['Id']:\n#     # load paper\n#     paper = papers[paper_id]\n    \n#     # extract sentences\n#     sentences = [clean_training_text(sentence) for section in paper \n#                  for sentence in section['text'].split('.')\n#                 ]\n#     sentences = shorten_sentences(sentences) # make sentences short\n#     sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 chars\n#     sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n        \n#     # collect all sentences in json\n#     for sentence in sentences:\n#         sentence_words = sentence.split()\n#         dummy_tags = ['O']*len(sentence_words)\n#         test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n#     # track which sentence belongs to which data point\n#     paper_length.append(len(sentences))\n    \n# print(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:47.663881Z","iopub.execute_input":"2021-06-08T10:46:47.664262Z","iopub.status.idle":"2021-06-08T10:46:47.72277Z","shell.execute_reply.started":"2021-06-08T10:46:47.66423Z","shell.execute_reply":"2021-06-08T10:46:47.721552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results","metadata":{}},{"cell_type":"code","source":"# os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\n# os.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\n# os.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\n# os.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\n# os.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:47.724613Z","iopub.execute_input":"2021-06-08T10:46:47.724938Z","iopub.status.idle":"2021-06-08T10:46:47.731848Z","shell.execute_reply.started":"2021-06-08T10:46:47.724907Z","shell.execute_reply":"2021-06-08T10:46:47.730029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # copy my_seqeval.py to the working directory because the input directory is non-writable\n# !cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# # make necessart directories and files\n# os.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:47.7337Z","iopub.execute_input":"2021-06-08T10:46:47.734028Z","iopub.status.idle":"2021-06-08T10:46:48.50898Z","shell.execute_reply.started":"2021-06-08T10:46:47.733996Z","shell.execute_reply":"2021-06-08T10:46:48.5073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def bert_predict():\n#     !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n#     --model_name_or_path \"$MODEL_PATH\" \\\n#     --train_file \"$TRAIN_FILE\" \\\n#     --validation_file \"$VALIDATION_FILE\" \\\n#     --test_file \"$TEST_FILE\" \\\n#     --output_dir \"$OUTPUT_DIR\" \\\n#     --report_to 'none' \\\n#     --seed 123 \\\n#     --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:48.511569Z","iopub.execute_input":"2021-06-08T10:46:48.512148Z","iopub.status.idle":"2021-06-08T10:46:48.520067Z","shell.execute_reply.started":"2021-06-08T10:46:48.512099Z","shell.execute_reply":"2021-06-08T10:46:48.518635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bert_outputs = []\n\n# for batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n#     # write data rows to input file\n#     with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n#         for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n#             json.dump(row, f)\n#             f.write('\\n')\n    \n#     # remove output dir\n#     !rm -r \"$OUTPUT_DIR\"\n    \n#     # do predict\n#     bert_predict()\n    \n#     # read predictions\n#     with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n#         this_preds = f.read().split('\\n')[:-1]\n#         bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:46:48.522227Z","iopub.execute_input":"2021-06-08T10:46:48.522706Z","iopub.status.idle":"2021-06-08T10:49:00.114768Z","shell.execute_reply.started":"2021-06-08T10:46:48.522661Z","shell.execute_reply":"2021-06-08T10:49:00.113117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"code","source":"# # get test sentences\n# test_sentences = [row['tokens'] for row in test_rows]\n\n# del test_rows","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:49:00.119954Z","iopub.execute_input":"2021-06-08T10:49:00.120335Z","iopub.status.idle":"2021-06-08T10:49:00.126659Z","shell.execute_reply.started":"2021-06-08T10:49:00.120303Z","shell.execute_reply":"2021-06-08T10:49:00.12557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bert_dataset_labels = [] # store all dataset labels for each publication\n\n# for length in paper_length:\n#     labels = set()\n#     for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n#         curr_phrase = ''\n#         for word, tag in zip(sentence, pred):\n#             if tag == 'B': # start a new phrase\n#                 if curr_phrase:\n#                     labels.add(curr_phrase)\n#                     curr_phrase = ''\n#                 curr_phrase = word\n#             elif tag == 'I' and curr_phrase: # continue the phrase\n#                 curr_phrase += ' ' + word\n#             else: # end last phrase (if any)\n#                 if curr_phrase:\n#                     labels.add(curr_phrase)\n#                     curr_phrase = ''\n#         # check if the label is the suffix of the sentence\n#         if curr_phrase:\n#             labels.add(curr_phrase)\n#             curr_phrase = ''\n    \n#     # record dataset labels for this publication\n#     bert_dataset_labels.append(labels)\n    \n#     del test_sentences[:length], bert_outputs[:length]","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:49:00.127953Z","iopub.execute_input":"2021-06-08T10:49:00.128318Z","iopub.status.idle":"2021-06-08T10:49:00.145717Z","shell.execute_reply.started":"2021-06-08T10:49:00.128281Z","shell.execute_reply":"2021-06-08T10:49:00.144594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# bert_dataset_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:49:00.147864Z","iopub.execute_input":"2021-06-08T10:49:00.148374Z","iopub.status.idle":"2021-06-08T10:49:00.164363Z","shell.execute_reply.started":"2021-06-08T10:49:00.148318Z","shell.execute_reply":"2021-06-08T10:49:00.163316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\n\nfor labels in bert_dataset_labels:\n    filtered = []\n    print(labels)\n\n    for label in sorted(labels, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n#         if len(filtered) == 0 or all(jf.jaro_winkler_similarity(label, got_label) < 0.75 for got_label in filtered):\n            filtered.append(label)\n            print(filtered)\n    \n    filtered_bert_labels.append('|'.join(filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:49:00.166076Z","iopub.execute_input":"2021-06-08T10:49:00.166435Z","iopub.status.idle":"2021-06-08T10:49:00.181443Z","shell.execute_reply.started":"2021-06-08T10:49:00.166376Z","shell.execute_reply":"2021-06-08T10:49:00.179897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_bert_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:49:00.183366Z","iopub.execute_input":"2021-06-08T10:49:00.183786Z","iopub.status.idle":"2021-06-08T10:49:00.20089Z","shell.execute_reply.started":"2021-06-08T10:49:00.183751Z","shell.execute_reply":"2021-06-08T10:49:00.199627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"# final_predictions = []\n# for literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n#     if literal_match:\n#         final_predictions.append(literal_match)\n#     else:\n#         print(\"we used BERT\")\n#         final_predictions.append(bert_pred)\n        \nfinal_predictions = []\nfor literal_match in literal_preds:\n    final_predictions.append(literal_match)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:49:00.207769Z","iopub.execute_input":"2021-06-08T10:49:00.208173Z","iopub.status.idle":"2021-06-08T10:49:00.215677Z","shell.execute_reply.started":"2021-06-08T10:49:00.208141Z","shell.execute_reply":"2021-06-08T10:49:00.214243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:49:00.21809Z","iopub.execute_input":"2021-06-08T10:49:00.218616Z","iopub.status.idle":"2021-06-08T10:49:00.23919Z","shell.execute_reply.started":"2021-06-08T10:49:00.218567Z","shell.execute_reply":"2021-06-08T10:49:00.237667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T10:49:00.240749Z","iopub.execute_input":"2021-06-08T10:49:00.241108Z","iopub.status.idle":"2021-06-08T10:49:00.256152Z","shell.execute_reply.started":"2021-06-08T10:49:00.241076Z","shell.execute_reply":"2021-06-08T10:49:00.255078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}