{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook gives a simple combination of literal matching and Named Entity Recognition using BERT (base model from huggingface).\n\nThe training phase of the BERT model was done in another kernel: Pytorch BERT for Named Entity Recognition.","metadata":{}},{"cell_type":"code","source":"MAX_SAMPLE = None # set a small number for experimentation, set None for production.","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:53:14.663372Z","iopub.execute_input":"2021-05-31T14:53:14.663854Z","iopub.status.idle":"2021-05-31T14:53:14.668301Z","shell.execute_reply.started":"2021-05-31T14:53:14.663824Z","shell.execute_reply":"2021-05-31T14:53:14.667095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install packages","metadata":{}},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:53:17.3492Z","iopub.execute_input":"2021-05-31T14:53:17.351291Z","iopub.status.idle":"2021-05-31T14:53:50.318755Z","shell.execute_reply.started":"2021-05-31T14:53:17.351251Z","shell.execute_reply":"2021-05-31T14:53:50.317566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-31T14:53:50.320676Z","iopub.execute_input":"2021-05-31T14:53:50.321098Z","iopub.status.idle":"2021-05-31T14:53:51.188402Z","shell.execute_reply.started":"2021-05-31T14:53:50.321055Z","shell.execute_reply":"2021-05-31T14:53:51.187614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:53:51.190606Z","iopub.execute_input":"2021-05-31T14:53:51.191283Z","iopub.status.idle":"2021-05-31T14:54:50.662529Z","shell.execute_reply.started":"2021-05-31T14:53:51.191236Z","shell.execute_reply":"2021-05-31T14:54:50.661407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\n\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\nfor paper_id in sample_submission['Id']:\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:54:50.664389Z","iopub.execute_input":"2021-05-31T14:54:50.664723Z","iopub.status.idle":"2021-05-31T14:54:50.701817Z","shell.execute_reply.started":"2021-05-31T14:54:50.664689Z","shell.execute_reply":"2021-05-31T14:54:50.7009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data exploration","metadata":{}},{"cell_type":"code","source":"#Notes\nfrom collections import Counter\nimport matplotlib.pyplot as plt\n\nprint(f'The training dataset contains {len(train.Id)} rows with {len(train.Id.unique())}.')\nprint(f'So the number of dataset per publication is {round(len(train.Id)/len(train.Id.unique()),2)}.')\n\ncounts = Counter(train.Id)\ncounts = list(counts.values())\ndict1 = Counter(counts)\n\nsorted_dict = {}\nsorted_keys = sorted(dict1)\n\nfor w in sorted_keys:\n    sorted_dict[w] = dict1[w]\n\nprint(sorted_dict)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:54:50.703142Z","iopub.execute_input":"2021-05-31T14:54:50.703453Z","iopub.status.idle":"2021-05-31T14:54:50.723976Z","shell.execute_reply.started":"2021-05-31T14:54:50.703424Z","shell.execute_reply":"2021-05-31T14:54:50.722907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(counts, bins=list(range(23)), log=True)\nplt.title(f\"Distribution of number of datasets per publications\")\nplt.xlabel(\"Datasets\")\nplt.ylabel(\"Counts\")","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:54:50.725812Z","iopub.execute_input":"2021-05-31T14:54:50.726139Z","iopub.status.idle":"2021-05-31T14:54:51.661338Z","shell.execute_reply.started":"2021-05-31T14:54:50.726109Z","shell.execute_reply":"2021-05-31T14:54:51.660318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard(str1, str2):\n    a = set(str1.lower().split())\n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:54:51.662826Z","iopub.execute_input":"2021-05-31T14:54:51.663437Z","iopub.status.idle":"2021-05-31T14:54:51.669588Z","shell.execute_reply.started":"2021-05-31T14:54:51.663391Z","shell.execute_reply":"2021-05-31T14:54:51.668846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_similar_predictions(prediction_string, treshhold):\n    words = list(prediction_string.split(\"|\"))\n    trash = []\n    for i in range(len(words)):\n        word = words[i]\n        for j in range(i+1,len(words)):\n            if jaccard(word,words[j]) > treshhold:\n                trash.append(word)\n                break\n\n    for prediction in trash:\n        words.remove(prediction)\n\n    return '|'.join(words)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:54:51.671866Z","iopub.execute_input":"2021-05-31T14:54:51.672529Z","iopub.status.idle":"2021-05-31T14:54:51.683632Z","shell.execute_reply.started":"2021-05-31T14:54:51.672484Z","shell.execute_reply":"2021-05-31T14:54:51.682796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal matching","metadata":{}},{"cell_type":"markdown","source":"### Create a knowledge bank","metadata":{}},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:52:59.117881Z","iopub.status.idle":"2021-05-31T14:52:59.118463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:52:59.131573Z","iopub.status.idle":"2021-05-31T14:52:59.133672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"literal_preds = []\n\nfor paper_id in sample_submission['Id']:\n    paper = papers[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if label in text_1 or label in text_2:\n            labels.add(clean_text(label))\n    \n    literal_preds.append('|'.join(labels))\n","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:52:59.142541Z","iopub.execute_input":"2021-05-31T14:52:59.143245Z","iopub.status.idle":"2021-05-31T14:52:59.162981Z","shell.execute_reply.started":"2021-05-31T14:52:59.143197Z","shell.execute_reply":"2021-05-31T14:52:59.16167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"NaN = sample_submission.PredictionString[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = []\nfor literal_match in literal_preds:\n    if literal_match:\n        final_predictions.append(remove_similar_predictions(literal_match,0.9))\n    else:\n        final_predictions.append(NaN)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(f'submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}