{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\n\nThis notebook added an splendid `External Dataset` shared by [@mlconsult](https://www.kaggle.com/mlconsult) to `MLM` shared by [@tungmphung](https://www.kaggle.com/tungmphung). Thanks to everyone willing to share their knowledge, let's make the DS community better ;)\n\n**Please check the original notebooks and discussion here:**\n- [Matching + Baseline + MLM w/ Evaluation Metric](https://www.kaggle.com/chienhsianghung/matching-baseline-mlm-w-evaluation-metric)\n- [Coleridge Initiative| Naive CV (Local Validation)](https://www.kaggle.com/chienhsianghung/coleridge-initiative-naive-cv-local-validation)\n- [Evaluation metric implementation](https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/discussion/230091)\n- [[Coleridge] Predict with Masked Dataset Modeling](https://www.kaggle.com/tungmphung/coleridge-predict-with-masked-dataset-modeling)\n- [score 57ish with additional govt datasets](https://www.kaggle.com/mlconsult/score-57ish-with-additional-govt-datasets)\n\n**Previous results from:**\n[Matching + Baseline + MLM w/ Evaluation Metric](https://www.kaggle.com/chienhsianghung/matching-baseline-mlm-w-evaluation-metric)\n\n|   | CV | LB |\n| --- | --- | --- |\n| og | 0.604 | 0.536 |\n| og s1.5 |   | 0.535 |\n| og s2.5 |   | 0.535 |\n| pred_mlm_labels | 0.108 |   |\n| lables_list | 0.548 |   |\n| all | 0.261 |   |\n| og2all | 0.454 |   |\n| 150 400 |   | 0.536 |\n| 200 300 |   | 0.536 |\n| 2021 |   | 0.536 |\n| 42 |   | 0.536 |\n| 42 s1.5 |   | 0.535 |\n| 42 .70 |   | 0.536 |\n| 42 .80 |   | 0.536 |\n| MATCH_ONLY |   | 0.533 |\n| BASELINE_HELPING |   | 0.536 |\n| 1 |   | 0.536 |","metadata":{}},{"cell_type":"markdown","source":"# Setting","metadata":{}},{"cell_type":"markdown","source":"# Install packages","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"COMPUTE_CV = False\nBS_CLEANING = False","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:37:33.461056Z","iopub.execute_input":"2021-06-06T04:37:33.461418Z","iopub.status.idle":"2021-06-06T04:37:33.466276Z","shell.execute_reply.started":"2021-06-06T04:37:33.461338Z","shell.execute_reply":"2021-06-06T04:37:33.465145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eng_stws = []\nwith open('../input/engst-06061111/englishST.txt','r') as fff:\n    for line in fff:\n        line = line.strip().lower()\n        if line != '':\n            eng_stws.append(line)\neng_stws = list(set(eng_stws))","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:37:33.471198Z","iopub.execute_input":"2021-06-06T04:37:33.47207Z","iopub.status.idle":"2021-06-06T04:37:33.485699Z","shell.execute_reply.started":"2021-06-06T04:37:33.472029Z","shell.execute_reply":"2021-06-06T04:37:33.484694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\n\nfrom IPython.display import clear_output\nclear_output()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-06T04:37:33.487452Z","iopub.execute_input":"2021-06-06T04:37:33.48804Z","iopub.status.idle":"2021-06-06T04:39:00.846959Z","shell.execute_reply.started":"2021-06-06T04:37:33.488003Z","shell.execute_reply":"2021-06-06T04:39:00.846063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/abb-whl/abbreviations-0.2.5-py3-none-any.whl\n","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:39:00.849948Z","iopub.execute_input":"2021-06-06T04:39:00.850214Z","iopub.status.idle":"2021-06-06T04:39:26.580648Z","shell.execute_reply.started":"2021-06-06T04:39:00.850187Z","shell.execute_reply":"2021-06-06T04:39:26.579639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install ../input/simple-transformers-package/simpletransformers/ipykernel-5.5.5-py3-none-any.whl\n# !pip install ../input/simple-transformers-package/simpletransformers/pydeck-0.6.2-py2.py3-none-any.whl\n# !pip install ../input/simple-transformers-package/simpletransformers/streamlit-0.82.0-py2.py3-none-any.whl\n# !pip install ../input/simple-transformers-package/simpletransformers/simpletransformers-0.61.5-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:39:26.584198Z","iopub.execute_input":"2021-06-06T04:39:26.584475Z","iopub.status.idle":"2021-06-06T04:39:26.587875Z","shell.execute_reply.started":"2021-06-06T04:39:26.584447Z","shell.execute_reply":"2021-06-06T04:39:26.586984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-dependencies ../input/simple-transformers-package/simpletransformers/simpletransformers-0.61.5-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:39:26.591054Z","iopub.execute_input":"2021-06-06T04:39:26.591281Z","iopub.status.idle":"2021-06-06T04:39:48.74605Z","shell.execute_reply.started":"2021-06-06T04:39:26.591252Z","shell.execute_reply":"2021-06-06T04:39:48.745202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nwith open('../input/ner-dataset-for-show-us-the-data/ner_distant.json', 'r') as f:\n    js = json.load(f)\nprint(js.keys())","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:39:48.750467Z","iopub.execute_input":"2021-06-06T04:39:48.750743Z","iopub.status.idle":"2021-06-06T04:39:49.69012Z","shell.execute_reply.started":"2021-06-06T04:39:48.750716Z","shell.execute_reply":"2021-06-06T04:39:49.689197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"js['sentences_not_including_dataset_all'][:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:39:49.691459Z","iopub.execute_input":"2021-06-06T04:39:49.691792Z","iopub.status.idle":"2021-06-06T04:39:49.699379Z","shell.execute_reply.started":"2021-06-06T04:39:49.691756Z","shell.execute_reply":"2021-06-06T04:39:49.698413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"js['sentences_including_annotations_all'][:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:39:49.700548Z","iopub.execute_input":"2021-06-06T04:39:49.700978Z","iopub.status.idle":"2021-06-06T04:39:49.71079Z","shell.execute_reply.started":"2021-06-06T04:39:49.700942Z","shell.execute_reply":"2021-06-06T04:39:49.709849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"from simpletransformers.ner import NERModel\nimport pandas as pd\nimport logging\nimport torch\n\n# ログの設定\nlogging.basicConfig(level=logging.INFO)\ntransformers_logger = logging.getLogger(\"transformers\")\ntransformers_logger.setLevel(logging.WARNING)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:39:49.712159Z","iopub.execute_input":"2021-06-06T04:39:49.712679Z","iopub.status.idle":"2021-06-06T04:39:58.257838Z","shell.execute_reply.started":"2021-06-06T04:39:49.712643Z","shell.execute_reply":"2021-06-06T04:39:58.256916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ner_model = NERModel(\n    \"roberta\", \"../input/ner-0605-st/content/outputs/best_model\", use_cuda=True if torch.cuda.is_available() else False\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:39:58.259248Z","iopub.execute_input":"2021-06-06T04:39:58.259565Z","iopub.status.idle":"2021-06-06T04:40:08.802609Z","shell.execute_reply.started":"2021-06-06T04:39:58.259529Z","shell.execute_reply":"2021-06-06T04:40:08.801798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions, raw_outputs = ner_model.predict([\"Hermione was the best dataset study national in her class\",\n                                         \"Final dataset study and its survey apperaed\", \"slosh model is great\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:08.803924Z","iopub.execute_input":"2021-06-06T04:40:08.804248Z","iopub.status.idle":"2021-06-06T04:40:15.024353Z","shell.execute_reply.started":"2021-06-06T04:40:08.804214Z","shell.execute_reply":"2021-06-06T04:40:15.023481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.025589Z","iopub.execute_input":"2021-06-06T04:40:15.026097Z","iopub.status.idle":"2021-06-06T04:40:15.033384Z","shell.execute_reply.started":"2021-06-06T04:40:15.026058Z","shell.execute_reply":"2021-06-06T04:40:15.032578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_params = {\n#     'max_seq_length': 512,\n#     'max_length': 128,\n#     'train_batch_size': 32,\n#     'eval_batch_size': 32,\n#     'num_train_epochs': 2,\n#     'evaluate_during_training': True,\n#     'evaluate_during_training_steps': 10000,\n#     'use_multiprocessing': False,\n#     'fp16': False,\n#     'save_steps': -1,\n#     'save_eval_checkpoints': False,\n#     'save_model_every_epoch': False,\n#     'no_cache': True,\n#     'reprocess_input_data': True,\n#     'overwrite_output_dir': True,\n#     'preprocess_inputs': False,\n#     'num_return_sequences': 1 ,\n#     \"output_dir\": \"./ner_outputs/\",\n#     \"cache_dir\": \"./ner_cache_dir/\",\n# }\n\n# model = NERModel(\n#     \"roberta\", \"roberta-base\", use_cuda=True if torch.cuda.is_available() else False,\n#     labels=['O','B-DATASET','I-DATASET'], args=train_params\n# )","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.035018Z","iopub.execute_input":"2021-06-06T04:40:15.035373Z","iopub.status.idle":"2021-06-06T04:40:15.04758Z","shell.execute_reply.started":"2021-06-06T04:40:15.035337Z","shell.execute_reply":"2021-06-06T04:40:15.046739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import math, copy\n# train_ner_num = math.floor(len(train_data) * 0.9)\n# train_data_for_ner, test_data_for_ner = train_data[:train_ner_num], train_data[train_ner_num:]\n\n# train_for_smpformer, test_data_for_smpformer = [], []\n\n# for d in train_data_for_ner:\n#     train_for_smpformer += d\n\n# test_data_first_idx = copy.copy(test_data_for_ner[0][0][0])\n\n# for d in test_data_for_ner:\n#     for one_anno in d:\n#         test_data_for_smpformer.append([one_anno[0] - test_data_first_idx,\n#                                        one_anno[1], one_anno[2]])","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.048971Z","iopub.execute_input":"2021-06-06T04:40:15.049569Z","iopub.status.idle":"2021-06-06T04:40:15.057137Z","shell.execute_reply.started":"2021-06-06T04:40:15.049528Z","shell.execute_reply":"2021-06-06T04:40:15.056359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import math\n# train_for_smpformer = pd.DataFrame(train_for_smpformer, columns=[\"sentence_id\", \"words\", \"labels\"])\n# test_data_for_smpformer = pd.DataFrame(test_data_for_smpformer, columns=[\"sentence_id\", \"words\", \"labels\"])\n\n# model.train_model(train_for_smpformer, eval_data=test_data_for_smpformer)\n\n# # Evaluate the model\n# print('eval ner')\n# result, model_outputs, preds_list = model.eval_model(test_data_for_smpformer)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.060125Z","iopub.execute_input":"2021-06-06T04:40:15.060386Z","iopub.status.idle":"2021-06-06T04:40:15.067915Z","shell.execute_reply.started":"2021-06-06T04:40:15.06035Z","shell.execute_reply":"2021-06-06T04:40:15.067065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions, raw_outputs = model.predict([\"Hermione was the best dataset study national in her class\",\n#                                          \"study\"])\n","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.06909Z","iopub.execute_input":"2021-06-06T04:40:15.069415Z","iopub.status.idle":"2021-06-06T04:40:15.083259Z","shell.execute_reply.started":"2021-06-06T04:40:15.069382Z","shell.execute_reply":"2021-06-06T04:40:15.082386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# from zipfile import ZipFile\n\n# dirName = \"./\"\n# zipName = \"packages.zip\"\n\n# # Create a ZipFile Object\n# with ZipFile(zipName, 'w') as zipObj:\n#     # Iterate over all the files in directory\n#     for folderName, subfolders, filenames in os.walk(dirName):\n#         for filename in filenames:\n#             if (filename != zipName):\n#                 # create complete filepath of file in directory\n#                 filePath = os.path.join(folderName, filename)\n#                 # Add file to zip\n#                 zipObj.write(filePath)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.084832Z","iopub.execute_input":"2021-06-06T04:40:15.085216Z","iopub.status.idle":"2021-06-06T04:40:15.091832Z","shell.execute_reply.started":"2021-06-06T04:40:15.085179Z","shell.execute_reply":"2021-06-06T04:40:15.091107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.autonotebook import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline\n\nfrom typing import List\nimport string\nfrom functools import partial\nSEED=43\nsns.set()\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.09431Z","iopub.execute_input":"2021-06-06T04:40:15.094585Z","iopub.status.idle":"2021-06-06T04:40:15.179636Z","shell.execute_reply.started":"2021-06-06T04:40:15.09456Z","shell.execute_reply":"2021-06-06T04:40:15.178974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"markdown","source":"# Literal Matching","metadata":{}},{"cell_type":"markdown","source":"### Create a Knowledge Bank","metadata":{}},{"cell_type":"code","source":"all_labels = set()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.181599Z","iopub.execute_input":"2021-06-06T04:40:15.182114Z","iopub.status.idle":"2021-06-06T04:40:15.185965Z","shell.execute_reply.started":"2021-06-06T04:40:15.182078Z","shell.execute_reply":"2021-06-06T04:40:15.184879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_data_words = ['study', 'adni', 'disease', 'longitudinal', 'neuroimaging', 'initiative', \n                     'alzheimer', 'international', 'science', 'mathematics', 'trends', 'aging', \n                     'baltimore', 'early', 'childhood', 'survey', 'national', 'continuum', \n                     'codes', 'rural', 'urban', ' adni ', 'data', 'common', 'model', 'trends',\n                     'assesments','assesment','datasets','trend', 'research', 'code',\n                     'databases',\n                     'ibtracs', 'noaa', 'postsecondary', 'beginning', 'management', 'agricultural', \n                     'assessment', 'resource', 'students', 'optimum', 'interpolation', 'surface',\n                     'temperature', 'database', 'program', 'census', 'enumeration', 'poll', 'polls', 'demographics',\n                     'doctorates', 'progress', 'school', 'tide', 'coronavirus', 'analysis', 'dataset', 'analyze',\n                     'cohort', 'cohorts']\ncritical_words = ['survey', 'study', 'data', 'database', 'dataset', 'databases', 'datasets',\n                 'cohort', 'cohorts', 'program', 'programs', 'trand', 'trands', 'assessment', 'assessments',\n                 'analysis', 'analytics', 'census', 'research', 'census', 'model']","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.187405Z","iopub.execute_input":"2021-06-06T04:40:15.187785Z","iopub.status.idle":"2021-06-06T04:40:15.197243Z","shell.execute_reply.started":"2021-06-06T04:40:15.187749Z","shell.execute_reply":"2021-06-06T04:40:15.196328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/filtered-public-data-csv/filered.csv', 'r') as fpd:\n    for line in fpd:\n        line = line.strip()\n        if line != '' and len(line.split(' ')) > 2:\n            split_w = line.split(' ')\n            if split_w[0] in eng_stws:\n                continue\n            if split_w[-1] in eng_stws:\n                continue\n            if len(set(split_w) & set(common_data_words)) > 0: \n                all_labels.add(str(line).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.199916Z","iopub.execute_input":"2021-06-06T04:40:15.20016Z","iopub.status.idle":"2021-06-06T04:40:15.264498Z","shell.execute_reply.started":"2021-06-06T04:40:15.200137Z","shell.execute_reply":"2021-06-06T04:40:15.263872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Additional Govt Datasets","metadata":{}},{"cell_type":"code","source":"adnl_govt_labels = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\n\nfor line in adnl_govt_labels.title:\n    line = line.strip().lower()\n    if line != '' and len(line.split(' ')) > 0:\n        split_w = line.split(' ')\n        if split_w[0] in eng_stws:\n            continue\n        if split_w[-1] in eng_stws:\n            continue\n        all_labels.add(str(line).lower())    \n    \nall_labels = set(all_labels)\nprint(f'No. different labels: {len(all_labels)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.267526Z","iopub.execute_input":"2021-06-06T04:40:15.267789Z","iopub.status.idle":"2021-06-06T04:40:15.32633Z","shell.execute_reply.started":"2021-06-06T04:40:15.267766Z","shell.execute_reply":"2021-06-06T04:40:15.324784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\nif not BS_CLEANING:\n    def text_cleaning(text):\n        '''\n        Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n        text - Sentence that needs to be cleaned\n        '''\n        text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n        text = re.sub(' +', ' ', text)\n        emoji_pattern = re.compile(\"[\"\n                                   u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                                   u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                                   u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                                   u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                                   \"]+\", flags=re.UNICODE)\n        text = emoji_pattern.sub(r'', text)\n        return text\nelse:\n    def text_cleaning(text):\n        '''\n        Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n        text - Sentence that needs to be cleaned\n        '''\n        text = ''.join([k for k in text if k not in string.punctuation])\n        text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n        # text = re.sub(\"/'+/g\", ' ', text)\n        return text\n\n\ndef read_json_pub(filename, train_data_path, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.327555Z","iopub.execute_input":"2021-06-06T04:40:15.327923Z","iopub.status.idle":"2021-06-06T04:40:15.339992Z","shell.execute_reply.started":"2021-06-06T04:40:15.32789Z","shell.execute_reply":"2021-06-06T04:40:15.338978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Ken Matching","metadata":{}},{"cell_type":"markdown","source":"# Masked Dataset Modeling","metadata":{}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"markdown","source":"# Transform data to MLM format","metadata":{}},{"cell_type":"markdown","source":"### Load model and tokenizer","metadata":{}},{"cell_type":"markdown","source":"### Auxiliary functions","metadata":{}},{"cell_type":"code","source":"import pdb\ndef jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    if union == 0:\n        return 0\n    return float(intersection) / union\n\ndef clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s\n\ndef shorten_sentences(sentences):\n    \"\"\"\n    Sentences that have more than MAX_LENGTH words will be split\n    into multiple sentences with overlappings.\n    \"\"\"\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences\n\nconnection_tokens = {'s', 'of', 'and', 'in', 'on', 'for', 'data', 'dataset', 'survey', 'statistics'}\ndef find_mask_candidates(sentence):\n    \"\"\"\n    Extract masking candidates for Masked Dataset Modeling from a given $sentence.\n    A candidate should be a continuous sequence of at least 2 words, \n    each of these words either has the first letter in uppercase or is one of\n    the connection words ($connection_tokens). Furthermore, the connection \n    tokens are not allowed to appear at the beginning and the end of the\n    sequence.\n    \"\"\"\n    def candidate_qualified(words):\n        while len(words) and words[0].lower() in connection_tokens:\n            words = words[1:]\n        while len(words) and words[-1].lower() in connection_tokens:\n            words = words[:-1]\n        \n        return len(words) >= 2\n    \n    candidates = []\n    \n    phrase_start, phrase_end = -1, -1\n    for id in range(1, len(sentence)):\n        word = sentence[id]\n        if word[0].isupper() or word in connection_tokens:\n            if phrase_start == -1:\n                phrase_start = phrase_end = id\n            else:\n                phrase_end = id\n        else:\n            if phrase_start != -1:\n                if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n                    candidates.append((phrase_start, phrase_end))\n                phrase_start = phrase_end = -1\n    \n    if phrase_start != -1:\n        if candidate_qualified(sentence[phrase_start:phrase_end+1]):\n            candidates.append((phrase_start, phrase_end))\n    return candidates","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.34108Z","iopub.execute_input":"2021-06-06T04:40:15.341414Z","iopub.status.idle":"2021-06-06T04:40:15.357405Z","shell.execute_reply.started":"2021-06-06T04:40:15.341379Z","shell.execute_reply":"2021-06-06T04:40:15.356284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.358962Z","iopub.execute_input":"2021-06-06T04:40:15.359351Z","iopub.status.idle":"2021-06-06T04:40:15.371334Z","shell.execute_reply.started":"2021-06-06T04:40:15.359318Z","shell.execute_reply":"2021-06-06T04:40:15.370573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print([l for l in all_labels if len(l.strip()) < 6])","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.372527Z","iopub.execute_input":"2021-06-06T04:40:15.372938Z","iopub.status.idle":"2021-06-06T04:40:15.381571Z","shell.execute_reply.started":"2021-06-06T04:40:15.372888Z","shell.execute_reply":"2021-06-06T04:40:15.380531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform","metadata":{}},{"cell_type":"markdown","source":"### Predict","metadata":{}},{"cell_type":"markdown","source":"# Baseline Model","metadata":{}},{"cell_type":"code","source":"def read_append_return(filename, train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data\n    \n    \ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    # text = re.sub(\"/'+/g\", ' ', text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.383083Z","iopub.execute_input":"2021-06-06T04:40:15.383474Z","iopub.status.idle":"2021-06-06T04:40:15.393358Z","shell.execute_reply.started":"2021-06-06T04:40:15.383441Z","shell.execute_reply":"2021-06-06T04:40:15.392511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tempfile\nfrom typing import Dict, Iterable, List, Tuple\n\nimport torch\n\nfrom allennlp.common.util import JsonDict\nfrom allennlp.data import (\n    DataLoader,\n    DatasetReader,\n    Instance,\n    Vocabulary,\n    TextFieldTensors,\n)\nfrom allennlp.data.data_loaders import SimpleDataLoader, MultiProcessDataLoader\nfrom allennlp.data.fields import LabelField, TextField\nfrom allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer\nfrom allennlp.data.tokenizers import Token, Tokenizer, WhitespaceTokenizer\nfrom allennlp.models import Model\nfrom allennlp.modules import TextFieldEmbedder, Seq2VecEncoder\nfrom allennlp.modules.text_field_embedders import BasicTextFieldEmbedder\nfrom allennlp.modules.token_embedders import Embedding\nfrom allennlp.modules.seq2vec_encoders import BagOfEmbeddingsEncoder, LstmSeq2VecEncoder\nfrom allennlp.nn import util\nfrom allennlp.predictors import Predictor\nfrom allennlp.training.metrics import CategoricalAccuracy\nfrom allennlp.training.optimizers import AdamOptimizer\nfrom allennlp.training.trainer import Trainer, GradientDescentTrainer\nfrom allennlp.training.util import evaluate\nimport math\nimport torch\nfrom allennlp.modules.token_embedders.embedding import _read_embeddings_from_text_file\n\n\nclass ClassificationTsvReader(DatasetReader):\n    def __init__(\n        self,\n        sentence_including_annotations: List[str],\n        sentence_not_including_dataset: List[str],\n        tokenizer: Tokenizer = None,\n        token_indexers: Dict[str, TokenIndexer] = None,\n        max_tokens: int = None,\n        **kwargs\n    ):\n        super().__init__(**kwargs)\n        self.tokenizer = tokenizer or WhitespaceTokenizer()\n        self.token_indexers = token_indexers or {\"tokens\": SingleIdTokenIndexer()}\n        self.max_tokens = max_tokens\n        \n        self.train_sent_including_dataset, self.dev_sent_including_dataset, self.train_sent_not_including_dataset, self.dev_sent_not_including_dataset = \\\n         self._data_splitter_with_train_and_dev(sentence_including_annotations, sentence_not_including_dataset)\n        \n        print('length_of_data')\n        print('train including dataset sents', len(self.train_sent_including_dataset))\n        print('dev including dataset sents', len(self.dev_sent_including_dataset))\n        print('train not including dataset sents', len(self.train_sent_not_including_dataset))\n        print('dev not including dataset sents', len(self.dev_sent_not_including_dataset))\n        \n    def _data_splitter_with_train_and_dev(self, sentence_including_annotations, sentence_not_including_dataset):\n        train_frac, dev_frac = 0.85, 0.15\n        sentences_including_annotations_num = len(sentence_including_annotations)\n        train_sentences_including_annotations = sentence_including_annotations[:math.floor(sentences_including_annotations_num * train_frac)]\n        dev_sentences_including_annotations = sentence_including_annotations[math.floor(sentences_including_annotations_num * train_frac):]\n    \n        sentences_not_including_annotations_num = len(sentence_not_including_dataset)\n        train_sentences_not_including_annotations = sentence_not_including_dataset[:math.floor(sentences_not_including_annotations_num * train_frac)]\n        dev_sentences_not_including_annotations = sentence_not_including_dataset[math.floor(sentences_not_including_annotations_num * train_frac):]\n        \n        return train_sentences_including_annotations, \\\n               dev_sentences_including_annotations, \\\n               train_sentences_not_including_annotations, \\\n               dev_sentences_not_including_annotations\n    \n    def text_to_instance(self, text: str, label: str = None) -> Instance:\n        tokens = self.tokenizer.tokenize(text)\n        if self.max_tokens:\n            tokens = tokens[: self.max_tokens]\n        text_field = TextField(tokens, self.token_indexers)\n        fields = {'text': text_field}\n        if label:\n            fields['label'] = LabelField(label)\n        return Instance(fields)\n               \n        \n    def _read(self, train_or_dev: str) -> Iterable[Instance]:\n        if train_or_dev == 'train':\n            train_data = list()\n            \n            for sent in self.train_sent_including_dataset:\n                dataset_including_flag = '1'\n                yield self.text_to_instance(sent, dataset_including_flag)\n            for sent in self.train_sent_not_including_dataset:\n                dataset_including_flag = '0'\n                yield self.text_to_instance(sent, dataset_including_flag)\n            \n            return train_data\n            \n        elif train_or_dev == 'dev':\n            dev_data = list()\n            for sent in self.dev_sent_including_dataset:\n                dataset_including_flag = '1' \n                yield self.text_to_instance(sent, dataset_including_flag)\n            for sent in self.dev_sent_not_including_dataset:\n                dataset_including_flag = '0'\n                yield self.text_to_instance(sent, dataset_including_flag)\n            \n            return dev_data\n        else:\n            raise NotImplementedError\n\nclass SimpleClassifier(Model):\n    def __init__(\n        self, vocab: Vocabulary, embedder: TextFieldEmbedder, encoder: Seq2VecEncoder\n    ):\n        super().__init__(vocab)\n        self.embedder = embedder\n        self.encoder = encoder\n        num_labels = vocab.get_vocab_size(\"labels\")\n        self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_labels)\n        self.accuracy = CategoricalAccuracy()\n\n    def forward(\n        self, text: TextFieldTensors, label: torch.Tensor = None) -> Dict[str, torch.Tensor]:\n\n        # Shape: (batch_size, num_tokens, embedding_dim)\n        embedded_text = self.embedder(text)\n        # Shape: (batch_size, num_tokens)\n        mask = util.get_text_field_mask(text)\n        # Shape: (batch_size, encoding_dim)\n        encoded_text = self.encoder(embedded_text, mask)\n        # Shape: (batch_size, num_labels)\n        logits = self.classifier(encoded_text)\n        # Shape: (batch_size, num_labels)\n        probs = torch.nn.functional.softmax(logits)\n        # Shape: (1,)\n        output = {\"probs\": probs}\n        if label is not None:\n            self.accuracy(logits, label)\n            output['loss'] = torch.nn.functional.cross_entropy(logits, label)\n        return output\n\n    def get_metrics(self, reset: bool = False) -> Dict[str, float]:\n        return {\"accuracy\": self.accuracy.get_metric(reset)}\n\n\ndef build_dataset_reader(sentence_including_annotations: List,\n        sentence_not_including_dataset: List,\n                        max_tokens: int) -> DatasetReader:\n    return ClassificationTsvReader(sentence_including_annotations, sentence_not_including_dataset, \n                                   max_tokens=max_tokens)\n\n\ndef read_data(reader: DatasetReader) -> Tuple[List[Instance], List[Instance]]:\n    print(\"Reading data\")\n    training_data = list(reader.read(\"train\"))\n    validation_data = list(reader.read(\"dev\"))\n    return training_data, validation_data\n\n\ndef build_vocab(instances: Iterable[Instance]) -> Vocabulary:\n    print(\"Building the vocabulary\")\n    return Vocabulary.from_instances(instances)\n\n\ndef build_model(vocab: Vocabulary) -> Model:\n    print(\"Building the model\")\n    vocab_size = vocab.get_vocab_size(\"tokens\")\n    # '../input/glove6b/glove.6B.300d.txt'\n    \n    embed_matrix = _read_embeddings_from_text_file(\n        file_uri='../input/glove6b/glove.6B.300d.txt',\n        embedding_dim=300,\n        vocab=vocab\n    )\n    token_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),\n                                embedding_dim=300,\n                                weight=embed_matrix)\n    \n    embedder = BasicTextFieldEmbedder(\n        {\"tokens\": token_embedding}\n    )\n    encoder = LstmSeq2VecEncoder(input_size=300, hidden_size=300, num_layers=1, bidirectional=True)\n    # BagOfEmbeddingsEncoder(embedding_dim=300)\n    return SimpleClassifier(vocab, embedder, encoder)\n\n\ndef build_data_loaders(dataset_reader) -> Tuple[DataLoader, DataLoader]:\n    if torch.cuda.is_available():\n        train_loader =  MultiProcessDataLoader(dataset_reader, data_path='train', batch_size=64, shuffle=True,\n                                              cuda_device=0)\n        dev_loader =  MultiProcessDataLoader(dataset_reader, data_path='dev', batch_size=64, shuffle=False,\n                                            cuda_device=0)\n    else:\n        train_loader =  MultiProcessDataLoader(dataset_reader, data_path='train', batch_size=64, shuffle=True)\n        dev_loader =  MultiProcessDataLoader(dataset_reader, data_path='dev', batch_size=64, shuffle=False)\n    return train_loader, dev_loader\n\n\ndef build_trainer(\n    model: Model,\n    serialization_dir: str,\n    train_loader: DataLoader,\n    dev_loader: DataLoader,\n) -> Trainer:\n    if torch.cuda.is_available():\n        model.cuda()\n    \n    parameters = [(n, p) for n, p in model.named_parameters() if p.requires_grad]\n    optimizer = AdamOptimizer(parameters, lr=0.01)  # type: ignore\n    trainer = GradientDescentTrainer(\n        model=model,\n        serialization_dir=serialization_dir,\n        data_loader=train_loader,\n        validation_data_loader=dev_loader,\n        num_epochs=5,\n        optimizer=optimizer,\n        cuda_device=0 if torch.cuda.is_available() else -1\n    )\n    return trainer\n\n\ndef run_training_loop(sentence_including_annotations: List[str], \n                      sentence_not_including_dataset: List[str],\n                     max_tokens: int):\n    dataset_reader = build_dataset_reader(sentence_including_annotations, sentence_not_including_dataset,\n                                         max_tokens)\n    train, dev = dataset_reader.read('train'), dataset_reader.read('dev')\n    vocab = build_vocab(train)\n    vocab.extend_from_instances(dev)\n    model = build_model(vocab)\n\n    train_loader, dev_loader = build_data_loaders(dataset_reader)\n    train_loader.index_with(vocab)\n    dev_loader.index_with(vocab)\n\n    # You obviously won't want to create a temporary file for your training\n    # results, but for execution in binder for this guide, we need to do this.\n    with tempfile.TemporaryDirectory() as serialization_dir:\n        trainer = build_trainer(model, serialization_dir, train_loader, dev_loader)\n        trainer.train()\n\n    return model, dataset_reader","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:15.394773Z","iopub.execute_input":"2021-06-06T04:40:15.39533Z","iopub.status.idle":"2021-06-06T04:40:18.332732Z","shell.execute_reply.started":"2021-06-06T04:40:15.395294Z","shell.execute_reply":"2021-06-06T04:40:18.331959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences_including_annotations_all = js['sentences_including_annotations_all']\nsentences_not_including_dataset_all = js['sentences_not_including_dataset_all']","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:18.33414Z","iopub.execute_input":"2021-06-06T04:40:18.334472Z","iopub.status.idle":"2021-06-06T04:40:18.33939Z","shell.execute_reply.started":"2021-06-06T04:40:18.334438Z","shell.execute_reply":"2021-06-06T04:40:18.33788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(sentences_including_annotations_all))\nprint(len(sentences_not_including_dataset_all))\nmodel, dataset_reader = run_training_loop(sentences_including_annotations_all, sentences_not_including_dataset_all,\n                                         max_tokens=250)\nvocab = model.vocab","metadata":{"execution":{"iopub.status.busy":"2021-06-06T04:40:18.34109Z","iopub.execute_input":"2021-06-06T04:40:18.341596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from allennlp.predictors import Predictor\nfrom allennlp.training.util import evaluate\n\n\nclass SentenceClassifierPredictor(Predictor):\n    def predict(self, sentence: str) -> JsonDict:\n        return self.predict_json({\"sentence\": sentence})\n    \n    def predict_batch(self, sentences: List[str]):\n        return self.predict_batch_json([{\"sentence\": sentence} for sentence in sentences])\n\n    def _json_to_instance(self, json_dict: JsonDict) -> Instance:\n        sentence = json_dict[\"sentence\"]\n        return self._dataset_reader.text_to_instance(sentence)\n\npredictor = SentenceClassifierPredictor(model, dataset_reader)\n\noutput = predictor.predict_batch([\"A good movie!\", \"Slosh model is needed\"])\nprint(output)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk import sent_tokenize","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"markdown","source":"# Evaluation Metric","metadata":{}},{"cell_type":"code","source":"# Asthetics\nimport warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\n# Basic\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nimport json\nimport os\nimport random\nfrom tqdm.autonotebook import tqdm\nimport string\nimport re\nfrom functools import partial\n\n# Visualizations\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\nfrom wordcloud import WordCloud, STOPWORDS\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RANDOM_SEED = 30\ndef seed_everything(seed=RANDOM_SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything()\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data\ntqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain_df['text'] = train_df['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\n'''\nIdea below of also using the 'dataset_title' is burrowed from\nhttps://www.kaggle.com/josephassaker/coleridge-initiative-eda-na-ve-submission\n'''\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3 ) | all_labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(existing_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_word_label = []\nmany_word_label = []\n\nfor extist_label in existing_labels:\n    sp_ext = extist_label.strip().split(' ')\n    if len(sp_ext) == 1:\n        one_word_label.append(' '.join(sp_ext))\n    else:\n        many_word_label.append(' '.join(sp_ext))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(one_word_label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"many_word_label[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_list = []\nlables_list = []\nfrom abbreviations import schwartz_hearst\n\ndef batcher(iterable, n=1):\n    l = len(iterable)\n    for ndx in range(0, l, n):\n        yield iterable[ndx:min(ndx + n, l)]\n\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n\n    abb_pairs = {}\n    '''\n    {'QC': 'quality control', 'MDS': 'multidimensional scaling', 'MLMA': 'mixed-linear-model association',\n    'ii': 'included in COGENT; and', 'CHARGE': 'Cohorts for Heart and Aging Research in Genomic Epidemiology', \n    'FHS': 'Framingham Heart Study, Second Generation Cohort', 'CHS': 'Cardiovascular Health Study',\n    'NCNG': 'Norwegian Cognitive NeuroGenetics', 'lincRNA': 'long intergenic non-coding RNA', \n    'PING': 'Pediatric Imaging, Neurocognition, and Genetics', 'ADNI': \"Alzheimer's Disease Neuroimaging Initiative\", \n    'KDP': 'Kernel density plot'}\n    '''\n    row_id = row['Id']\n    temp_df = train_df[train_df['text'] == text_cleaning(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x).replace(',', ' ') for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n\n    try:\n        labels_from_ner = list()\n        sents = sent_tokenize(sample_text)\n        sents_num = len(sents)\n        sents = sents[: sents_num // 2]\n        for batched_sents in batcher(sents, 64):\n            whether_sent_includes_dataset = predictor.predict_batch(batched_sents)\n\n            filtered_texts = list()\n\n            for sent, if_dataset_include_result in zip(batched_sents, whether_sent_includes_dataset):\n                result =     [\n                (vocab.get_token_from_index(label_id, \"labels\"), prob)\n                for label_id, prob in enumerate(if_dataset_include_result[\"probs\"])]\n\n                if result[1][1] > 0.98:\n                    filtered_texts.append(sent)\n                \n                if 'et al' in sent:\n                    filtered_texts.append(sent)\n                \n                try:\n                    if len(set(sent.lower().split(' ')) & set(common_data_words)) > 0 or '(' in sent:\n                        pairs = schwartz_hearst.extract_abbreviation_definition_pairs(doc_text=sent) \n                        if len(pairs) > 0:\n                            filtered_texts.append(sent)\n                            for k, v in pairs.items():\n                                abb_pairs.update({k.lower(): v.lower()})\n                except:\n                    pass\n            \n            filtered_texts = list(set(filtered_texts))\n            if len(filtered_texts) > 0:\n                res = ner_model.predict(filtered_texts)\n                res = res[0]\n\n                tmp_ds_label = []\n                for sent_result in res:\n                    for one_word_pred in sent_result:\n                        for word, pred_label in one_word_pred.items():\n                            if pred_label in ['B-DATASET', 'I-DATASET']:\n                                tmp_ds_label.append(word)\n                            if pred_label == 'O':\n                                if len(tmp_ds_label) > 1: # critical params\n                                    if tmp_ds_label[0] not in eng_stws and tmp_ds_label[-1] not in eng_stws:\n                                        labels_from_ner.append(' '.join(tmp_ds_label))\n                                tmp_ds_label.clear()\n\n        labels_from_ner = [clean_text(label) for label in labels_from_ner if len(label.split(' ')) > 2]\n        filtered_labels = []\n\n        for label in sorted(labels_from_ner, key=len, reverse=True):\n            if len(filtered_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered_labels):\n                filtered_labels.append(label)\n\n        filtered_labels = [label.replace(',', ' ') for label in filtered_labels if label.split(' ')[0] not in eng_stws\n                          and label.split(' ')[-1] not in eng_stws]\n        \n\n        '''\n        ['floodplain management and hazard mitigation', 'north carolina division of transportation', \n        'nc sea level rise risk management study', 'erosion storm surge and sea level rise',\n        'usgs digital shoreline analysis system', 'area of change from shoreline movement', \n        'periodic reporting of sea level trends', 'estuarine shoreline change and erosion', \n        'storm surge vulnerability assessment', 'albemarle pamlico estuarine system', \n        'oceanic atmospheric administration', 'sea level affecting marshes model',\n        'digital flood insurance rate maps', 'national flood insurance program',\n        'cape hatteras national seashore', 'coastal elevation data quality',\n        'sea level rise inundation maps', 'storm surge vulnerability maps', \n        'tidal or wind tide inundation', 'nc floodplain mapping program', 'coastal vulnerability index', \n        'coastal observation station', 'surge simulation modeling', 'area of shoreline change', \n        'linear regression rate', 'sea level rise mapping', \n        'coastal erosion study', 'envelope of water', 'noaa ocm slr']\n        '''\n    except:\n        filtered_labels = list()\n    \n    set_list = set(cleaned_labels)\n    \n    \n    # critical_words\n    final_filtered_from_ner = [d for d in filtered_labels if d.lower().split(' ')[-1] in critical_words]\n    print('** filtered ner prediction **', final_filtered_from_ner)\n    print('** cleaned str match **',cleaned_labels)\n    \n    filtered_abb = [v.lower() for v in abb_pairs.values()]\n    filtered_abb = [v for v in filtered_abb if v.split(' ')[-1] not in eng_stws and v.split(' ')[0] not in eng_stws]\n    filtered_abb = [clean_text(v) for v in filtered_abb if len(set(v.split(' ')) & set(critical_words)) > 0]\n    print('** filtered_abb **', filtered_abb)\n    \n    entire_labels = []\n    entire_labels += [clean_text(label) for label in list(set_list)]\n    \n    if len(entire_labels) == 0 :\n        entire_labels += [clean_text(label) for label in list(set(filtered_abb))]\n        entire_labels = list(set(entire_labels))\n        \n        if len(entire_labels) == 0:\n\n            for label in sorted(final_filtered_from_ner, key=len, reverse=True):\n                if len(entire_labels) == 0 or all(jaccard_similarity(label, got_label) < 0.50 for got_label in entire_labels):\n                    entire_labels.append(label)\n        entire_labels = entire_labels[:5]\n    \n    entire_labels.sort()\n    print('**final_labels aggregated**', entire_labels)\n    lables_list.append('|'.join(entire_labels))\n    id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['PredictionString'] = lables_list\nsample_sub[['Id', 'PredictionString']].to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(id_list) == len(lables_list)\nassert len(sample_sub) == len(id_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.DataFrame()\nassert len(sample_sub['Id']) == len(id_list)\nsample_sub[['Id', 'PredictionString']].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"|   | CV | LB |\n| --- | --- | --- |\n| s57 adnl_govt_labels |   | 0.573 |\n| 42 w/ adnl_govt_labels |   | 0.557 |\n| adnl_govt_labels_26897 | 0.136 |   |\n| adnl_govt_labels |   | 0.556 |\n| adnl_govt_labels KEN |   |   |\n| 42 w/ adnl_govt_labels KEN |   | 0.557 |\n| **RKM adnl_govt_labels w/ 42** | **0.514** | **0.574** |\n| RKM adnl_govt_labels BS_CLEANING w/ 42 |   | 0.568 |\n| RKM adnl_govt_labels_26897 |   | 0.244 |\n| RKM adnl_govt_labels w/ 42_48 |   | 0.574 |\n| **RKM adnl_govt_labels w/ 42_36** |   | **0.574** |\n| RKM adnl_govt_labels w/ 42_60 |   | 0.574 |\n| RKM adnl_govt_labels w/ 42_24 |   | 0.573 |\n| RKM adnl_govt_labels w/ 42_60 |   |   |","metadata":{}}]}