{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Custom classifier and NER model with parallel loading\n- Tokenize sentences\n- Filter sentences with RF classifier\n- Add predictions from data labels","metadata":{}},{"cell_type":"markdown","source":"## Libraries\nSet spaCy dependencies","metadata":{}},{"cell_type":"code","source":"%%time\n!pip install ../input/pandarallel151whl/pandarallel-1.5.1-py3-none-any.whl\n!pip uninstall fastai en-core-web-sm en-core-web-lg spacy -y -q\n!pip install ../input/spacy3/catalogue-2.0.3-py3-none-any.whl ../input/spacy3/typer-0.3.2-py3-none-any.whl ../input/spacy3/srsly-2.4.1-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/pathy-0.5.2-py3-none-any.whl ../input/spacy3/smart_open-3.0.0-py3-none-any.whl ../input/spacy3/pydantic-1.7.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/thinc-8.0.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy-3.0.6-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy_legacy-3.0.5-py2.py3-none-any.whl -q\n!pip install ../input/spacy3/en_core_web_lg-3.0.0-py3-none-any.whl ../input/spacy3/en_core_web_md-3.0.0-py3-none-any.whl ../input/spacy3/en_core_web_sm-3.0.0-py3-none-any.whl -q\n!pip install ../input/spacy3/spacy_alignments-0.8.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy_transformers-1.0.2-py2.py3-none-any.whl ../input/spacy3/en_core_web_trf-3.0.0-py3-none-any.whl -q\n\nfrom pandarallel import pandarallel\npandarallel.initialize(progress_bar=True)\n\nimport spacy\nassert spacy.__version__ == '3.0.6'\nfrom spacy import displacy\n\nimport glob\nimport json\nimport re\nimport pickle\nimport pandas as pd\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:12:39.698732Z","iopub.execute_input":"2021-07-08T23:12:39.699489Z","iopub.status.idle":"2021-07-08T23:14:41.387190Z","shell.execute_reply.started":"2021-07-08T23:12:39.699432Z","shell.execute_reply":"2021-07-08T23:14:41.386035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models\nRF and NER components (trained separately, imported as datasets)","metadata":{}},{"cell_type":"markdown","source":"### RF model","metadata":{}},{"cell_type":"code","source":"%%time\npkl_filename = '../input/balanced-rf/RF_balanced_model.pkl'\n\nwith open(pkl_filename, 'rb') as file:\n    best_model = pickle.load(file)\n\nbest_model","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:14:41.389273Z","iopub.execute_input":"2021-07-08T23:14:41.389589Z","iopub.status.idle":"2021-07-08T23:14:41.462643Z","shell.execute_reply.started":"2021-07-08T23:14:41.389555Z","shell.execute_reply":"2021-07-08T23:14:41.461539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### NER model","metadata":{}},{"cell_type":"code","source":"%%time\ncustom_ner_model = spacy.load(\"../input/original-model/spaCy/output/model-best\")","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:14:41.465271Z","iopub.execute_input":"2021-07-08T23:14:41.465961Z","iopub.status.idle":"2021-07-08T23:14:45.770739Z","shell.execute_reply.started":"2021-07-08T23:14:41.465907Z","shell.execute_reply":"2021-07-08T23:14:45.769610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions","metadata":{}},{"cell_type":"code","source":"def jaccard(str1, str2): \n    \"\"\"\n    Defined by the competition\n    \"\"\"\n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))\n\ndef clean_text(txt):\n    \"\"\"\n    Defined by competition\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef scrub_text(txt):\n    \"\"\"\n    Extends text cleaning\n    \"\"\"\n    scrub = re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n    return ''.join([i for i in scrub if not i.isdigit()])\n\ndef upper_text(txt):\n    \"\"\"\n    Removes special characters and punctuation, retains uppercasing\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt))\n\ndef find_acronyms(txt):\n    \"\"\"\n    Returns yes if there is a sequence of capital letters\n    \"\"\"\n    matches = re.findall(r\"\\b[A-Z\\.]{2,}s?\\b\", txt)\n    if matches:\n        return 1\n    else:\n        return 0\n\ndef extract_acronyms(txt):\n    \"\"\"\n    Finds and return a sequence of capital letters\n    \"\"\"\n    ac = []\n    matches = re.findall(r\"\\b[A-Z\\.]{2,}s?\\b\", txt)\n    if matches:\n        for match in matches:\n            ac.append(match)\n        return ac\n    else:\n        return ac\n    \ndef count_acronyms(txt):\n    \"\"\"\n    Counts sequences of capital letters\n    \"\"\"\n    matches = re.findall(r\"\\b[A-Z\\.]{2,}s?\\b\", txt)\n    if matches:\n        return len(matches)\n    else:\n        return 0\n\ndef flatten_list(object):\n    \"\"\"\n    Flattens lists of lists\n    \"\"\"\n    gather = []\n    for item in object:\n        if isinstance(item, (list, tuple, set)):\n            gather.extend(flatten_list(item))            \n        else:\n            gather.append(item)\n    return gather\n\ndef filter_set(main_set, condition):\n    \"\"\"\n    Removes items from set based on condition\n    \"\"\"\n    for elem in list(main_set):\n        if condition(elem):\n            main_set.discard(elem)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:14:45.773252Z","iopub.execute_input":"2021-07-08T23:14:45.773910Z","iopub.status.idle":"2021-07-08T23:14:45.788437Z","shell.execute_reply.started":"2021-07-08T23:14:45.773862Z","shell.execute_reply":"2021-07-08T23:14:45.787672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target\nPredict if a sentence contains a dataset reference","metadata":{}},{"cell_type":"code","source":"%%time\n\nsubmission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\ndf_train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\ntest_files = glob.glob('../input/coleridgeinitiative-show-us-the-data/test/*.json')\n\ndf_test_pubs = pd.DataFrame()\nfor test_file in test_files: \n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'Id', test_file.split('/')[-1].split('.')[0])\n    df_test_pubs = pd.concat([df_test_pubs, file_data])\n\ndf_test_pubs['clean_text'] = df_test_pubs['text'].parallel_apply(clean_text)\ndf_test_pubs['scrub_text'] = df_test_pubs['text'].parallel_apply(scrub_text)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:14:45.789943Z","iopub.execute_input":"2021-07-08T23:14:45.790387Z","iopub.status.idle":"2021-07-08T23:14:46.751243Z","shell.execute_reply.started":"2021-07-08T23:14:45.790344Z","shell.execute_reply":"2021-07-08T23:14:46.750181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sentences = []\nfor row in df_test_pubs.itertuples():\n    sentences = row[3].split(\".\")\n    for sent in sentences:\n        test_sentences.append((row[1], row[2], sent))\n\ndf_test_sent = pd.DataFrame(test_sentences, columns=['Id', 'section_title', 'sent'])\n\ndf_test_sent['sent_clean'] = df_test_sent['sent'].parallel_apply(clean_text)\ndf_test_sent['section_clean']= df_test_sent['section_title'].parallel_apply(clean_text)\n\ndf_test_sent['sent'] = df_test_sent['sent'].astype(str)\ndf_test_sent['section_title'] = df_test_sent['section_title'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:14:46.752665Z","iopub.execute_input":"2021-07-08T23:14:46.752954Z","iopub.status.idle":"2021-07-08T23:14:47.596044Z","shell.execute_reply.started":"2021-07-08T23:14:46.752919Z","shell.execute_reply":"2021-07-08T23:14:47.594908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Labels\nCreate labels and merge into a set to check if a sentence contains a known data label\n* Training data titles\n* Training data acronyms\n* Data.gov data titles\n* ICPSR study names","metadata":{}},{"cell_type":"code","source":"# icpsr_studies = pd.read_csv('../input/icpsr-study-names/icpsr_studies.csv')\n# icpsr_studies_raw = icpsr_studies['NAME'].unique()\n# icpsr_studies_clean = icpsr_studies['NAME'].parallel_apply(clean_text).unique()\n# icpsr_studies_scrub = icpsr_studies['NAME'].parallel_apply(scrub_text).unique()\n\n# icpsr_labels_1 = set(icpsr_studies_raw);\n# icpsr_labels_2 = set(icpsr_studies_clean);\n# icpsr_labels_3 = set(icpsr_studies_scrub);\n\n# icpsr_studies_set = set.union(icpsr_labels_1, icpsr_labels_2, icpsr_labels_3);\n\n# icspr_series = pd.read_csv('../input/icpsr-study-names/icpsr_series.csv').dropna()\n# icspr_series_raw = icspr_series['TITLE'].unique()\n# icspr_series_clean = icspr_series['TITLE'].parallel_apply(clean_text).unique()\n# icspr_series_scrub = icspr_series['TITLE'].parallel_apply(scrub_text).unique()\n\n# icpsr_series_1 = set(icspr_series_raw);\n# icpsr_series_2 = set(icspr_series_clean);\n# icpsr_series_3 = set(icspr_series_scrub);\n\n# icpsr_series_set = set.union(icpsr_series_1, icpsr_series_2, icpsr_series_3);\n\ndatagov_labels = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\ndatagov_labels['title'] = datagov_labels['title'].astype(str)\ndatagov_labels_raw = datagov_labels['title'].unique()\ndatagov_labels_clean = datagov_labels['title'].parallel_apply(clean_text).unique()\ndatagov_labels_scrub = datagov_labels['title'].parallel_apply(scrub_text).unique()\n\ndatagov_labels_1 = set(datagov_labels_raw);\ndatagov_labels_2 = set(datagov_labels_clean);\ndatagov_labels_3 = set(datagov_labels_scrub);\n\ndatagov_labels_set = set.union(datagov_labels_1, datagov_labels_2, datagov_labels_3)\n\ndf_train['dataset_title'] = df_train['dataset_title'].astype(str)\ndf_train['dataset_label'] = df_train['dataset_label'].astype(str)\n\ntrain_title = df_train['dataset_title'].unique()\ntrain_title = set(train_title);\n\ntrain_label = df_train['dataset_label'].unique()\ntrain_label = set(train_label);\n\nacronyms_label = df_train['dataset_title'].parallel_apply(extract_acronyms).dropna()\nac_label = set(flatten_list(acronyms_label))\n\nacronyms_title = df_train['dataset_label'].parallel_apply(extract_acronyms).dropna()\nac_title = set(flatten_list(acronyms_title))\n\nacronym_upper = set.union(ac_label, ac_title);\n\nall_labels = set.union(train_title,\n                       train_label,\n                       acronym_upper,\n#                        icpsr_studies_set, \n#                        icpsr_series_set, \n                       datagov_labels_set);\n\nlen(all_labels)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:14:47.597635Z","iopub.execute_input":"2021-07-08T23:14:47.597956Z","iopub.status.idle":"2021-07-08T23:14:49.393935Z","shell.execute_reply.started":"2021-07-08T23:14:47.597920Z","shell.execute_reply":"2021-07-08T23:14:49.392759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features","metadata":{}},{"cell_type":"markdown","source":"* has indicator terms (binary, count)\n* in section (binary)\n* has acronym (binary, count)\n* has title (binary)","metadata":{}},{"cell_type":"code","source":"%%time\n\ndf_test_sent['freqData'] = df_test_sent['sent_clean'].str.count('data')\ndf_test_sent['freqEdu'] = df_test_sent['sent_clean'].str.count('edu')\ndf_test_sent['freqSample'] = df_test_sent['sent_clean'].str.count('sample')\ndf_test_sent['freqNational'] = df_test_sent['sent_clean'].str.count('national')\ndf_test_sent['freqSurvey'] = df_test_sent['sent_clean'].str.count('survey')\ndf_test_sent['freqPublic'] = df_test_sent['sent_clean'].str.count('public')\ndf_test_sent['freqAvail'] = df_test_sent['sent_clean'].str.count('avail')\ndf_test_sent['freqNSF'] = df_test_sent['sent_clean'].str.count('nsf')\ndf_test_sent['freqGov'] = df_test_sent['sent_clean'].str.count('gov')\ndf_test_sent['freqAccess'] = df_test_sent['sent_clean'].str.count('access')\n\ndf_test_sent['hasData'] = np.where(df_test_sent['sent_clean'].str.contains('data'), 1, 0)\ndf_test_sent['hasEdu'] = np.where(df_test_sent['sent_clean'].str.contains('edu'), 1, 0)\ndf_test_sent['hasSample'] = np.where(df_test_sent['sent_clean'].str.contains('sample'), 1, 0)\ndf_test_sent['hasNational'] = np.where(df_test_sent['sent_clean'].str.contains('national'), 1, 0)\ndf_test_sent['hasSurvey'] = np.where(df_test_sent['sent_clean'].str.contains('survey'), 1, 0)\ndf_test_sent['hasPublic'] = np.where(df_test_sent['sent_clean'].str.contains('public'), 1, 0)\ndf_test_sent['hasAvail'] = np.where(df_test_sent['sent_clean'].str.contains('survey'), 1, 0)\ndf_test_sent['hasNSF'] = np.where(df_test_sent['sent_clean'].str.contains('nsf'), 1, 0)\ndf_test_sent['hasGov'] = np.where(df_test_sent['sent_clean'].str.contains('gov'), 1, 0)\ndf_test_sent['hasAccess'] = np.where(df_test_sent['sent_clean'].str.contains('access'), 1, 0)\n\ndf_test_sent['inIntro'] = np.where(df_test_sent['section_clean'].str.contains('intro'), 1, 0)\ndf_test_sent['inDisc'] = np.where(df_test_sent['section_clean'].str.contains('discus'), 1, 0)\ndf_test_sent['inAbst'] = np.where(df_test_sent['section_clean'].str.contains('abstr'), 1, 0)\ndf_test_sent['inResult'] = np.where(df_test_sent['section_clean'].str.contains('resul'), 1, 0)\ndf_test_sent['inConcl'] = np.where(df_test_sent['section_clean'].str.contains('conclu'), 1, 0)\ndf_test_sent['inMethod'] = np.where(df_test_sent['section_clean'].str.contains('meth'), 1, 0)\ndf_test_sent['inBack'] = np.where(df_test_sent['section_clean'].str.contains('back'), 1, 0)\ndf_test_sent['inData'] = np.where(df_test_sent['section_clean'].str.contains('data'), 1, 0)\ndf_test_sent['inSumm'] = np.where(df_test_sent['section_clean'].str.contains('summ'), 1, 0)\ndf_test_sent['inAckno'] = np.where(df_test_sent['section_clean'].str.contains('acknowl'), 1, 0)\n\ndf_test_sent['hasAcronym'] = df_test_sent['sent'].parallel_apply(find_acronyms)\ndf_test_sent['freqAcronym'] = df_test_sent['sent'].parallel_apply(count_acronyms)\n\nicpsr = pd.read_csv('../input/icpsr-study-names/icpsr_studies.csv')\nicpsr_labels = icpsr['NAME'].apply(clean_text).str.replace('\\d+', '')\ndf_test_sent['hasICPSRTitle'] = df_test_sent['sent_clean'].apply(lambda x: any([k in x for k in icpsr_labels]))\ndf_test_sent['hasICPSRTitle'] = df_test_sent['hasICPSRTitle'].astype('category').cat.codes\n\ndatagov = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\ndatagov_labels = datagov['title'].apply(clean_text).str.replace('\\d+', '')\ndf_test_sent['hasDATAGOVTitle'] = df_test_sent['sent_clean'].apply(lambda x: any([k in x for k in datagov_labels]))\ndf_test_sent['hasDATAGOVTitle'] = df_test_sent['hasDATAGOVTitle'].astype('category').cat.codes\n\ndf_test_sent.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:14:49.395551Z","iopub.execute_input":"2021-07-08T23:14:49.395828Z","iopub.status.idle":"2021-07-08T23:15:01.853297Z","shell.execute_reply.started":"2021-07-08T23:14:49.395797Z","shell.execute_reply":"2021-07-08T23:15:01.852004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_sent = df_test_sent.drop(columns=['section_title', \n                                          'sent_clean', \n                                          'section_clean'])\n\ndf_test_sent.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:15:01.856214Z","iopub.execute_input":"2021-07-08T23:15:01.856578Z","iopub.status.idle":"2021-07-08T23:15:01.883784Z","shell.execute_reply.started":"2021-07-08T23:15:01.856538Z","shell.execute_reply":"2021-07-08T23:15:01.882794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_new = df_test_sent.iloc[:, 2:]\n\nprint(df_test_sent.shape)\nprint(X_new.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:15:01.885248Z","iopub.execute_input":"2021-07-08T23:15:01.885543Z","iopub.status.idle":"2021-07-08T23:15:01.891466Z","shell.execute_reply.started":"2021-07-08T23:15:01.885516Z","shell.execute_reply":"2021-07-08T23:15:01.890459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classifier\nFilter tokenized test sentences by keeping likely  sentences classified as `prob_1`","metadata":{}},{"cell_type":"code","source":"np.random.seed(1)\n\n%time predict = best_model.predict(X_new)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:15:01.892578Z","iopub.execute_input":"2021-07-08T23:15:01.892884Z","iopub.status.idle":"2021-07-08T23:15:02.013516Z","shell.execute_reply.started":"2021-07-08T23:15:01.892855Z","shell.execute_reply":"2021-07-08T23:15:02.012300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prob = best_model.predict_proba(X_new)\n\ndf_test_sent['prob_0'] = prob[:,0] \ndf_test_sent['prob_1'] = prob[:,1]","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:15:02.014959Z","iopub.execute_input":"2021-07-08T23:15:02.015408Z","iopub.status.idle":"2021-07-08T23:15:02.128182Z","shell.execute_reply.started":"2021-07-08T23:15:02.015367Z","shell.execute_reply":"2021-07-08T23:15:02.127348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_candidates = df_test_sent.query('prob_1 >= 0.95')\n\nprint(\"Percent of all sentences to pass to NER model: \\n\")\nprint((len(df_candidates)/len(df_test_sent))*100)","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:15:02.129422Z","iopub.execute_input":"2021-07-08T23:15:02.130172Z","iopub.status.idle":"2021-07-08T23:15:02.146094Z","shell.execute_reply.started":"2021-07-08T23:15:02.130112Z","shell.execute_reply":"2021-07-08T23:15:02.145036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Solution\n* Apply custom NER model to candidate sentences only and extract predicted entities\n* Add labels to predictions","metadata":{}},{"cell_type":"code","source":"df_test_pubs.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:15:02.147324Z","iopub.execute_input":"2021-07-08T23:15:02.147644Z","iopub.status.idle":"2021-07-08T23:15:02.170326Z","shell.execute_reply.started":"2021-07-08T23:15:02.147613Z","shell.execute_reply":"2021-07-08T23:15:02.169358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nresult = []\n\nfor index in submission_df.Id:\n    raw_text = df_test_pubs[df_test_pubs['Id'] == index].text.str.cat(sep='\\n')\n    clean_text = df_test_pubs[df_test_pubs['Id'] == index].clean_text.str.cat(sep='\\n')\n    scrub_text = df_test_pubs[df_test_pubs['Id'] == index].scrub_text.str.cat(sep='\\n')\n    all_text = raw_text + \" \" + clean_text + \" \" + scrub_text\n    label = set()\n    for mention in all_labels:\n        if mention in all_text:\n#             label.add(clean_text(mention))\n            label.add(re.sub(r'[^\\w\\s]','',mention).lower())\n    publication_text = df_candidates[df_candidates['Id'] == index].sent\n    for candidate in publication_text:\n        candidate = upper_text(candidate)\n        doc = custom_ner_model(candidate)\n        if len(doc.ents) > 0:\n            label.add(clean_text(doc.ents))\n    label_list = sorted(list(label))\n    result.append('|'.join(label_list))\n\nfor hit in result:\n    print(hit, \"\\n\")","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:15:02.171681Z","iopub.execute_input":"2021-07-08T23:15:02.172224Z","iopub.status.idle":"2021-07-08T23:15:02.898815Z","shell.execute_reply.started":"2021-07-08T23:15:02.172183Z","shell.execute_reply":"2021-07-08T23:15:02.897717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df['PredictionString'] = result\nsubmission_df.to_csv(\"./submission.csv\",index=False)\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2021-07-08T23:15:02.900225Z","iopub.execute_input":"2021-07-08T23:15:02.900497Z","iopub.status.idle":"2021-07-08T23:15:02.937444Z","shell.execute_reply.started":"2021-07-08T23:15:02.900468Z","shell.execute_reply":"2021-07-08T23:15:02.935480Z"},"trusted":true},"execution_count":null,"outputs":[]}]}