{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# String matching V2\nA notebook for matching strings of dataset titles to text in publications using more labels from external datasets.\n\nImprovements\n- clean df text\n- clean and unique labels\n","metadata":{}},{"cell_type":"code","source":"!pip install ../input/pandarallel151whl/pandarallel-1.5.1-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:13:04.073226Z","iopub.execute_input":"2021-06-22T22:13:04.073694Z","iopub.status.idle":"2021-06-22T22:13:30.755590Z","shell.execute_reply.started":"2021-06-22T22:13:04.073655Z","shell.execute_reply":"2021-06-22T22:13:30.754320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport re\nimport pandas as pd\nfrom pandarallel import pandarallel","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T22:13:30.758013Z","iopub.execute_input":"2021-06-22T22:13:30.758512Z","iopub.status.idle":"2021-06-22T22:13:30.765002Z","shell.execute_reply.started":"2021-06-22T22:13:30.758458Z","shell.execute_reply":"2021-06-22T22:13:30.763790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Initialize parallel processing","metadata":{}},{"cell_type":"code","source":"pandarallel.initialize(progress_bar=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:13:30.767065Z","iopub.execute_input":"2021-06-22T22:13:30.767398Z","iopub.status.idle":"2021-06-22T22:13:30.780713Z","shell.execute_reply.started":"2021-06-22T22:13:30.767365Z","shell.execute_reply":"2021-06-22T22:13:30.779604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`clean_text` function\n- lowercasing all text\n- removing all punctuation\n\n`scrub_text` function \n- removing nonalphabetic tokens\n- removing words shorter than one character","metadata":{"execution":{"iopub.status.busy":"2021-06-22T11:09:35.298475Z","iopub.execute_input":"2021-06-22T11:09:35.298999Z","iopub.status.idle":"2021-06-22T11:09:35.305029Z","shell.execute_reply.started":"2021-06-22T11:09:35.298964Z","shell.execute_reply":"2021-06-22T11:09:35.303743Z"}}},{"cell_type":"code","source":"def clean_text(txt):\n    \"\"\"\n    dedined by competition\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', txt.lower()).strip()\n\ndef scrub_text(txt):\n    \"\"\"\n    extends text cleaning\n    \"\"\"\n    scrub = re.sub('[^A-Za-z0-9]+', ' ', txt.lower()).strip()\n    return ''.join([i for i in scrub.lower() if not i.isdigit()])\n\ndef jaccard(str1, str2): \n    \"\"\"\n    defined by the competition\n    \"\"\"\n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))\n\ndef extract_acronyms(txt):\n    \"\"\"\n    finds and returns a sequence of capital letters\n    for use on dataset_titles, dataset_labels, or full text\n    \"\"\"\n    ac = []\n    matches = re.findall(r\"\\b[A-Z\\.]{2,}s?\\b\", txt)\n    if matches:\n        for match in matches:\n            ac.append(match)\n        return ac\n    else:\n        return ac\n    \ndef flatten_list(object):\n    gather = []\n    for item in object:\n        if isinstance(item, (list, tuple, set)):\n            gather.extend(flatten_list(item))            \n        else:\n            gather.append(item)\n    return gather\n\ndef filter_set(main_set, condition):\n    \"\"\"\n    used to remove items from label set based on a condition\n    \"\"\"\n    for elem in list(main_set):\n        if condition(elem):\n            main_set.discard(elem)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:13:30.782602Z","iopub.execute_input":"2021-06-22T22:13:30.782907Z","iopub.status.idle":"2021-06-22T22:13:30.812446Z","shell.execute_reply.started":"2021-06-22T22:13:30.782880Z","shell.execute_reply":"2021-06-22T22:13:30.810613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load data","metadata":{}},{"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\ndf_train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\ntest_files = glob.glob('../input/coleridgeinitiative-show-us-the-data/test/*.json')\n\ndf_test_pubs = pd.DataFrame()\nfor test_file in test_files: \n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'Id', test_file.split('/')[-1].split('.')[0])\n    df_test_pubs = pd.concat([df_test_pubs, file_data])\n\ndf_test_pubs['clean_text'] = df_test_pubs['text'].parallel_apply(clean_text)\ndf_test_pubs['scrub_text'] = df_test_pubs['text'].parallel_apply(scrub_text)\ndf_test_pubs","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:13:30.814472Z","iopub.execute_input":"2021-06-22T22:13:30.814851Z","iopub.status.idle":"2021-06-22T22:13:31.492082Z","shell.execute_reply.started":"2021-06-22T22:13:30.814816Z","shell.execute_reply":"2021-06-22T22:13:31.490968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Labels","metadata":{}},{"cell_type":"markdown","source":"SCRUBBED\n* Load study titles (n=7,442) and series (n=293) harvested from ICPSR\n* Load data titles (n=299,743) harvested from Data.gov CKAN (on 6/4/21)\n    * remove digits\n    * mask single word entries\n* Extract acronyms from training data titles and labels","metadata":{}},{"cell_type":"code","source":"icpsr_studies = pd.read_csv('../input/icpsr-study-names/icpsr_studies.csv')\nicpsr_studies_raw = icpsr_studies['NAME'].unique()\nicpsr_studies_clean = icpsr_studies['NAME'].parallel_apply(clean_text).unique()\nicpsr_studies_scrub = icpsr_studies['NAME'].parallel_apply(scrub_text).unique()\n\nicpsr_labels_1 = set(icpsr_studies_raw);\nicpsr_labels_2 = set(icpsr_studies_clean);\nicpsr_labels_3 = set(icpsr_studies_scrub);\n\nicpsr_studies_set = set.union(icpsr_labels_1, icpsr_labels_2, icpsr_labels_3);\n\nicspr_series = pd.read_csv('../input/icpsr-study-names/icpsr_series.csv').dropna()\nicspr_series_raw = icspr_series['TITLE'].unique()\nicspr_series_clean = icspr_series['TITLE'].parallel_apply(clean_text).unique()\nicspr_series_scrub = icspr_series['TITLE'].parallel_apply(scrub_text).unique()\n\nicpsr_series_1 = set(icspr_series_raw);\nicpsr_series_2 = set(icspr_series_clean);\nicpsr_series_3 = set(icspr_series_scrub);\n\nicpsr_series_set = set.union(icpsr_series_1, icpsr_series_2, icpsr_series_3);\n                                       \ndatagov_titles = pd.read_csv('../input/ckan-datagov-titles/ckan_data_gov_names.csv').dropna()\ndatagov_titles_unique = datagov_titles['index'].parallel_apply(scrub_text).unique()\ndatagov_titles_set = set(datagov_titles_unique);\nfilter_set(datagov_titles_set, lambda x : len(x) < 15)\n\ndatagov_labels = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\ndatagov_labels_raw = datagov_labels['title'].unique()\ndatagov_labels_clean = datagov_labels['title'].parallel_apply(clean_text).unique()\ndatagov_labels_scrub = datagov_labels['title'].parallel_apply(scrub_text).unique()\n\ndatagov_labels_1 = set(datagov_labels_raw);\ndatagov_labels_2 = set(datagov_labels_clean);\ndatagov_labels_3 = set(datagov_labels_scrub);\n\ndatagov_labels_set = set.union(datagov_labels_1, datagov_labels_2, datagov_labels_3)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:13:31.493539Z","iopub.execute_input":"2021-06-22T22:13:31.493833Z","iopub.status.idle":"2021-06-22T22:13:36.650612Z","shell.execute_reply.started":"2021-06-22T22:13:31.493802Z","shell.execute_reply":"2021-06-22T22:13:36.649409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training labels, titles, acronyms","metadata":{}},{"cell_type":"code","source":"train_title = df_train['dataset_title'].unique()\ntrain_title = set(train_title);\n\ntrain_label = df_train['dataset_label'].unique()\ntrain_label = set(train_label);\n\n# scrub_train_title = df_train['dataset_title'].parallel_apply(scrub_text).dropna()\n# scrub_train_title = set(scrub_train_title);\n\n# scrub_train_label = df_train['dataset_label'].parallel_apply(scrub_text).dropna()\n# scrub_train_label = set(scrub_train_label);\n\nacronyms_label = df_train['dataset_title'].parallel_apply(extract_acronyms).dropna()\nac_label = set(flatten_list(acronyms_label))\n\nacronyms_title = df_train['dataset_label'].parallel_apply(extract_acronyms).dropna()\nac_title = set(flatten_list(acronyms_title))\n\nacronym_upper = set.union(ac_label, ac_title);","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:13:36.652306Z","iopub.execute_input":"2021-06-22T22:13:36.652655Z","iopub.status.idle":"2021-06-22T22:13:37.628324Z","shell.execute_reply.started":"2021-06-22T22:13:36.652620Z","shell.execute_reply":"2021-06-22T22:13:37.627242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Merge all labels into a set and tidy","metadata":{}},{"cell_type":"code","source":"all_labels = set.union(train_title,\n                       train_label,\n                       acronym_upper,\n                       icpsr_studies_set, \n                       icpsr_series_set, \n                       datagov_titles_set, \n                       datagov_labels_set);\n\n# all_labels.remove('none')\n\nlen(all_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:13:37.629662Z","iopub.execute_input":"2021-06-22T22:13:37.629956Z","iopub.status.idle":"2021-06-22T22:13:37.692316Z","shell.execute_reply.started":"2021-06-22T22:13:37.629928Z","shell.execute_reply":"2021-06-22T22:13:37.691065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predictions","metadata":{}},{"cell_type":"code","source":"preds = []\nfor index in submission_df['Id']:\n    raw_text = df_test_pubs[df_test_pubs['Id'] == index].text.str.cat(sep='\\n')\n    clean_text = df_test_pubs[df_test_pubs['Id'] == index].clean_text.str.cat(sep='\\n')\n    scrub_text = df_test_pubs[df_test_pubs['Id'] == index].scrub_text.str.cat(sep='\\n')\n    all_text = raw_text + \" \" + clean_text + \" \" + scrub_text\n    label = set()\n    for mention in all_labels:\n        if mention in all_text:\n            label.add(re.sub(r'[^\\w\\s]','',mention).lower().strip())\n    sorted_list = sorted(list(label))\n    preds.append('|'.join(sorted_list))\n\nfor prediction in preds:\n    print(prediction)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:18:52.243932Z","iopub.execute_input":"2021-06-22T22:18:52.244335Z","iopub.status.idle":"2021-06-22T22:20:44.237959Z","shell.execute_reply.started":"2021-06-22T22:18:52.244301Z","shell.execute_reply":"2021-06-22T22:20:44.236650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df['PredictionString'] = preds\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:15:29.335786Z","iopub.execute_input":"2021-06-22T22:15:29.336071Z","iopub.status.idle":"2021-06-22T22:15:29.345389Z","shell.execute_reply.started":"2021-06-22T22:15:29.336043Z","shell.execute_reply":"2021-06-22T22:15:29.344601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T22:15:29.346586Z","iopub.execute_input":"2021-06-22T22:15:29.347031Z","iopub.status.idle":"2021-06-22T22:15:29.360964Z","shell.execute_reply.started":"2021-06-22T22:15:29.346998Z","shell.execute_reply":"2021-06-22T22:15:29.359962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}