{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport pandas as pd\nimport numpy as np\nimport spacy","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-26T22:58:27.515561Z","iopub.execute_input":"2021-06-26T22:58:27.516148Z","iopub.status.idle":"2021-06-26T22:58:27.520293Z","shell.execute_reply.started":"2021-06-26T22:58:27.516081Z","shell.execute_reply":"2021-06-26T22:58:27.51944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Function definitions","metadata":{}},{"cell_type":"code","source":"def extract(tok, exclusion_phrases=[]):\n    \"\"\"Looks backwards and forwards from the token to extract a dataset mention.\n    The optional exclusion_phrases argument is a list of strings that, if any one is present,\n    no dataset mention is returned.\n    \"\"\"\n    matches = []\n    special_tokens1 = ['a', 'the', 'in', 'on', 'of', 'across', 'and']\n    special_tokens2 = ['-', '\\n', '\\\"', \"'s\"]\n    special_tokens3 = ['-', \"'s\", '19']\n    special_tokens4 = ['center', 'centers', 'centre', 'centres', 'program', 'programmes', \\\n                       'division', 'branch', 'branches', 'administration']\n    \n    match = [tok.text]\n    \n    # if the token is not at the beginning of the sentence...\n    if not tok.is_sent_start:\n        ptr = -1\n        prev_tok = tok.nbor(ptr)\n        \n        # Add words to the left of the token.\n        while not prev_tok.text in special_tokens1 and (prev_tok.text.istitle() or \\\n            prev_tok.text.isupper()) or prev_tok.text in special_tokens1 + special_tokens3 and \\\n            not prev_tok.is_sent_start:\n            match.insert(0, prev_tok.text)\n            \n            if not prev_tok.is_sent_start:\n                ptr = ptr - 1\n                prev_tok = tok.nbor(ptr)\n            else:\n                break\n        \n        # Chop off extraneous words\n        while match[0][0].islower() or match[0].lower() in special_tokens1 or \\\n            match[0] in special_tokens2:\n            match.pop(0)\n        \n        # If the token is not at the end of the sentence...\n        if not tok.is_sent_end:\n            ptr = 1\n            nxt_tok = tok.nbor(ptr)\n            \n            # Add words to the right of the token\n            while nxt_tok.text.istitle() or nxt_tok.text in special_tokens1 + special_tokens3:\n                match.append(nxt_tok.text)\n                \n                if not nxt_tok.is_sent_end:\n                    ptr = ptr + 1\n                    nxt_tok = tok.nbor(ptr)\n                else:\n                    break\n            \n            # Chop off extraneous words\n            while match[-1][0].islower() or match[-1].lower() in special_tokens1 or \\\n                match[-1] in special_tokens2 or '-' in match and \\\n                (match.index(tok.text) < match.index('-')):\n                match.pop(-1)\n            \n            # If the dataset mention is at least two words long and the last word is not an\n            # desirable word...\n            if len(match) > 2 and not match[-1].lower() in special_tokens4:\n                match_text = ' '.join(match)\n                \n                # If no exclusion phrases are present...\n                if all([x not in match_text.lower() for x in exclusion_phrases]):\n                    # remove extra spaces for hyphens and apostrophe s that were \n                    # introduced in the ' ' .join\n                    match_text = match_text.replace(' -', '-')\n                    match_text = match_text.replace('- ', '-')\n                    match_text = match_text.replace(\" 's\", \"'s\")\n                    \n                    # split if two datasets, which sometimes happens\n                    if 'and the' in match_text:\n                        two_matches = match_text.split(' and the ')\n                        \n                        if len(two_matches[0].split(' ')) > 2:\n                            matches.append(two_matches[0])\n                        \n                        if len(two_matches[1].split(' ')) > 2:\n                            matches.append(two_matches[1])\n                    else:\n                        matches.append(match_text)\n    \n    matches = list(set(matches))\n    \n    return matches    ","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:27.523014Z","iopub.execute_input":"2021-06-26T22:58:27.523361Z","iopub.status.idle":"2021-06-26T22:58:27.543816Z","shell.execute_reply.started":"2021-06-26T22:58:27.523329Z","shell.execute_reply":"2021-06-26T22:58:27.542689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extractor(txt):\n    \"\"\"Executes the extractor functions as the keywords are encountered in the text\"\"\"\n    \n    def chunks(text, n):\n        \"\"\"Used to chunk text if it is longer than SpaCy's allowed nlp.max_length\"\"\"\n        for idx in range(0, len(text), n):\n            yield text[idx:idx + n]\n    \n    all_matches = []\n    \n    # put a space on either side of a hyphen so that it will be tokenized separately\n    txt = re.sub(r'(.)-(.)', r'\\1 - \\2', txt)\n    \n    # put a space on either side of a year that is touching other text so that the year\n    # will be tokenized separately\n    txt = re.sub(r'(.)((19|20)[0-9]{2})(.)', r'\\1 \\2 \\4', txt)\n    \n    for txt_chunk in chunks(txt, nlp.max_length):\n        for doc in nlp.pipe([txt_chunk]):\n            for tok in doc:\n                if tok.text in keywords.keys():\n                    all_matches.extend(extract(tok, keywords[tok.text]))\n                \n    all_matches = list(set(all_matches))\n    \n    return all_matches","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:27.546023Z","iopub.execute_input":"2021-06-26T22:58:27.546465Z","iopub.status.idle":"2021-06-26T22:58:27.562942Z","shell.execute_reply.started":"2021-06-26T22:58:27.546428Z","shell.execute_reply":"2021-06-26T22:58:27.561288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    \"\"\"Clean the text as specified in the competition instructions\"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:27.565455Z","iopub.execute_input":"2021-06-26T22:58:27.565856Z","iopub.status.idle":"2021-06-26T22:58:27.579631Z","shell.execute_reply.started":"2021-06-26T22:58:27.565822Z","shell.execute_reply":"2021-06-26T22:58:27.578371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_string_matches(txt):\n    \"\"\"Find matches of strings in hi_freq_datasets\"\"\"\n    matches = []\n    \n    for label in hi_freq_datasets:\n        # X out all matches of this label so that shorter strings representing the same\n        # dataset do not hit the mention that has already been accounted for\n        txt, n_subs = re.subn(label, 'X'*len(label), txt)\n        \n        if n_subs > 0:\n            matches.append(clean_text(label))\n    \n    matches = list(set(matches))\n    \n    return matches","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:27.581481Z","iopub.execute_input":"2021-06-26T22:58:27.581841Z","iopub.status.idle":"2021-06-26T22:58:27.593069Z","shell.execute_reply.started":"2021-06-26T22:58:27.581804Z","shell.execute_reply":"2021-06-26T22:58:27.591841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Apply Extractor Functions\nThrough some exploratory methods, it appeared that words like Study, Survey, Database, Dataset, Archive, Assessment, Catalog, Collection, Registry, and Initiative appeared frequently in presumed dataset mentions. For each word, I create a function to examine words around each of these words to extract dataset mentions. While I initially considered doing the extraction with regular expressions, I discovered the versatility of SpaCy’s sentencizer and leveraged it. The sentencizer permits an easy, intuitive, pythonic way to examine tokens appearing before and after words, including built-in functions for checking tokens for case combinations and sentence positions.\n\nI iteratively developed these extractor functions by running them on the training data, ranking the extracted dataset mentions by their document frequency, and modifying the functions to make cleaner extractions with more focus put on the dataset mentions with higher document frequency.","metadata":{}},{"cell_type":"markdown","source":"Create a dictionary called `keywords` that has dataset words of interest as keys and a list of strings to exclude as values ","metadata":{}},{"cell_type":"code","source":"keywords = {'Study':['of study', 'case study'], 'Survey':['geologic', 'of survey', 'system'], \\\n            'Database':[], 'Dataset':[], 'Archive':[], 'Assessment':[], 'Catalog':[], \\\n            'Collection':[], 'Registry':[], 'Initiative':[]}","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:27.594873Z","iopub.execute_input":"2021-06-26T22:58:27.595329Z","iopub.status.idle":"2021-06-26T22:58:27.607987Z","shell.execute_reply.started":"2021-06-26T22:58:27.595284Z","shell.execute_reply":"2021-06-26T22:58:27.606807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load SpaCy's small English model and remove every pipe except for the sentencizer. Removing uneeded pipes substantial reduces run time.","metadata":{}},{"cell_type":"code","source":"nlp = spacy.load('en_core_web_sm')\n\nnlp.add_pipe(nlp.create_pipe('sentencizer'))\nnlp.remove_pipe('tagger')\nnlp.remove_pipe('parser')\nnlp.remove_pipe('ner')\n\nnlp.pipeline","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:27.609507Z","iopub.execute_input":"2021-06-26T22:58:27.61005Z","iopub.status.idle":"2021-06-26T22:58:28.975649Z","shell.execute_reply.started":"2021-06-26T22:58:27.610015Z","shell.execute_reply":"2021-06-26T22:58:28.974464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load up all the IDs and raw text  of the test data into a dataframe.","metadata":{}},{"cell_type":"code","source":"file_dir = '../input/coleridgeinitiative-show-us-the-data/test'\n\nfile_id = []\ntext = []\n\nfiles = os.listdir(file_dir)\n\nfor file in files:\n    file_id.append(os.path.splitext(os.path.basename(file))[0])\n    tmp_text = ' '.join(pd.read_json(os.path.join(file_dir, file), orient='records')['text'])\n    text.append(tmp_text)\n\ndf = pd.DataFrame(data={'Id':file_id, 'raw_text':text})","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:28.97832Z","iopub.execute_input":"2021-06-26T22:58:28.978785Z","iopub.status.idle":"2021-06-26T22:58:29.072329Z","shell.execute_reply.started":"2021-06-26T22:58:28.978738Z","shell.execute_reply":"2021-06-26T22:58:29.071151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Apply the dataset extractor to the raw text in each dataframe row","metadata":{}},{"cell_type":"code","source":"preds_list = df['raw_text'].apply(extractor)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:29.074174Z","iopub.execute_input":"2021-06-26T22:58:29.07458Z","iopub.status.idle":"2021-06-26T22:58:29.649596Z","shell.execute_reply.started":"2021-06-26T22:58:29.074536Z","shell.execute_reply":"2021-06-26T22:58:29.648466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Frequency filtering\nAfter combining the outputs of all extractor functions, I rank them by their document frequency and retain only the top 95 percentile. The rationale behind retaining the top 95 percentile is that infrequent dataset mentions are less likely to be important to the sponsor’s objective and, hence, less likely to be labeled in the testing data.","metadata":{}},{"cell_type":"code","source":"hi_freq_datasets = []\n\nfor preds in preds_list:\n    hi_freq_datasets.extend(preds)\n\ndf_temp = pd.Series(hi_freq_datasets).value_counts().to_frame('counts')\ndf_temp = df_temp[df_temp.index.str.len() > 0].copy() # remove blanks\nthresh = np.percentile(df_temp, 95)\nhi_freq_datasets = list(df_temp[df_temp.counts >= thresh].index)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:29.650804Z","iopub.execute_input":"2021-06-26T22:58:29.651161Z","iopub.status.idle":"2021-06-26T22:58:29.666216Z","shell.execute_reply.started":"2021-06-26T22:58:29.651132Z","shell.execute_reply":"2021-06-26T22:58:29.665046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Slap on all the datasets from the training data and all the ones that I found through trying\nhigh-frequency mentions from the training text through trial-and-error. (This only increased\nmy private LB score by 0.002, so it really wasn't necessary.)","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\nknown_datasets = pd.unique(df_train[['dataset_title', 'dataset_label']].values.ravel('K')).tolist()\nknown_datasets = known_datasets + ['National Postsecondary Student Aid Study', \\\n    'Schools and Staffing Survey', 'National Survey of College Graduates', \\\n    'Framingham Heart Study', 'National Survey of Recent College Graduates', \\\n    'Program for International Student Assessment', 'Health and Retirement Study', \\\n    'Private School Universe Survey', 'Teaching and Learning International Survey', \\\n    'National Crime Victimization Survey', 'International Mathematics and Science Study', \\\n    'Consumer Expenditure Survey', 'Current Population Survey', 'American Community Survey', \\\n    'National Health Interview Survey', 'Progress in International Reading Literacy Study', \\\n    'Scientists and Engineers Statistical Data System', \\\n    'International Best Track Archive for Climate Stewardship IBTrACS', \\\n    'Sea Lake and Overland Surges from Hurricanes SLOSH', \\\n    'National Longitudinal Survey of Youth', 'National Study of Postsecondary Faculty', \\\n    'National Educational Longitudinal Study', \\\n    'Global Initiative on Sharing All Influenza Data', \\\n    'National Longitudinal Study of Adolescent Health', \\\n    'National Longitudinal Study of Adolescent to Adult Health', 'ADCIRC', \\\n    'Private School Survey', 'National Land Cover', 'COVID-19 Open Data', \\\n    'National Education Longitudinal Study of 1988', \\\n    'High School Longitudinal Study of 2009', \\\n    'National Health and Nutrition Examination Survey']\n\nhi_freq_datasets = hi_freq_datasets + known_datasets\nhi_freq_datasets = list(set(hi_freq_datasets))\nhi_freq_datasets.sort(key=len, reverse=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:29.66767Z","iopub.execute_input":"2021-06-26T22:58:29.668566Z","iopub.status.idle":"2021-06-26T22:58:29.813393Z","shell.execute_reply.started":"2021-06-26T22:58:29.6685Z","shell.execute_reply":"2021-06-26T22:58:29.812331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Dataset Search\nThe last step is to search through the documents again but with a slight twist. The list of high-frequency dataset mentions is sorted by decreasing length. For each document, the sorted list is iterated through, replacing matches with Xs. For example, in theory, if “ADNI” and “Alzheimer’s Disease Neuroimaging Initiative (ADNI)” were both in the list and only “Alzheimer’s Disease Neuroimaging Initiative (ADNI)” was in the document, this method prevents predicting both “ADNI” and “Alzheimer’s Disease Neuroimaging Initiative (ADNI)”. To explain further, because it’s longer, “Alzheimer’s Disease Neuroimaging Initiative (ADNI)” would be found first, saved as a prediction, and Xed out. When the shorter “ADNI” is subsequently searched for, it will not be found. Only “Alzheimer’s Disease Neuroimaging Initiative (ADNI)” will be predicted as desired.","metadata":{}},{"cell_type":"code","source":"df['PredictionString'] = df['raw_text'].apply(get_string_matches).str.join('|')","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:29.814958Z","iopub.execute_input":"2021-06-26T22:58:29.815395Z","iopub.status.idle":"2021-06-26T22:58:29.903884Z","shell.execute_reply.started":"2021-06-26T22:58:29.815352Z","shell.execute_reply":"2021-06-26T22:58:29.902746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit predictions","metadata":{}},{"cell_type":"code","source":"df[['Id', 'PredictionString']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-26T22:58:29.905185Z","iopub.execute_input":"2021-06-26T22:58:29.905491Z","iopub.status.idle":"2021-06-26T22:58:29.914428Z","shell.execute_reply.started":"2021-06-26T22:58:29.905461Z","shell.execute_reply":"2021-06-26T22:58:29.913461Z"},"trusted":true},"execution_count":null,"outputs":[]}]}