{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook extracts words before Abbreviations in prentheses and extracts captilaized word sequences.  These extractions are then processed by a spacy classifier to predict if they are a dataset or not.","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport numpy as np\nimport pandas as pd\nimport re\nimport simplejson\nfrom joblib import Parallel, delayed\nfrom typing import *\nimport time\n\nimport spacy\nfrom spacy import displacy\nfrom spacy.util import minibatch, compounding\nimport random\n\nprint('packages loaded')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_titles_pos = pd.read_csv('../input/training-dataset-titles/dataset226_no_entities.csv')\ntraining_titles_pos['target']=1\n\ntraining_titles_neg = pd.read_csv('../input/training-dataset-titles/not_dataset.csv')\ntraining_titles_neg[\"title\"] = training_titles_neg[\"title\"].str.lower()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(text: str) -> str:               return re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\ndef clean_texts(texts: List[str]) -> List[str]: return [ clean_text(text) for text in texts ] \n\ndef read_json(index: str, test_train) -> Dict:\n    filename = f\"../input/coleridgeinitiative-show-us-the-data/{test_train}/{index}.json\"\n    with open(filename) as f:\n        json = simplejson.load(f)\n    return json\n        \ndef json2text(index: str, test_train) -> str:\n    json  = read_json(index, test_train)\n    texts = [\n        row[\"section_title\"] + \" \" + row[\"text\"] \n        for row in json\n    ]\n    # texts = clean_texts(texts)\n    text  = \" \".join(texts)\n    return text\n\ndef filename_to_index(filename):\n    return re.sub(\"^.*/|\\.[^.]+$\", '', filename)\n\ndef glob_to_indices(globpath):\n    return list(map(filename_to_index, glob.glob(globpath)))\n\n# Inspired by: https://www.kaggle.com/hamditarek/merge-multiple-json-files-to-a-dataframe\ndef dataset_df(test_train=\"test\"):\n    indices = glob_to_indices(f\"../input/coleridgeinitiative-show-us-the-data/{test_train}/*.json\")    \n    texts   = Parallel(-1)( \n        delayed(json2text)(index, test_train)\n        for index in indices  \n    )\n    df = pd.DataFrame([\n        { \"id\": index, \"text\": text}\n        for index, text in zip(indices, texts)\n    ])\n    df.to_csv(f\"{test_train}.json.csv\", index=False)\n    return df\n\ntrain_data = dataset_df(\"train\")\n#train_data[\"text\"] = train_data[\"text\"].str.lower()\ntest_data  = dataset_df(\"test\")\n#test_data[\"text\"] = train_data[\"text\"].str.lower()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# random sample of training data\ntrain_sample=train_data.sample(n = 50, replace=False)\n\ntrain_sample","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list1=[]\n\nfor index, row in training_titles_neg.iterrows():\n    #(\"higher education research and development survey\", {'cats': {'POSITIVE': 1}} )\n    a='(\"'\n    b='\",'\n    z=\"{'cats':{'POSITIVE': 0}}),\"\n    data=a+row['title']+b+z\n    #print (data)\n    list1.append(data)\n    #print (list1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spacy_train_data=[(\"adni\",{'cats':{'POSITIVE': 1}}),\n(\"cccsl\",{'cats':{'POSITIVE': 1}}),\n(\"ibtracs\",{'cats':{'POSITIVE': 1}}),\n(\"noaa c cap\",{'cats':{'POSITIVE': 1}}),\n(\"noaa c-cap\",{'cats':{'POSITIVE': 1}}),\n(\"slosh model\",{'cats':{'POSITIVE': 1}}),\n(\"noaa tide gauge\",{'cats':{'POSITIVE': 1}}),\n(\"noaa tide station\",{'cats':{'POSITIVE': 1}}),\n(\"jh-crown registry\",{'cats':{'POSITIVE': 1}}),\n(\"jh crown registry\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data\",{'cats':{'POSITIVE': 1}}),\n(\"noaa tidal station\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 death data\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 death data\",{'cats':{'POSITIVE': 1}}),\n(\"common core of data\",{'cats':{'POSITIVE': 1}}),\n(\"world ocean database\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 deaths data\",{'cats':{'POSITIVE': 1}}),\n(\"census of agriculture\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"noaa water level station\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"nces common core of data\",{'cats':{'POSITIVE': 1}}),\n(\"baccalaureate and beyond\",{'cats':{'POSITIVE': 1}}),\n(\"noaa world ocean database\",{'cats':{'POSITIVE': 1}}),\n(\"aging integrated database\",{'cats':{'POSITIVE': 1}}),\n(\"2019-ncov genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"2019 ncov genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data covid-19\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"nass census of agriculture\",{'cats':{'POSITIVE': 1}}),\n(\"2019-ncov genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"anss comprehensive catalog\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data covid 19\",{'cats':{'POSITIVE': 1}}),\n(\"2019 ncov genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"usda census of agriculture\",{'cats':{'POSITIVE': 1}}),\n(\"covid open research dataset\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of covid-19\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 open research data\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of covid 19\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"rural urban continuum codes\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 open research data\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"noaa storm surge inundation\",{'cats':{'POSITIVE': 1}}),\n(\"survey of earned doctorates\",{'cats':{'POSITIVE': 1}}),\n(\"rural-urban continuum codes\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of 2019-ncov\",{'cats':{'POSITIVE': 1}}),\n(\"education longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of 2019 ncov\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of covid 19\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of covid-19\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of 2019 ncov\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of sars-cov-2\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of 2019-ncov\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of sars cov 2\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of sars-cov-2\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 open research dataset\",{'cats':{'POSITIVE': 1}}),\n(\"high school longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of sars cov 2\",{'cats':{'POSITIVE': 1}}),\n(\"survey of doctorate recipients\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 image data collection\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 image data collection\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 open research dataset\",{'cats':{'POSITIVE': 1}}),\n(\"coastal change analysis program\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 full genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"beginning postsecondary student\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 full genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"aging integrated database agid \",{'cats':{'POSITIVE': 1}}),\n(\"nsf survey of earned doctorates\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 full genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"aging integrated database (agid)\",{'cats':{'POSITIVE': 1}}),\n(\"beginning postsecondary students\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 full genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"school survey on crime and safety\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data covid 19 dataset\",{'cats':{'POSITIVE': 1}}),\n(\"early childhood longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data covid-19 dataset\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 complete genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 complete genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"2019-ncov complete genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"2019 ncov complete genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"north american breeding bird survey\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 complete genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"ncses survey of doctorate recipients\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 complete genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"baltimore longitudinal study of aging\",{'cats':{'POSITIVE': 1}}),\n(\"ffrdc research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"national education longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"national teacher and principal survey\",{'cats':{'POSITIVE': 1}}),\n(\"anss comprehensive earthquake catalog\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 open research dataset cord 19 \",{'cats':{'POSITIVE': 1}}),\n(\"agricultural resource management survey\",{'cats':{'POSITIVE': 1}}),\n(\"agricultural resources management survey\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 open research dataset (cord-19)\",{'cats':{'POSITIVE': 1}}),\n(\"usgs north american breeding bird survey\",{'cats':{'POSITIVE': 1}}),\n(\"national water level observation network\",{'cats':{'POSITIVE': 1}}),\n(\"north american breeding bird survey bbs \",{'cats':{'POSITIVE': 1}}),\n(\"north american breeding bird survey (bbs)\",{'cats':{'POSITIVE': 1}}),\n(\"nsf ffrdc research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"national assessment of education progress\",{'cats':{'POSITIVE': 1}}),\n(\"alzheimers disease neuroimaging initiative\",{'cats':{'POSITIVE': 1}}),\n(\"coastal change analysis program land cover\",{'cats':{'POSITIVE': 1}}),\n(\"baccalaureate and beyond longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"baltimore longitudinal study of aging blsa \",{'cats':{'POSITIVE': 1}}),\n(\"sea lake and overland surges from hurricanes\",{'cats':{'POSITIVE': 1}}),\n(\"baltimore longitudinal study of aging (blsa)\",{'cats':{'POSITIVE': 1}}),\n(\"noaa national water level observation network\",{'cats':{'POSITIVE': 1}}),\n(\"optimum interpolation sea surface temperature\",{'cats':{'POSITIVE': 1}}),\n(\"sea surface temperature optimum interpolation\",{'cats':{'POSITIVE': 1}}),\n(\"survey of industrial research and development\",{'cats':{'POSITIVE': 1}}),\n(\"nws storm surge risk\",{'cats':{'POSITIVE': 1}}),\n(\"storm surge risk\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid 19 antiviral candidate compounds data\",{'cats':{'POSITIVE': 1}}),\n(\"sea surface temperature - optimum interpolation\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid-19 antiviral candidate compounds data\",{'cats':{'POSITIVE': 1}}),\n(\"higher education research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid open radiology database\",{'cats':{'POSITIVE': 1}}),\n(\"alzheimer s disease neuroimaging initiative adni \",{'cats':{'POSITIVE': 1}}),\n(\"noaa sea lake and overland surges from hurricanes\",{'cats':{'POSITIVE': 1}}),\n(\"noaa sea lake and overland surges from hurricanes\",{'cats':{'POSITIVE': 1}}),\n(\"nsf survey of industrial research and development\",{'cats':{'POSITIVE': 1}}),\n(\"arms farm financial and crop production practices\",{'cats':{'POSITIVE': 1}}),\n(\"noaa optimum interpolation sea surface temperature\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid 19 antiviral candidate compounds dataset\",{'cats':{'POSITIVE': 1}}),\n(\"alzheimer's disease neuroimaging initiative (adni)\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid-19 antiviral candidate compounds dataset\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid 19 antiviral candidate compounds data set\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid-19 antiviral candidate compounds data set\",{'cats':{'POSITIVE': 1}}),\n(\"survey of state government research and development\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid-19 open radiology database\",{'cats':{'POSITIVE': 1}}),\n(\"beginning postsecondary students longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid 19 open radiology database\",{'cats':{'POSITIVE': 1}}),\n(\"nsf higher education research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"trends in international mathematics and science study\",{'cats':{'POSITIVE': 1}}),\n(\"noaa c-cap\",{'cats':{'POSITIVE': 1}}),\n(\"noaa c cap\",{'cats':{'POSITIVE': 1}}),\n(\"survey of science and engineering research facilities\",{'cats':{'POSITIVE': 1}}),\n(\"advanced national seismic system comprehensive catalog\",{'cats':{'POSITIVE': 1}}),\n(\"complexity science hub covid-19 control strategies list\",{'cats':{'POSITIVE': 1}}),\n(\"survey of earned doctorates\",{'cats':{'POSITIVE': 1}}),\n(\"complexity science hub covid 19 control strategies list\",{'cats':{'POSITIVE': 1}}),\n(\"international best track archive for climate stewardship\",{'cats':{'POSITIVE': 1}}),\n(\"nsf survey of science and engineering research facilities\",{'cats':{'POSITIVE': 1}}),\n(\"survey of doctorate recipients\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid 19 open radiology database ricord \",{'cats':{'POSITIVE': 1}}),\n(\"common core of data\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid-19 open radiology database (ricord)\",{'cats':{'POSITIVE': 1}}),\n(\"noaa international best track archive for climate stewardship\",{'cats':{'POSITIVE': 1}}),\n(\"program for the international assessment of adult competencies\",{'cats':{'POSITIVE': 1}}),\n(\"complexity science hub covid 19 control strategies list cccsl \",{'cats':{'POSITIVE': 1}}),\n(\"complexity science hub covid-19 control strategies list (cccsl)\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 precision medicine analytics platform registry (jh-crown)\",{'cats':{'POSITIVE': 1}}),\n(\"advanced national seismic system anss comprehensive catalog comcat \",{'cats':{'POSITIVE': 1}}),\n(\"world ocean database\",{'cats':{'POSITIVE': 1}}),\n(\"advanced national seismic system (anss) comprehensive catalog (comcat)\",{'cats':{'POSITIVE': 1}}),\n(\"survey of industrial research and development\",{'cats':{'POSITIVE': 1}}),\n(\"survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"higher education research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"nsf survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"characterizing health associated risks and your baseline disease in sars cov 2\",{'cats':{'POSITIVE': 1}}),\n(\"characterizing health associated risks and your baseline disease in sars-cov-2\",{'cats':{'POSITIVE': 1}}),\n(\"ncses survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"genetics of alzheimer s disease data storage site\",{'cats':{'POSITIVE': 1}}),\n(\"genetics of alzheimer's disease data storage site\",{'cats':{'POSITIVE': 1}}),\n(\"survey of science and engineering research facilities\",{'cats':{'POSITIVE': 1}}),\n(\"survey of earned doctorates\",{'cats':{'POSITIVE': 1}}),\n(\"survey of doctorate recipients\",{'cats':{'POSITIVE': 1}}),\n(\"characterizing health associated risks and your baseline disease in sars cov 2 charybdis \",{'cats':{'POSITIVE': 1}}),\n(\"characterizing health associated risks and your baseline disease in sars-cov-2 (charybdis)\",{'cats':{'POSITIVE': 1}}),\n(\"alzheimer s disease data storage site niagads \",{'cats':{'POSITIVE': 1}}),\n(\"alzheimer's disease data storage site (niagads)\",{'cats':{'POSITIVE': 1}}),\n(\"optimum interpolation sea surface temperature\",{'cats':{'POSITIVE': 1}}),\n(\"survey of industrial research and development\",{'cats':{'POSITIVE': 1}}),\n(\"survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"higher education research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"survey of science and engineering research facilities\",{'cats':{'POSITIVE': 1}}),\n(\"survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"local food economic impact assessment\",{'cats':{'POSITIVE': 1}}),\n(\"tvb model\",{'cats':{'POSITIVE': 1}}),\n(\"mexican american study project\",{'cats':{'POSITIVE': 1}}),\n(\"national longitudinal study of adolescent health\",{'cats':{'POSITIVE': 1}}),\n(\"national health interview survey\",{'cats':{'POSITIVE': 1}}),\n(\"project talent longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"sea lake and overland surges from hurricanes slosh model\",{'cats':{'POSITIVE': 1}}),\n(\"schools and staffing survey\",{'cats':{'POSITIVE': 1}}),\n(\"private school universe survey\",{'cats':{'POSITIVE': 1}}),\n(\"fast response survey system\",{'cats':{'POSITIVE': 1}}),\n(\"schools and staffing survey\",{'cats':{'POSITIVE': 1}}),\n(\"national assessment of educational progress\",{'cats':{'POSITIVE': 1}}),\n(\"coinmon core of data\",{'cats':{'POSITIVE': 1}}),\n(\"national postsecondary student aid study\",{'cats':{'POSITIVE': 1}}),\n(\"postsecondary student aid study\",{'cats':{'POSITIVE': 1}}),\n(\"national study of postsecondary faculty\",{'cats':{'POSITIVE': 1}}),\n(\"survey on vocational programs in secondary schools\",{'cats':{'POSITIVE': 1}}),\n(\"district survey of alternative schools and programs\",{'cats':{'POSITIVE': 1}}),\n(\"state nonfiscal survey of public elementary secondary education\",{'cats':{'POSITIVE': 1}}),\n(\"national public education financial survey\",{'cats':{'POSITIVE': 1}}),\n(\"annual survey of government finances school systems f 33 survey\",{'cats':{'POSITIVE': 1}}),\n(\"integrated postsecondary education data system\",{'cats':{'POSITIVE': 1}}),\n(\"ipeds\",{'cats':{'POSITIVE': 1}}),\n(\"nsopf\",{'cats':{'POSITIVE': 1}}),\n(\"bps data\",{'cats':{'POSITIVE': 1}}),\n(\"postsecondary students longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"public elementary secondary school universe survey\",{'cats':{'POSITIVE': 1}}),\n(\"ccd local education agency universe survey\",{'cats':{'POSITIVE': 1}}),\n(\"ccd national public education financial survey\",{'cats':{'POSITIVE': 1}}),\n(\"nces national education longitudinal study of 1988\",{'cats':{'POSITIVE': 1}}),\n(\"agricultural resource management survey\",{'cats':{'POSITIVE': 1}}),\n(\"aez model\",{'cats':{'POSITIVE': 1}}),\n(\"usda corn yield data\",{'cats':{'POSITIVE': 1}}),\n(\"argonne national laboratory s greet\",{'cats':{'POSITIVE': 1}}),\n(\"greet model\",{'cats':{'POSITIVE': 1}}),\n(\"argonne national laboratory s cclub model\",{'cats':{'POSITIVE': 1}}),\n(\"national education longitudinal survey\",{'cats':{'POSITIVE': 1}}),\n(\"education longitudinal study of 2002\",{'cats':{'POSITIVE': 1}}),\n(\"progress in international reading literacy study\",{'cats':{'POSITIVE': 1}}),\n(\"usgs national water quality assessment\",{'cats':{'POSITIVE': 1}}),\n(\"ipeds fall enrollment dataset\",{'cats':{'POSITIVE': 1}}),\n(\"fall enrollment dataset\",{'cats':{'POSITIVE': 1}}),\n(\"delta cost project\",{'cats':{'POSITIVE': 1}}),\n(\"annual survey of colleges standard research compilation\",{'cats':{'POSITIVE': 1}}),\n(\"chesapeake bay watershed land cover data series\",{'cats':{'POSITIVE': 1}}),\n(\"nccpi corn and soybeans sub model\",{'cats':{'POSITIVE': 1}}),\n(\"national longitudinal survey of youth\",{'cats':{'POSITIVE': 1}}),\n(\"national survey of teachers\",{'cats':{'POSITIVE': 1}}),\n(\"high school transcript study\",{'cats':{'POSITIVE': 1}}),\n(\"clinical dementia rating\",{'cats':{'POSITIVE': 0}}),\n(\"services web feature services\",{'cats':{'POSITIVE': 0}}),\n(\"university south carolina\",{'cats':{'POSITIVE': 0}}),\n(\"latent dirichlet allocation\",{'cats':{'POSITIVE': 0}}),\n(\"montreal neurological institute\",{'cats':{'POSITIVE': 0}}),\n(\"national board professional teaching standards\",{'cats':{'POSITIVE': 0}}),\n(\"multiple endmember spectral mixture analysis\",{'cats':{'POSITIVE': 0}}),\n(\"bay katmai national park\",{'cats':{'POSITIVE': 0}}),\n(\"united nations development programme\",{'cats':{'POSITIVE': 0}}),\n(\"compact high resolution imaging spectrometer\",{'cats':{'POSITIVE': 0}}),\n(\"office management budget\",{'cats':{'POSITIVE': 0}}),\n(\"florida department natural resources\",{'cats':{'POSITIVE': 0}}),\n(\"university neuroinformatics research group\",{'cats':{'POSITIVE': 0}}),\n(\"national canadian pacific rail service\",{'cats':{'POSITIVE': 0}}),\n(\"portobello marine laboratory\",{'cats':{'POSITIVE': 0}}),\n(\"hilbert schmidt independence criterion\",{'cats':{'POSITIVE': 0}}),\n(\"genomic distance-based regression\",{'cats':{'POSITIVE': 0}}),\n(\"national weather service\",{'cats':{'POSITIVE': 0}}),\n(\"deep boltzmann machine\",{'cats':{'POSITIVE': 0}}),\n(\"multiple indicators multiple cause\",{'cats':{'POSITIVE': 0}}),\n(\"quasi maximum likelihood\",{'cats':{'POSITIVE': 0}}),\n(\"the eastern equatorial pacific\",{'cats':{'POSITIVE': 0}}),\n(\"autoregressive distributed lag\",{'cats':{'POSITIVE': 0}}),\n(\"city chesapeake mosquito control commission\",{'cats':{'POSITIVE': 0}}),\n(\"the california verbal learning test\",{'cats':{'POSITIVE': 0}}),\n(\"long short term memory\",{'cats':{'POSITIVE': 0}}),\n(\"detection computer-aided detection\",{'cats':{'POSITIVE': 0}}),\n(\"stacked sparse autoencoder\",{'cats':{'POSITIVE': 0}}),\n(\"brain tumor segmentation\",{'cats':{'POSITIVE': 0}}),\n(\"geriatric depression scale\",{'cats':{'POSITIVE': 0}}),\n(\"australian institute teaching school leadership\",{'cats':{'POSITIVE': 0}}),\n(\"glasgow coma scale\",{'cats':{'POSITIVE': 0}}),\n(\"shared environmental information system\",{'cats':{'POSITIVE': 0}}),\n(\"international conferences agricultural statistics\",{'cats':{'POSITIVE': 0}}),\n(\"least absolute shrinkage selection operator\",{'cats':{'POSITIVE': 0}}),\n(\"color trails test\",{'cats':{'POSITIVE': 0}}),\n(\"international consortium brain mapping\",{'cats':{'POSITIVE': 0}}),\n(\"hardyweinberg disequilibrium\",{'cats':{'POSITIVE': 0}}),\n(\"advanced very high resolution radiometer\",{'cats':{'POSITIVE': 0}}),\n(\"spatial empirical bayesian\",{'cats':{'POSITIVE': 0}}),\n(\"rectified linear unit\",{'cats':{'POSITIVE': 0}}),\n(\"networks generative adversarial networks\",{'cats':{'POSITIVE': 0}}),\n(\"digital database screening mammography\",{'cats':{'POSITIVE': 0}}),\n(\"rhode island emergency management agency\",{'cats':{'POSITIVE': 0}}),\n(\"national marine fisheries service\",{'cats':{'POSITIVE': 0}}),\n(\"office marine aviation operations\",{'cats':{'POSITIVE': 0}}),\n(\"national oceanic atmospheric administration\",{'cats':{'POSITIVE': 0}}),\n(\"national hurricane center\",{'cats':{'POSITIVE': 0}}),\n(\"federal emergency management administration\",{'cats':{'POSITIVE': 0}}),\n(\"the social science genetic association consortium\",{'cats':{'POSITIVE': 0}}),\n(\"usda economic research service\",{'cats':{'POSITIVE': 0}}),\n(\"world health organization\",{'cats':{'POSITIVE': 0}}),\n(\"global positioning systems\",{'cats':{'POSITIVE': 0}}),\n(\"national center education statistics\",{'cats':{'POSITIVE': 0}}),\n(\"amyotrophic lateral sclerosis\",{'cats':{'POSITIVE': 0}}),\n(\"the economic research service\",{'cats':{'POSITIVE': 0}}),\n(\"clinical advisory board\",{'cats':{'POSITIVE': 0}}),\n(\"institute education sciences\",{'cats':{'POSITIVE': 0}}),\n(\"national centers environmental prediction\",{'cats':{'POSITIVE': 0}}),\n(\"coastal environmental research committee\",{'cats':{'POSITIVE': 0}}),\n(\"non-governmental organizations\",{'cats':{'POSITIVE': 0}}),\n(\"high performance computing\",{'cats':{'POSITIVE': 0}}),\n(\"engineer research development center\",{'cats':{'POSITIVE': 0}}),\n(\"usda economic research service\",{'cats':{'POSITIVE': 0}}),\n(\"national oceanic atmospheric administration\",{'cats':{'POSITIVE': 0}}),\n(\"national hurricane center\",{'cats':{'POSITIVE': 0}}),\n(\"economic research service\",{'cats':{'POSITIVE': 0}}),\n(\"higher order statistics\",{'cats':{'POSITIVE': 0}}),\n(\"common data elements\",{'cats':{'POSITIVE': 0}}),\n(\"north east south east\",{'cats':{'POSITIVE': 0}}),\n(\"national climatic data center\",{'cats':{'POSITIVE': 0}}),\n(\"national institute aging\",{'cats':{'POSITIVE': 0}}),\n(\"genome wide\",{'cats':{'POSITIVE': 0}}),\n(\"national science foundation\",{'cats':{'POSITIVE': 0}}),\n(\"national center education statistics\",{'cats':{'POSITIVE': 0}}),\n(\"local binary pattern\",{'cats':{'POSITIVE': 0}}),\n(\"higher order statistics\",{'cats':{'POSITIVE': 0}}),\n(\"north east south east\",{'cats':{'POSITIVE': 0}}),\n(\"world health organization\",{'cats':{'POSITIVE': 0}}),\n(\"clinically diagnosed dementia alzheimer type\",{'cats':{'POSITIVE': 0}}),\n(\"u s geological survey\",{'cats':{'POSITIVE': 0}}),\n(\"c pittsburgh compound b\",{'cats':{'POSITIVE': 0}}),\n(\"principal component analysis\",{'cats':{'POSITIVE': 0}}),\n(\"u s department education\",{'cats':{'POSITIVE': 0}}),\n(\"u s trained\",{'cats':{'POSITIVE': 0}}),\n(\"coronaviruses\",{'cats':{'POSITIVE': 0}}),\n(\"ordinary least square\",{'cats':{'POSITIVE': 0}}),\n(\"materials and methods u s geological survey\",{'cats':{'POSITIVE': 0}}),\n(\"mean decrease accuracy\",{'cats':{'POSITIVE': 0}}),\n(\"undergraduate training program\",{'cats':{'POSITIVE': 0}}),\n(\"education for all\",{'cats':{'POSITIVE': 0}}),\n(\"foldcurv curvind gauscurv\",{'cats':{'POSITIVE': 0}}),\n(\"community atmosphere model\",{'cats':{'POSITIVE': 0}}),\n(\"annual percent change\",{'cats':{'POSITIVE': 0}}),\n(\"new york stock exchange\",{'cats':{'POSITIVE': 0}}),\n(\"average treatment effect\",{'cats':{'POSITIVE': 0}}),\n(\"demographic\",{'cats':{'POSITIVE': 0}}),\n(\"introduction coronaviruses\",{'cats':{'POSITIVE': 0}}),\n(\"autism brain imaging data exchange\",{'cats':{'POSITIVE': 0}}),\n(\"national cancer data base\",{'cats':{'POSITIVE': 0}}),\n(\"programa internacional avalia o alunos\",{'cats':{'POSITIVE': 0}}),\n(\"national science foundation s\",{'cats':{'POSITIVE': 0}}),\n(\"tes bakat skolastik\",{'cats':{'POSITIVE': 0}}),\n(\"oregon department environmental quality\",{'cats':{'POSITIVE': 0}}),\n(\"study variables facility oncology registry data standards\",{'cats':{'POSITIVE': 0}}),\n(\"state university entrance\",{'cats':{'POSITIVE': 0}}),\n(\"south china sea\",{'cats':{'POSITIVE': 0}}),\n(\"introduction human\",{'cats':{'POSITIVE': 0}}),\n(\"dulbecco s eagle s\",{'cats':{'POSITIVE': 0}}),\n(\"institute medical genetics cardiff\",{'cats':{'POSITIVE': 0}}),\n(\"sars cov 2 a\",{'cats':{'POSITIVE': 0}}),\n(\"disease coronaviruses\",{'cats':{'POSITIVE': 0}}),\n(\"office management budget s\",{'cats':{'POSITIVE': 0}}),\n(\"rna dependent rna\",{'cats':{'POSITIVE': 0}}),\n(\"t4 dna ligase\",{'cats':{'POSITIVE': 0}}),\n(\"u s department agriculture\",{'cats':{'POSITIVE': 0}}),\n(\"chicago parent program\",{'cats':{'POSITIVE': 0}}),\n(\"indian ocean\",{'cats':{'POSITIVE': 0}}),\n(\"turkish republic northern cyprus\",{'cats':{'POSITIVE': 0}}),\n(\"the south african schools act\",{'cats':{'POSITIVE': 0}}),\n(\"blood oxygen level dependent\",{'cats':{'POSITIVE': 0}}),\n(\"duke university s institutional review board\",{'cats':{'POSITIVE': 0}}),\n(\"international archive education data\",{'cats':{'POSITIVE': 0}}),\n(\"nonprofit technology network\",{'cats':{'POSITIVE': 0}}),\n(\"muller s opportunity tolearn\",{'cats':{'POSITIVE': 0}}),\n(\"global extreme sea level analysis\",{'cats':{'POSITIVE': 0}}),\n(\"the addenbrooke s cognitive examination\",{'cats':{'POSITIVE': 0}}),\n(\"information resource incorporated\",{'cats':{'POSITIVE': 0}}),\n(\"norwegian cognitive neurogenetics\",{'cats':{'POSITIVE': 0}}),\n(\"introduction cape hatteras national seashore\",{'cats':{'POSITIVE': 0}}),\n(\"heart study second generation cohort\",{'cats':{'POSITIVE': 0}}),\n(\"national hurricane center\",{'cats':{'POSITIVE': 0}}),\n(\"second follow up\",{'cats':{'POSITIVE': 0}}),\n(\"title iv\",{'cats':{'POSITIVE': 0}}),\n(\"jackknife ii\",{'cats':{'POSITIVE': 0}}),\n(\"statistics brief\",{'cats':{'POSITIVE': 0}}),\n(\"research cnn rnn based\",{'cats':{'POSITIVE': 0}}),\n(\"ct ct\",{'cats':{'POSITIVE': 0}}),\n(\"designed cnn\",{'cats':{'POSITIVE': 0}}),\n(\"pca and umap assisted k means\",{'cats':{'POSITIVE': 0}}),\n(\"in figure\",{'cats':{'POSITIVE': 0}}),\n(\"roundy frank\",{'cats':{'POSITIVE': 0}}),\n(\"abnormality recognition early ad\",{'cats':{'POSITIVE': 0}}),\n(\"lite ed2 5 top of atmosphere\",{'cats':{'POSITIVE': 0}}),\n(\"global forest resource assessment\",{'cats':{'POSITIVE': 0}}),\n(\"foodprint model\",{'cats':{'POSITIVE': 0}}),\n(\"local tci slr relied\",{'cats':{'POSITIVE': 0}}),\n(\"ad t1 weighted\",{'cats':{'POSITIVE': 0}}),\n(\"italy england sars cov 2\",{'cats':{'POSITIVE': 0}}),\n(\"wuhan italy\",{'cats':{'POSITIVE': 0}}),\n(\"the nsf\",{'cats':{'POSITIVE': 0}}),\n(\"comparison previous study the\",{'cats':{'POSITIVE': 0}}),\n(\"the netherlands\",{'cats':{'POSITIVE': 0}}),\n(\"international database\",{'cats':{'POSITIVE': 0}}),\n(\"assessment martin\",{'cats':{'POSITIVE': 0}}),\n(\"sas system windows\",{'cats':{'POSITIVE': 0}}),\n(\"section ii\",{'cats':{'POSITIVE': 0}}),\n(\"d cnn\",{'cats':{'POSITIVE': 0}}),\n(\"in table\",{'cats':{'POSITIVE': 0}}),\n(\"different adni 1database\",{'cats':{'POSITIVE': 0}}),\n(\"adni 1 adni 2\",{'cats':{'POSITIVE': 0}}),\n(\"results we\",{'cats':{'POSITIVE': 0}}),\n(\"whereas foland ross\",{'cats':{'POSITIVE': 0}}),\n(\"performance iq\",{'cats':{'POSITIVE': 0}}),\n(\"verbal iq\",{'cats':{'POSITIVE': 0}}),\n(\"nc flood risk information system\",{'cats':{'POSITIVE': 0}}),\n(\"nc sea level rise risk management study\",{'cats':{'POSITIVE': 0}}),\n(\"canada\",{'cats':{'POSITIVE': 0}}),\n(\"germany\",{'cats':{'POSITIVE': 0}})\n]\nprint ('training data loaded')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data = [(\"higher education research and development survey\", {'cats': {'POSITIVE': 1}} ), (\"the social science genetic association consortium\", {'cats': {'POSITIVE': 0}})]\n\nnlp = spacy.load('en_core_web_sm')\n\nif 'textcat' not in nlp.pipe_names:\n    textcat = nlp.create_pipe(\"textcat\")\n    nlp.add_pipe(textcat, last=True)\nelse:\n    textcat = nlp.get_pipe(\"textcat\")\n\ntextcat.add_label('POSITIVE')\n#textcat.add_label('NEGATIVE')\n\nother_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']\n\nn_iter = 15\n\n\nwith nlp.disable_pipes(*other_pipes):\n    optimizer = nlp.begin_training()\n    print(\"Training model...\")\n    \n    for i in range(n_iter):\n        losses = {}\n        batches = minibatch(spacy_train_data, size=compounding(4,32,1.001))\n        for batch in batches:\n            texts, annotations = zip(*batch)\n            nlp.update(texts, annotations, sgd=optimizer,drop=0.2, losses=losses)\n            \nprint('done')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/training-dataset-titles/data_used_language.csv')\ndf.drop_duplicates(keep=\"first\", inplace=True)\n\ndata_language=df['words'].astype(str).values.tolist()\n\n#print (data_language)\n\nlong_string=\"In principle, it would have been possible to generate as well a multiplicity of alternatives same SCbi-MFM counterparts, leading to an even more varied augmented dataset, but we were limited by the computational resources needed to build a multiplicity of alternative SCbi-MFM 's for each of the subjects in our ADNI-derived dataset (nonlinear FC-to-SC completion is way harder computationally than SC-to-FC completion)\"\n\nwords = [word for word in data_language if word in long_string]\n\nprint (len(words))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.parsing.preprocessing import remove_stopwords\nfrom nltk.tokenize import sent_tokenize\n\n#### remove >.5 jaccard matches from predicitons\ndef jaccard_similarity(s1, s2):\n    l1 = s1.split()\n    l2 = s2.split()    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\n#############################\n\nstart_time = time.time()\n#test\ncolumn_names = [\"Id\", \"PredictionString\"]\n#train\n#column_names = [\"Id\", \"PredictionString\", \"matched\"]\nsubmission = pd.DataFrame(columns = column_names)\n\nno_delete = ['study', 'dataset', 'model','survey','data','adni','codes', 'genome', 'program','assessment','database','census','initiative','gauge','system','stewardship','surge']\n\n#doc1 = nlp(u\"Data used in preparation of this article were obtained from\")\n#doc1 = nlp(\"using data from\")\n#doc1 = nlp(\"used estimate studied data from\")\n\nfor index, row in test_data.iterrows():\n    #print ('############################')\n    #print (row['id'])\n    to_append=[row['id'],'']\n    #to_append=[row['id'],'','']\n    passage=row['text']\n    passage=passage.replace(\"'s\",\"s\")\n    passage=passage.replace(\"-\",\" \")\n    passage=passage.replace(\",\",\" \")\n    ##### isin\n    #for index, row2 in training_titles_pos.iterrows():\n        #query_string = str(row2['title'].lower())\n        #if query_string in passage.lower(): #and \"data\" in sentence\n            #print ('-->',query_string)\n    #print ('____________________')\n    \n    ##################### sentences \n    #sentences=passage.split(\".\")\n    #sentences=sent_tokenize(passage)\n    #for sentence in sentences:\n        #if len(sentence)>100:\n            #words = [word for word in data_language if word in sentence.lower()]\n            #if len(words)>=2:\n    ######## ACRONYMS\n    for match in re.finditer(r\"(\\(([A-Z]{2,})\\))\", passage):\n    #for match in re.finditer(r\"(\\((.*?)\\))\", data):\n        caps=[]\n        start_index = match.start()\n        abbr = match.group(1)\n        size = len(abbr)\n        words = passage[:start_index].split()[-size:]\n        #print (abbr,words)\n        for word in words:\n            if word[0].isupper():\n                caps.append(word)\n        definition = \" \".join(caps)\n        #print (definition)\n        if sum(1 for c in definition if c.isupper()) < 15:\n            words = [word for word in no_delete if word in definition.lower()]\n            doc=nlp(definition)\n            score=doc.cats['POSITIVE']\n            #print(definition,abbr, score)\n            if len(words)>0 and  score > .99:\n                #print(definition,abbr, score)\n                if to_append[1]!='' and definition not in to_append[1]:\n                    to_append[1] = to_append[1]+'|'+definition+'|'+abbr\n                    to_append[1] = to_append[1]+'|'+abbr\n                if to_append[1]=='':\n                    to_append[1] = definition\n                    to_append[1] = to_append[1]+'|'+abbr\n                            \n    ######\n    #### cap word sequence\n    if to_append[1]=='':        \n        mylist=re.findall('([A-Z][\\w-]*(?:\\s+[A-Z][\\w-]*)+)', remove_stopwords(passage))\n        mylist = list(dict.fromkeys(mylist))\n        for match in mylist:\n            upper_score=sum(1 for c in match if c.isupper())\n            #print (match, upper_score)\n            if upper_score < 15:\n                words = [word for word in no_delete if word in match.lower()]\n                doc=nlp(match)\n                score=doc.cats['POSITIVE']\n                #print (match, score)\n                if len(words)>0 and len(match.split())>=2 and score > .99:\n                    #print (match, score)\n                    #print ('__________________')\n                    if to_append[1]!='' and match not in to_append[1]:\n                        to_append[1] = to_append[1]+'|'+match\n                    if to_append[1]=='':\n                        to_append[1] = match\n            \n                            \n    #if row['id']=='3f316b38-1a24-45a9-8d8c-4e05a42257c6':\n        #print (\"start next paper\")\n        #print (passage)\n    \n    #print (to_append[1])\n    ###### remove similar jaccard\n    got_label=to_append[1].split('|')\n    filtered=[]\n    filtered_labels = ''\n    for label in sorted(got_label, key=len):\n        label = clean_text(label)\n        if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < .5 for got_label in filtered):\n            filtered.append(label)\n            if filtered_labels!='':\n                filtered_labels=filtered_labels+'|'+label\n            if filtered_labels=='':\n                filtered_labels=label\n    \n    to_append[1] = filtered_labels  \n    \n    \n    #print ('################')\n    #print (to_append)\n    #print ('################')\n    ###### remove similar jaccard\n    df_length = len(submission)\n    submission.loc[df_length] = to_append\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\nsubmission.to_csv('submission.csv', index = False)\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}