{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Spacy classifier\n> Baseline from Ken Miller. See [here](https://www.kaggle.com/mlconsult/44-3-spacy-classifier-no-matching).","metadata":{}},{"cell_type":"code","source":"# additional python packages\n!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install -q ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install -q ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install -q ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport simplejson\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\n# dataset manipulation\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# pytorch\nimport torch\nfrom datasets import load_dataset\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline\n\nfrom typing import List\nimport string\nfrom functools import partial\n\nimport pickle\nfrom joblib import Parallel, delayed\n\nfrom collections import defaultdict, Counter\nimport gc\n\n# tf\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\n# language preprocessing\nimport nltk\n\nfrom typing import *\n\n# spacy\nimport spacy\nfrom spacy import displacy\nfrom spacy.util import minibatch, compounding\n\n# set seed\nsns.set()\nrandom.seed(123)\nnp.random.seed(456)\ntorch.manual_seed(42)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\nprint('packages loaded')","metadata":{"execution":{"iopub.status.busy":"2021-09-25T01:16:08.788592Z","iopub.execute_input":"2021-09-25T01:16:08.788929Z","iopub.status.idle":"2021-09-25T01:16:23.933726Z","shell.execute_reply.started":"2021-09-25T01:16:08.788873Z","shell.execute_reply":"2021-09-25T01:16:23.931448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(text: str) -> str:               return re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\ndef clean_texts(texts: List[str]) -> List[str]: return [ clean_text(text) for text in texts ] \n\ndef read_json(index: str, test_train) -> Dict:\n    filename = f\"../input/coleridgeinitiative-show-us-the-data/{test_train}/{index}.json\"\n    with open(filename) as f:\n        json = simplejson.load(f)\n    return json\n        \ndef json2text(index: str, test_train) -> str:\n    json  = read_json(index, test_train)\n    texts = [\n        row[\"section_title\"] + \" \" + row[\"text\"] \n        for row in json\n    ]\n    text  = \" \".join(texts)\n    return text\n\ndef filename_to_index(filename):\n    return re.sub(\"^.*/|\\.[^.]+$\", '', filename)\n\ndef glob_to_indices(globpath):\n    return list(map(filename_to_index, glob.glob(globpath)))\n\n# Inspired by: https://www.kaggle.com/hamditarek/merge-multiple-json-files-to-a-dataframe\ndef dataset_df(test_train=\"test\"):\n    indices = glob_to_indices(f\"../input/coleridgeinitiative-show-us-the-data/{test_train}/*.json\")    \n    texts   = Parallel(-1)( \n        delayed(json2text)(index, test_train)\n        for index in indices  \n    )\n    df = pd.DataFrame([\n        { \"id\": index, \"text\": text}\n        for index, text in zip(indices, texts)\n    ])\n    df.to_csv(f\"{test_train}.json.csv\", index=False)\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-09-25T01:16:25.313361Z","iopub.execute_input":"2021-09-25T01:16:25.315918Z","iopub.status.idle":"2021-09-25T01:16:25.329371Z","shell.execute_reply.started":"2021-09-25T01:16:25.315883Z","shell.execute_reply":"2021-09-25T01:16:25.328168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\npapers = {}\nfor paper_id in sample_submission['Id'].values:\n    with open(f'../input/coleridgeinitiative-show-us-the-data/test/{paper_id}.json', 'r') as f:\n        sections = json.load(f)\n        paper = ''\n        for section in sections:\n            paper = paper + section['text'] + ' .'\n    papers[paper_id] = paper\n    del paper","metadata":{"execution":{"iopub.status.busy":"2021-09-25T01:17:44.975974Z","iopub.execute_input":"2021-09-25T01:17:44.976675Z","iopub.status.idle":"2021-09-25T01:17:45.052581Z","shell.execute_reply.started":"2021-09-25T01:17:44.976635Z","shell.execute_reply":"2021-09-25T01:17:45.051513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spacy_train_data=[(\"adni\",{'cats':{'POSITIVE': 1}}),\n(\"cccsl\",{'cats':{'POSITIVE': 1}}),\n(\"ibtracs\",{'cats':{'POSITIVE': 1}}),\n(\"noaa c cap\",{'cats':{'POSITIVE': 1}}),\n(\"noaa c-cap\",{'cats':{'POSITIVE': 1}}),\n(\"slosh model\",{'cats':{'POSITIVE': 1}}),\n(\"noaa tide gauge\",{'cats':{'POSITIVE': 1}}),\n(\"noaa tide station\",{'cats':{'POSITIVE': 1}}),\n(\"jh-crown registry\",{'cats':{'POSITIVE': 1}}),\n(\"jh crown registry\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data\",{'cats':{'POSITIVE': 1}}),\n(\"noaa tidal station\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 death data\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 death data\",{'cats':{'POSITIVE': 1}}),\n(\"common core of data\",{'cats':{'POSITIVE': 1}}),\n(\"world ocean database\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 deaths data\",{'cats':{'POSITIVE': 1}}),\n(\"census of agriculture\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"noaa water level station\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"nces common core of data\",{'cats':{'POSITIVE': 1}}),\n(\"baccalaureate and beyond\",{'cats':{'POSITIVE': 1}}),\n(\"noaa world ocean database\",{'cats':{'POSITIVE': 1}}),\n(\"aging integrated database\",{'cats':{'POSITIVE': 1}}),\n(\"2019-ncov genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"2019 ncov genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data covid-19\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"nass census of agriculture\",{'cats':{'POSITIVE': 1}}),\n(\"2019-ncov genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"anss comprehensive catalog\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data covid 19\",{'cats':{'POSITIVE': 1}}),\n(\"2019 ncov genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"usda census of agriculture\",{'cats':{'POSITIVE': 1}}),\n(\"covid open research dataset\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of covid-19\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 open research data\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of covid 19\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"rural urban continuum codes\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 open research data\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"noaa storm surge inundation\",{'cats':{'POSITIVE': 1}}),\n(\"survey of earned doctorates\",{'cats':{'POSITIVE': 1}}),\n(\"rural-urban continuum codes\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of 2019-ncov\",{'cats':{'POSITIVE': 1}}),\n(\"education longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of 2019 ncov\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of covid 19\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of covid-19\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of 2019 ncov\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of sars-cov-2\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of 2019-ncov\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequence of sars cov 2\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of sars-cov-2\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 open research dataset\",{'cats':{'POSITIVE': 1}}),\n(\"high school longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"genome sequences of sars cov 2\",{'cats':{'POSITIVE': 1}}),\n(\"survey of doctorate recipients\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 image data collection\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 image data collection\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 open research dataset\",{'cats':{'POSITIVE': 1}}),\n(\"coastal change analysis program\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 full genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"beginning postsecondary student\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 full genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"aging integrated database agid \",{'cats':{'POSITIVE': 1}}),\n(\"nsf survey of earned doctorates\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 full genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"aging integrated database (agid)\",{'cats':{'POSITIVE': 1}}),\n(\"beginning postsecondary students\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 full genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"school survey on crime and safety\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data covid 19 dataset\",{'cats':{'POSITIVE': 1}}),\n(\"early childhood longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"our world in data covid-19 dataset\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 complete genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 complete genome sequence\",{'cats':{'POSITIVE': 1}}),\n(\"2019-ncov complete genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"2019 ncov complete genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"north american breeding bird survey\",{'cats':{'POSITIVE': 1}}),\n(\"sars-cov-2 complete genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"ncses survey of doctorate recipients\",{'cats':{'POSITIVE': 1}}),\n(\"sars cov 2 complete genome sequences\",{'cats':{'POSITIVE': 1}}),\n(\"baltimore longitudinal study of aging\",{'cats':{'POSITIVE': 1}}),\n(\"ffrdc research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"national education longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"national teacher and principal survey\",{'cats':{'POSITIVE': 1}}),\n(\"anss comprehensive earthquake catalog\",{'cats':{'POSITIVE': 1}}),\n(\"covid 19 open research dataset cord 19 \",{'cats':{'POSITIVE': 1}}),\n(\"agricultural resource management survey\",{'cats':{'POSITIVE': 1}}),\n(\"agricultural resources management survey\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 open research dataset (cord-19)\",{'cats':{'POSITIVE': 1}}),\n(\"usgs north american breeding bird survey\",{'cats':{'POSITIVE': 1}}),\n(\"national water level observation network\",{'cats':{'POSITIVE': 1}}),\n(\"north american breeding bird survey bbs \",{'cats':{'POSITIVE': 1}}),\n(\"north american breeding bird survey (bbs)\",{'cats':{'POSITIVE': 1}}),\n(\"nsf ffrdc research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"national assessment of education progress\",{'cats':{'POSITIVE': 1}}),\n(\"alzheimers disease neuroimaging initiative\",{'cats':{'POSITIVE': 1}}),\n(\"coastal change analysis program land cover\",{'cats':{'POSITIVE': 1}}),\n(\"baccalaureate and beyond longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"baltimore longitudinal study of aging blsa \",{'cats':{'POSITIVE': 1}}),\n(\"sea lake and overland surges from hurricanes\",{'cats':{'POSITIVE': 1}}),\n(\"baltimore longitudinal study of aging (blsa)\",{'cats':{'POSITIVE': 1}}),\n(\"noaa national water level observation network\",{'cats':{'POSITIVE': 1}}),\n(\"optimum interpolation sea surface temperature\",{'cats':{'POSITIVE': 1}}),\n(\"sea surface temperature optimum interpolation\",{'cats':{'POSITIVE': 1}}),\n(\"survey of industrial research and development\",{'cats':{'POSITIVE': 1}}),\n(\"nws storm surge risk\",{'cats':{'POSITIVE': 1}}),\n(\"storm surge risk\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid 19 antiviral candidate compounds data\",{'cats':{'POSITIVE': 1}}),\n(\"sea surface temperature - optimum interpolation\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid-19 antiviral candidate compounds data\",{'cats':{'POSITIVE': 1}}),\n(\"higher education research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid open radiology database\",{'cats':{'POSITIVE': 1}}),\n(\"alzheimer s disease neuroimaging initiative adni \",{'cats':{'POSITIVE': 1}}),\n(\"noaa sea lake and overland surges from hurricanes\",{'cats':{'POSITIVE': 1}}),\n(\"noaa sea lake and overland surges from hurricanes\",{'cats':{'POSITIVE': 1}}),\n(\"nsf survey of industrial research and development\",{'cats':{'POSITIVE': 1}}),\n(\"arms farm financial and crop production practices\",{'cats':{'POSITIVE': 1}}),\n(\"noaa optimum interpolation sea surface temperature\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid 19 antiviral candidate compounds dataset\",{'cats':{'POSITIVE': 1}}),\n(\"alzheimer's disease neuroimaging initiative (adni)\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid-19 antiviral candidate compounds dataset\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid 19 antiviral candidate compounds data set\",{'cats':{'POSITIVE': 1}}),\n(\"cas covid-19 antiviral candidate compounds data set\",{'cats':{'POSITIVE': 1}}),\n(\"survey of state government research and development\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid-19 open radiology database\",{'cats':{'POSITIVE': 1}}),\n(\"beginning postsecondary students longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid 19 open radiology database\",{'cats':{'POSITIVE': 1}}),\n(\"nsf higher education research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"trends in international mathematics and science study\",{'cats':{'POSITIVE': 1}}),\n(\"noaa c-cap\",{'cats':{'POSITIVE': 1}}),\n(\"noaa c cap\",{'cats':{'POSITIVE': 1}}),\n(\"survey of science and engineering research facilities\",{'cats':{'POSITIVE': 1}}),\n(\"advanced national seismic system comprehensive catalog\",{'cats':{'POSITIVE': 1}}),\n(\"complexity science hub covid-19 control strategies list\",{'cats':{'POSITIVE': 1}}),\n(\"survey of earned doctorates\",{'cats':{'POSITIVE': 1}}),\n(\"complexity science hub covid 19 control strategies list\",{'cats':{'POSITIVE': 1}}),\n(\"international best track archive for climate stewardship\",{'cats':{'POSITIVE': 1}}),\n(\"nsf survey of science and engineering research facilities\",{'cats':{'POSITIVE': 1}}),\n(\"survey of doctorate recipients\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid 19 open radiology database ricord \",{'cats':{'POSITIVE': 1}}),\n(\"common core of data\",{'cats':{'POSITIVE': 1}}),\n(\"rsna international covid-19 open radiology database (ricord)\",{'cats':{'POSITIVE': 1}}),\n(\"noaa international best track archive for climate stewardship\",{'cats':{'POSITIVE': 1}}),\n(\"program for the international assessment of adult competencies\",{'cats':{'POSITIVE': 1}}),\n(\"complexity science hub covid 19 control strategies list cccsl \",{'cats':{'POSITIVE': 1}}),\n(\"complexity science hub covid-19 control strategies list (cccsl)\",{'cats':{'POSITIVE': 1}}),\n(\"covid-19 precision medicine analytics platform registry (jh-crown)\",{'cats':{'POSITIVE': 1}}),\n(\"advanced national seismic system anss comprehensive catalog comcat \",{'cats':{'POSITIVE': 1}}),\n(\"world ocean database\",{'cats':{'POSITIVE': 1}}),\n(\"advanced national seismic system (anss) comprehensive catalog (comcat)\",{'cats':{'POSITIVE': 1}}),\n(\"survey of industrial research and development\",{'cats':{'POSITIVE': 1}}),\n(\"survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"higher education research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"nsf survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"characterizing health associated risks and your baseline disease in sars cov 2\",{'cats':{'POSITIVE': 1}}),\n(\"characterizing health associated risks and your baseline disease in sars-cov-2\",{'cats':{'POSITIVE': 1}}),\n(\"ncses survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"genetics of alzheimer s disease data storage site\",{'cats':{'POSITIVE': 1}}),\n(\"genetics of alzheimer's disease data storage site\",{'cats':{'POSITIVE': 1}}),\n(\"survey of science and engineering research facilities\",{'cats':{'POSITIVE': 1}}),\n(\"survey of earned doctorates\",{'cats':{'POSITIVE': 1}}),\n(\"survey of doctorate recipients\",{'cats':{'POSITIVE': 1}}),\n(\"characterizing health associated risks and your baseline disease in sars cov 2 charybdis \",{'cats':{'POSITIVE': 1}}),\n(\"characterizing health associated risks and your baseline disease in sars-cov-2 (charybdis)\",{'cats':{'POSITIVE': 1}}),\n(\"alzheimer s disease data storage site niagads \",{'cats':{'POSITIVE': 1}}),\n(\"alzheimer's disease data storage site (niagads)\",{'cats':{'POSITIVE': 1}}),\n(\"optimum interpolation sea surface temperature\",{'cats':{'POSITIVE': 1}}),\n(\"survey of industrial research and development\",{'cats':{'POSITIVE': 1}}),\n(\"survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"higher education research and development survey\",{'cats':{'POSITIVE': 1}}),\n(\"survey of science and engineering research facilities\",{'cats':{'POSITIVE': 1}}),\n(\"survey of graduate students and postdoctorates in science and engineering\",{'cats':{'POSITIVE': 1}}),\n(\"local food economic impact assessment\",{'cats':{'POSITIVE': 1}}),\n(\"tvb model\",{'cats':{'POSITIVE': 1}}),\n(\"mexican american study project\",{'cats':{'POSITIVE': 1}}),\n(\"national longitudinal study of adolescent health\",{'cats':{'POSITIVE': 1}}),\n(\"national health interview survey\",{'cats':{'POSITIVE': 1}}),\n(\"project talent longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"sea lake and overland surges from hurricanes slosh model\",{'cats':{'POSITIVE': 1}}),\n(\"schools and staffing survey\",{'cats':{'POSITIVE': 1}}),\n(\"private school universe survey\",{'cats':{'POSITIVE': 1}}),\n(\"fast response survey system\",{'cats':{'POSITIVE': 1}}),\n(\"schools and staffing survey\",{'cats':{'POSITIVE': 1}}),\n(\"national assessment of educational progress\",{'cats':{'POSITIVE': 1}}),\n(\"coinmon core of data\",{'cats':{'POSITIVE': 1}}),\n(\"national postsecondary student aid study\",{'cats':{'POSITIVE': 1}}),\n(\"postsecondary student aid study\",{'cats':{'POSITIVE': 1}}),\n(\"national study of postsecondary faculty\",{'cats':{'POSITIVE': 1}}),\n(\"survey on vocational programs in secondary schools\",{'cats':{'POSITIVE': 1}}),\n(\"district survey of alternative schools and programs\",{'cats':{'POSITIVE': 1}}),\n(\"state nonfiscal survey of public elementary secondary education\",{'cats':{'POSITIVE': 1}}),\n(\"national public education financial survey\",{'cats':{'POSITIVE': 1}}),\n(\"annual survey of government finances school systems f 33 survey\",{'cats':{'POSITIVE': 1}}),\n(\"integrated postsecondary education data system\",{'cats':{'POSITIVE': 1}}),\n(\"ipeds\",{'cats':{'POSITIVE': 1}}),\n(\"nsopf\",{'cats':{'POSITIVE': 1}}),\n(\"bps data\",{'cats':{'POSITIVE': 1}}),\n(\"postsecondary students longitudinal study\",{'cats':{'POSITIVE': 1}}),\n(\"public elementary secondary school universe survey\",{'cats':{'POSITIVE': 1}}),\n(\"ccd local education agency universe survey\",{'cats':{'POSITIVE': 1}}),\n(\"ccd national public education financial survey\",{'cats':{'POSITIVE': 1}}),\n(\"nces national education longitudinal study of 1988\",{'cats':{'POSITIVE': 1}}),\n(\"agricultural resource management survey\",{'cats':{'POSITIVE': 1}}),\n(\"aez model\",{'cats':{'POSITIVE': 1}}),\n(\"usda corn yield data\",{'cats':{'POSITIVE': 1}}),\n(\"argonne national laboratory s greet\",{'cats':{'POSITIVE': 1}}),\n(\"greet model\",{'cats':{'POSITIVE': 1}}),\n(\"argonne national laboratory s cclub model\",{'cats':{'POSITIVE': 1}}),\n(\"national education longitudinal survey\",{'cats':{'POSITIVE': 1}}),\n(\"education longitudinal study of 2002\",{'cats':{'POSITIVE': 1}}),\n(\"progress in international reading literacy study\",{'cats':{'POSITIVE': 1}}),\n(\"usgs national water quality assessment\",{'cats':{'POSITIVE': 1}}),\n(\"ipeds fall enrollment dataset\",{'cats':{'POSITIVE': 1}}),\n(\"fall enrollment dataset\",{'cats':{'POSITIVE': 1}}),\n(\"delta cost project\",{'cats':{'POSITIVE': 1}}),\n(\"annual survey of colleges standard research compilation\",{'cats':{'POSITIVE': 1}}),\n(\"chesapeake bay watershed land cover data series\",{'cats':{'POSITIVE': 1}}),\n(\"nccpi corn and soybeans sub model\",{'cats':{'POSITIVE': 1}}),\n(\"national longitudinal survey of youth\",{'cats':{'POSITIVE': 1}}),\n(\"national survey of teachers\",{'cats':{'POSITIVE': 1}}),\n(\"high school transcript study\",{'cats':{'POSITIVE': 1}}),\n(\"clinical dementia rating\",{'cats':{'POSITIVE': 0}}),\n(\"services web feature services\",{'cats':{'POSITIVE': 0}}),\n(\"university south carolina\",{'cats':{'POSITIVE': 0}}),\n(\"latent dirichlet allocation\",{'cats':{'POSITIVE': 0}}),\n(\"montreal neurological institute\",{'cats':{'POSITIVE': 0}}),\n(\"national board professional teaching standards\",{'cats':{'POSITIVE': 0}}),\n(\"multiple endmember spectral mixture analysis\",{'cats':{'POSITIVE': 0}}),\n(\"bay katmai national park\",{'cats':{'POSITIVE': 0}}),\n(\"united nations development programme\",{'cats':{'POSITIVE': 0}}),\n(\"compact high resolution imaging spectrometer\",{'cats':{'POSITIVE': 0}}),\n(\"office management budget\",{'cats':{'POSITIVE': 0}}),\n(\"florida department natural resources\",{'cats':{'POSITIVE': 0}}),\n(\"university neuroinformatics research group\",{'cats':{'POSITIVE': 0}}),\n(\"national canadian pacific rail service\",{'cats':{'POSITIVE': 0}}),\n(\"portobello marine laboratory\",{'cats':{'POSITIVE': 0}}),\n(\"hilbert schmidt independence criterion\",{'cats':{'POSITIVE': 0}}),\n(\"genomic distance-based regression\",{'cats':{'POSITIVE': 0}}),\n(\"national weather service\",{'cats':{'POSITIVE': 0}}),\n(\"deep boltzmann machine\",{'cats':{'POSITIVE': 0}}),\n(\"multiple indicators multiple cause\",{'cats':{'POSITIVE': 0}}),\n(\"quasi maximum likelihood\",{'cats':{'POSITIVE': 0}}),\n(\"the eastern equatorial pacific\",{'cats':{'POSITIVE': 0}}),\n(\"autoregressive distributed lag\",{'cats':{'POSITIVE': 0}}),\n(\"city chesapeake mosquito control commission\",{'cats':{'POSITIVE': 0}}),\n(\"the california verbal learning test\",{'cats':{'POSITIVE': 0}}),\n(\"long short term memory\",{'cats':{'POSITIVE': 0}}),\n(\"detection computer-aided detection\",{'cats':{'POSITIVE': 0}}),\n(\"stacked sparse autoencoder\",{'cats':{'POSITIVE': 0}}),\n(\"brain tumor segmentation\",{'cats':{'POSITIVE': 0}}),\n(\"geriatric depression scale\",{'cats':{'POSITIVE': 0}}),\n(\"australian institute teaching school leadership\",{'cats':{'POSITIVE': 0}}),\n(\"glasgow coma scale\",{'cats':{'POSITIVE': 0}}),\n(\"shared environmental information system\",{'cats':{'POSITIVE': 0}}),\n(\"international conferences agricultural statistics\",{'cats':{'POSITIVE': 0}}),\n(\"least absolute shrinkage selection operator\",{'cats':{'POSITIVE': 0}}),\n(\"color trails test\",{'cats':{'POSITIVE': 0}}),\n(\"international consortium brain mapping\",{'cats':{'POSITIVE': 0}}),\n(\"hardyweinberg disequilibrium\",{'cats':{'POSITIVE': 0}}),\n(\"advanced very high resolution radiometer\",{'cats':{'POSITIVE': 0}}),\n(\"spatial empirical bayesian\",{'cats':{'POSITIVE': 0}}),\n(\"rectified linear unit\",{'cats':{'POSITIVE': 0}}),\n(\"networks generative adversarial networks\",{'cats':{'POSITIVE': 0}}),\n(\"digital database screening mammography\",{'cats':{'POSITIVE': 0}}),\n(\"rhode island emergency management agency\",{'cats':{'POSITIVE': 0}}),\n(\"national marine fisheries service\",{'cats':{'POSITIVE': 0}}),\n(\"office marine aviation operations\",{'cats':{'POSITIVE': 0}}),\n(\"national oceanic atmospheric administration\",{'cats':{'POSITIVE': 0}}),\n(\"national hurricane center\",{'cats':{'POSITIVE': 0}}),\n(\"federal emergency management administration\",{'cats':{'POSITIVE': 0}}),\n(\"the social science genetic association consortium\",{'cats':{'POSITIVE': 0}}),\n(\"usda economic research service\",{'cats':{'POSITIVE': 0}}),\n(\"world health organization\",{'cats':{'POSITIVE': 0}}),\n(\"global positioning systems\",{'cats':{'POSITIVE': 0}}),\n(\"national center education statistics\",{'cats':{'POSITIVE': 0}}),\n(\"amyotrophic lateral sclerosis\",{'cats':{'POSITIVE': 0}}),\n(\"the economic research service\",{'cats':{'POSITIVE': 0}}),\n(\"clinical advisory board\",{'cats':{'POSITIVE': 0}}),\n(\"institute education sciences\",{'cats':{'POSITIVE': 0}}),\n(\"national centers environmental prediction\",{'cats':{'POSITIVE': 0}}),\n(\"coastal environmental research committee\",{'cats':{'POSITIVE': 0}}),\n(\"non-governmental organizations\",{'cats':{'POSITIVE': 0}}),\n(\"high performance computing\",{'cats':{'POSITIVE': 0}}),\n(\"engineer research development center\",{'cats':{'POSITIVE': 0}}),\n(\"usda economic research service\",{'cats':{'POSITIVE': 0}}),\n(\"national oceanic atmospheric administration\",{'cats':{'POSITIVE': 0}}),\n(\"national hurricane center\",{'cats':{'POSITIVE': 0}}),\n(\"economic research service\",{'cats':{'POSITIVE': 0}}),\n(\"higher order statistics\",{'cats':{'POSITIVE': 0}}),\n(\"common data elements\",{'cats':{'POSITIVE': 0}}),\n(\"north east south east\",{'cats':{'POSITIVE': 0}}),\n(\"national climatic data center\",{'cats':{'POSITIVE': 0}}),\n(\"national institute aging\",{'cats':{'POSITIVE': 0}}),\n(\"genome wide\",{'cats':{'POSITIVE': 0}}),\n(\"national science foundation\",{'cats':{'POSITIVE': 0}}),\n(\"national center education statistics\",{'cats':{'POSITIVE': 0}}),\n(\"local binary pattern\",{'cats':{'POSITIVE': 0}}),\n(\"higher order statistics\",{'cats':{'POSITIVE': 0}}),\n(\"north east south east\",{'cats':{'POSITIVE': 0}}),\n(\"world health organization\",{'cats':{'POSITIVE': 0}}),\n(\"clinically diagnosed dementia alzheimer type\",{'cats':{'POSITIVE': 0}}),\n(\"u s geological survey\",{'cats':{'POSITIVE': 0}}),\n(\"c pittsburgh compound b\",{'cats':{'POSITIVE': 0}}),\n(\"principal component analysis\",{'cats':{'POSITIVE': 0}}),\n(\"u s department education\",{'cats':{'POSITIVE': 0}}),\n(\"u s trained\",{'cats':{'POSITIVE': 0}}),\n(\"coronaviruses\",{'cats':{'POSITIVE': 0}}),\n(\"ordinary least square\",{'cats':{'POSITIVE': 0}}),\n(\"materials and methods u s geological survey\",{'cats':{'POSITIVE': 0}}),\n(\"mean decrease accuracy\",{'cats':{'POSITIVE': 0}}),\n(\"undergraduate training program\",{'cats':{'POSITIVE': 0}}),\n(\"education for all\",{'cats':{'POSITIVE': 0}}),\n(\"foldcurv curvind gauscurv\",{'cats':{'POSITIVE': 0}}),\n(\"community atmosphere model\",{'cats':{'POSITIVE': 0}}),\n(\"annual percent change\",{'cats':{'POSITIVE': 0}}),\n(\"new york stock exchange\",{'cats':{'POSITIVE': 0}}),\n(\"average treatment effect\",{'cats':{'POSITIVE': 0}}),\n(\"demographic\",{'cats':{'POSITIVE': 0}}),\n(\"introduction coronaviruses\",{'cats':{'POSITIVE': 0}}),\n(\"autism brain imaging data exchange\",{'cats':{'POSITIVE': 0}}),\n(\"national cancer data base\",{'cats':{'POSITIVE': 0}}),\n(\"programa internacional avalia o alunos\",{'cats':{'POSITIVE': 0}}),\n(\"national science foundation s\",{'cats':{'POSITIVE': 0}}),\n(\"tes bakat skolastik\",{'cats':{'POSITIVE': 0}}),\n(\"oregon department environmental quality\",{'cats':{'POSITIVE': 0}}),\n(\"study variables facility oncology registry data standards\",{'cats':{'POSITIVE': 0}}),\n(\"state university entrance\",{'cats':{'POSITIVE': 0}}),\n(\"south china sea\",{'cats':{'POSITIVE': 0}}),\n(\"introduction human\",{'cats':{'POSITIVE': 0}}),\n(\"dulbecco s eagle s\",{'cats':{'POSITIVE': 0}}),\n(\"institute medical genetics cardiff\",{'cats':{'POSITIVE': 0}}),\n(\"sars cov 2 a\",{'cats':{'POSITIVE': 0}}),\n(\"disease coronaviruses\",{'cats':{'POSITIVE': 0}}),\n(\"office management budget s\",{'cats':{'POSITIVE': 0}}),\n(\"rna dependent rna\",{'cats':{'POSITIVE': 0}}),\n(\"t4 dna ligase\",{'cats':{'POSITIVE': 0}}),\n(\"u s department agriculture\",{'cats':{'POSITIVE': 0}}),\n(\"chicago parent program\",{'cats':{'POSITIVE': 0}}),\n(\"indian ocean\",{'cats':{'POSITIVE': 0}}),\n(\"turkish republic northern cyprus\",{'cats':{'POSITIVE': 0}}),\n(\"the south african schools act\",{'cats':{'POSITIVE': 0}}),\n(\"blood oxygen level dependent\",{'cats':{'POSITIVE': 0}}),\n(\"duke university s institutional review board\",{'cats':{'POSITIVE': 0}}),\n(\"international archive education data\",{'cats':{'POSITIVE': 0}}),\n(\"nonprofit technology network\",{'cats':{'POSITIVE': 0}}),\n(\"muller s opportunity tolearn\",{'cats':{'POSITIVE': 0}}),\n(\"global extreme sea level analysis\",{'cats':{'POSITIVE': 0}}),\n(\"the addenbrooke s cognitive examination\",{'cats':{'POSITIVE': 0}}),\n(\"information resource incorporated\",{'cats':{'POSITIVE': 0}}),\n(\"norwegian cognitive neurogenetics\",{'cats':{'POSITIVE': 0}}),\n(\"introduction cape hatteras national seashore\",{'cats':{'POSITIVE': 0}}),\n(\"heart study second generation cohort\",{'cats':{'POSITIVE': 0}}),\n(\"national hurricane center\",{'cats':{'POSITIVE': 0}}),\n(\"second follow up\",{'cats':{'POSITIVE': 0}}),\n(\"title iv\",{'cats':{'POSITIVE': 0}}),\n(\"jackknife ii\",{'cats':{'POSITIVE': 0}}),\n(\"statistics brief\",{'cats':{'POSITIVE': 0}}),\n(\"research cnn rnn based\",{'cats':{'POSITIVE': 0}}),\n(\"ct ct\",{'cats':{'POSITIVE': 0}}),\n(\"designed cnn\",{'cats':{'POSITIVE': 0}}),\n(\"pca and umap assisted k means\",{'cats':{'POSITIVE': 0}}),\n(\"in figure\",{'cats':{'POSITIVE': 0}}),\n(\"roundy frank\",{'cats':{'POSITIVE': 0}}),\n(\"abnormality recognition early ad\",{'cats':{'POSITIVE': 0}}),\n(\"lite ed2 5 top of atmosphere\",{'cats':{'POSITIVE': 0}}),\n(\"global forest resource assessment\",{'cats':{'POSITIVE': 0}}),\n(\"foodprint model\",{'cats':{'POSITIVE': 0}}),\n(\"local tci slr relied\",{'cats':{'POSITIVE': 0}}),\n(\"ad t1 weighted\",{'cats':{'POSITIVE': 0}}),\n(\"italy england sars cov 2\",{'cats':{'POSITIVE': 0}}),\n(\"wuhan italy\",{'cats':{'POSITIVE': 0}}),\n(\"the nsf\",{'cats':{'POSITIVE': 0}}),\n(\"comparison previous study the\",{'cats':{'POSITIVE': 0}}),\n(\"the netherlands\",{'cats':{'POSITIVE': 0}}),\n(\"international database\",{'cats':{'POSITIVE': 0}}),\n(\"assessment martin\",{'cats':{'POSITIVE': 0}}),\n(\"sas system windows\",{'cats':{'POSITIVE': 0}}),\n(\"section ii\",{'cats':{'POSITIVE': 0}}),\n(\"d cnn\",{'cats':{'POSITIVE': 0}}),\n(\"in table\",{'cats':{'POSITIVE': 0}}),\n(\"different adni 1database\",{'cats':{'POSITIVE': 0}}),\n(\"adni 1 adni 2\",{'cats':{'POSITIVE': 0}}),\n(\"results we\",{'cats':{'POSITIVE': 0}}),\n(\"whereas foland ross\",{'cats':{'POSITIVE': 0}}),\n(\"performance iq\",{'cats':{'POSITIVE': 0}}),\n(\"verbal iq\",{'cats':{'POSITIVE': 0}}),\n(\"nc flood risk information system\",{'cats':{'POSITIVE': 0}}),\n(\"nc sea level rise risk management study\",{'cats':{'POSITIVE': 0}}),\n(\"canada\",{'cats':{'POSITIVE': 0}}),\n(\"germany\",{'cats':{'POSITIVE': 0}})\n]\nprint ('training data loaded')","metadata":{"execution":{"iopub.status.busy":"2021-09-25T01:17:48.691284Z","iopub.execute_input":"2021-09-25T01:17:48.691594Z","iopub.status.idle":"2021-09-25T01:17:48.820049Z","shell.execute_reply.started":"2021-09-25T01:17:48.691566Z","shell.execute_reply":"2021-09-25T01:17:48.818581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nlp = spacy.load('en_core_web_sm')\n\nif 'textcat' not in nlp.pipe_names:\n    textcat = nlp.create_pipe(\"textcat\")\n    nlp.add_pipe(textcat, last=True)\nelse:\n    textcat = nlp.get_pipe(\"textcat\")\n\ntextcat.add_label('POSITIVE')\nother_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']\nn_iter = 15\n\nwith nlp.disable_pipes(*other_pipes):\n    optimizer = nlp.begin_training()\n    print(\"Training model...\")\n    \n    for i in range(n_iter):\n        losses = {}\n        batches = minibatch(spacy_train_data, size=compounding(4,32,1.001))\n        for batch in batches:\n            texts, annotations = zip(*batch)\n            nlp.update(texts, annotations, sgd=optimizer,drop=0.2, losses=losses)\n            \nwith open('./spacy_model.pickle', 'wb') as f:\n    pickle.dump(nlp, f)\nprint('done')","metadata":{"execution":{"iopub.status.busy":"2021-09-25T01:17:51.752692Z","iopub.execute_input":"2021-09-25T01:17:51.753982Z","iopub.status.idle":"2021-09-25T01:18:28.197627Z","shell.execute_reply.started":"2021-09-25T01:17:51.753930Z","shell.execute_reply":"2021-09-25T01:18:28.196618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}