{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1 style='color:white; background:blue; border:0'><center>Show US the Data: Start Our Study</center></h1>\n\n![](https://oerc.osu.edu/sites/oerc/themes/oerc/images/projects/coleridge.png)\n\nThis competition challenges data scientists to show how publicly funded data and evidence are used to serve science and society. Data, evidence, and science are critical if government is address the many threats facing society: pandemics, climate change and coastal inundation, Alzheimer’s disease, child hunger, and support science and innovation, increase food production, maintain biodiversity, and address many other challenges. Yet much of the information about data necessary to inform evidence and science is locked inside publications.\n\nCan natural language processing find the hidden-in-plain-sight data citations? Can machine learning find the link between the words used in research articles and the data referenced in the article?","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Coleridge Initiative - String Literals¶\nThe is a fairly naive approach to solving this problem.\n\n* loop over train.csv\n    * perform basic string cleaning (lowercase + remove non-alphanumeric)\n    * create a lookup table for each possible description string (pub_title, dataset_title, dataset_label)\n    * map it back to the expected cleaned_label string\n* brute force search the test dataset for any string literals found in train.csv\n    * if multiple matches are found, then pick the one with the most matches","metadata":{}},{"cell_type":"code","source":"import simplejson\nimport re\nimport pydash\nimport sys\nimport os\nfrom collections import defaultdict\nfrom typing import *\nfrom joblib import Parallel, delayed\nfrom glob import glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\n\nimport tqdm\nfrom tqdm.auto import tqdm as tqdmp\ntqdmp.pandas()\n\n# NLP\nimport unicodedata, string, re, nltk, json\nfrom nltk.corpus import stopwords\nfrom nltk.util import ngrams\nfrom wordcloud import WordCloud, STOPWORDS\nfrom collections import Counter, defaultdict\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport matplotlib.dates as mdates\n\n# ignoring warnings\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\n# Some settings for visualizations\nsns.set(rc={'axes.facecolor':'black', 'figure.facecolor':'black', \n            'xtick.color': 'white', 'ytick.color': 'white', \n            'grid.color': 'white', 'axes.labelcolor': 'white',\n            'figure.dpi': 150, 'grid.linestyle': ':', 'grid.alpha': .6,\n            'font.family': 'fantasy'})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nss = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Note that ALL ground truth texts have been cleaned for matching purposes using the following code:\ndef text_cleaner(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef show_wordcloud(data, stop, mask = None, title = None, color = 'black'):\n    \"\"\"\n    Function for creating wordclouds (with or without mask)\n    \"\"\"\n    from wordcloud import WordCloud, ImageColorGenerator\n    wordcloud = WordCloud(background_color = color,\n                         stopwords = stop,\n                         mask = mask,\n                         max_words = 100,\n                         scale = 3,\n                         width = 4000, \n                         height = 2000,\n                         collocations = False,\n                         colormap = 'rainbow',\n                         random_state = 1)\n    \n    wordcloud = wordcloud.generate(data)\n    \n    plt.figure(1, figsize = (16, 8), dpi = 300)\n    plt.title(title, size = 15)\n    plt.axis('off')\n    if mask is None:\n        plt.imshow(wordcloud, interpolation = \"bilinear\")\n        plt.show()\n    else:\n        image_colors = ImageColorGenerator(mask)\n        plt.imshow(wordcloud.recolor(color_func = image_colors), \n                   interpolation = \"bilinear\")\n        plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Let's at first analyze the train pub_title data.","metadata":{}},{"cell_type":"code","source":"title_cleaned = train['pub_title'].apply(lambda x: text_cleaner(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title_length = title_cleaned.str.len()\n\nplt.title('Title length', size = 15, color = 'white')\nsns.distplot(title_length, kde = False, color = 'blue', \n             hist_kws = dict(alpha = 1))\nplt.xlabel('Length of title (symbols)')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title_words = title_cleaned.str.split().map(lambda x: len(x))\n\nplt.title('Title words', size = 15, color = 'white')\nsns.distplot(title_words, kde = False, color = 'blue', \n             hist_kws = dict(alpha = 1))\nplt.xlabel('Length of title (words)')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title_word_len = title_cleaned.str.split().apply(lambda x: [len(i) for i in x]).map(lambda x: np.mean(x))\n\nplt.title('Title words length', size = 15, color = 'white')\nsns.distplot(title_word_len, kde = False, color = 'blue', \n             hist_kws = dict(alpha = 1))\nplt.xlabel('Mean word length in title (symbols)')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words = title_cleaned.str.split().values.tolist()\ntitle_corpus = [word for i in words for word in i]\n\ntitle_counter = Counter(title_corpus)\ntitle_most = title_counter.most_common()\n\nstop = set(stopwords.words('english'))\n\ntitle_top_words, title_top_words_count = [], []\nfor word, count in title_most[:100]:\n    if word not in stop:\n        title_top_words.append(word)\n        title_top_words_count.append(count)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('TOP-10 title words', color = 'white', size = 15)\nsns.barplot(y = title_top_words[:10], x = title_top_words_count[:10], \n            edgecolor = 'black', color = 'blue')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"title_word_string = ' '.join(title_corpus)\nshow_wordcloud(title_word_string, stop)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Extract all publication texts\n\nPublications are provided in JSON format, broken up into sections with section titles. Let's extract all the texts with a little function.","metadata":{}},{"cell_type":"code","source":"def text_extractor(url_id):\n    url = '../input/coleridgeinitiative-show-us-the-data/train/{}.json'.format(url_id)\n    return ' '.join(pd.read_json(url).text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that there are multiple rows for some training documents, indicating multiple mentioned datasets","metadata":{}},{"cell_type":"code","source":"train_texts = train.drop_duplicates(subset = ['Id'])['Id'].progress_apply(lambda x: text_extractor(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_texts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_cleaned = train_texts.progress_apply(lambda x: text_cleaner(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_cleaned","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_length = text_cleaned.str.len()\n\nplt.title('Pub text length', size = 15, color = 'white')\nsns.distplot(text_length, kde = False, color = 'blue', \n             hist_kws = dict(alpha = 1))\nplt.xlabel('Length of pub text (symbols)')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_words = text_cleaned.str.split().map(lambda x: len(x))\n\nplt.title('Pub text words', size = 15, color = 'white')\nsns.distplot(text_words, kde = False, color = 'blue', \n             hist_kws = dict(alpha = 1))\nplt.xlabel('Length of pub text (words)')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_word_len = text_cleaned.str.split().progress_apply(lambda x: [len(i) for i in x]).map(lambda x: np.mean(x))\n\nplt.title('Pub text words length', size = 15, color = 'white')\nsns.distplot(text_word_len, kde = False, color = 'blue', \n             hist_kws = dict(alpha = 1))\nplt.xlabel('Mean word length in pub text (symbols)')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words = text_cleaned.str.split().values.tolist()\ntext_corpus = [word for i in words for word in i]\n\ntext_counter = Counter(title_corpus)\ntext_most = title_counter.most_common()\n\nstop = set(stopwords.words('english'))\n\ntext_top_words, text_top_words_count = [], []\nfor word, count in text_most[:100]:\n    if word not in stop:\n        text_top_words.append(word)\n        text_top_words_count.append(count)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('TOP-20 text words', color = 'white', size = 15)\nsns.barplot(y = text_top_words[:20], x = text_top_words_count[:20], \n            edgecolor = 'black', color = 'blue')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The goal in this competition is not just to match known dataset strings but to generalize to datasets that have never been seen before using NLP and statistical techniques. Not all datasets have been identified in train, but you have been provided enough information to generalize.\n\nThe objective of the competition is to identify the mention of datasets within scientific publications. Your predictions will be short excerpts from the publications that appear to note a dataset.\n\nSubmissions are evaluated on a Jaccard-based FBeta score between predicted texts and ground truth texts, with Beta = 0 (an F0 or precision score). Multiple predictions are delineated with a pipe (|) character in the submission file.\n\nThe following is Python reference code for the Jaccard score:","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extract String Literals\nLets create a lookup table for all possible strings used to describe each dataset","metadata":{}},{"cell_type":"code","source":"def clean_text(text: str) -> str:               return re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\ndef clean_texts(texts: List[str]) -> List[str]: return [ clean_text(text) for text in texts ] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_lookup(df):\n    lookup = defaultdict(set)\n    for _, row in df.iterrows():\n        lookup[ row['cleaned_label'] ] |= set(clean_texts([ row['dataset_label'], row['dataset_title'], row['pub_title'] ]))\n    return lookup\n\nnext(iter(generate_lookup(train_df).items()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Dataset Validation\nThis validates that this algoritm works on the training dataset, and produces a 100% score","metadata":{}},{"cell_type":"code","source":"def read_json(index: str, test_train=\"test\") -> Dict:\n    filename = f\"../input/coleridgeinitiative-show-us-the-data/{test_train}/{index}.json\"\n    with open(filename) as f:\n        json = simplejson.load(f)\n    return json\n        \ndef json2text(index: str, test_train=\"test\") -> str:\n    json  = read_json(index, test_train)\n    texts = [\n        row[\"section_title\"] + \" \" + row[\"text\"] \n        for row in json\n    ]\n    texts = clean_texts(texts)\n    text  = \" \".join(texts)\n    return text\n\n\ndef extract_label(text: str, lookup: Dict[str, Set[str]]) -> str:\n    labels = []\n    for label, values in lookup.items():\n        if any([\n            value in text\n            for value in values\n        ]):\n            labels += [ label.strip() ]\n            \n    # label = \"|\".join(set(labels))  # multi label support\n    label = Counter(labels).most_common(1)[0][0] if len(labels) else \"\"  # single most-popular label\n    # print('extract_label', labels, '->', label)\n    return label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef train_accuracy(df, limit=sys.maxsize) -> float:\n    limit   = 100 if os.environ.get('KAGGLE_KERNEL_RUN_TYPE', 'Localhost') == 'Interactive' else limit\n    lookup  = generate_lookup(df)\n    labels  = Parallel(-1)(\n        delayed(extract_label)(json2text(index, \"train\"), lookup)\n        for index in df['Id'][:limit]\n    )\n    correct   = 0\n    expecteds = df['cleaned_label'][:limit]\n    for label, expected in zip(labels, expecteds):\n        expected_set = set(expected.split(\"|\"))\n        label_set    = set(label.split(\"|\"))\n        matches      = expected_set & label_set\n        correct     += len(matches) / len(label_set)\n\n    # correct = np.count_nonzero( np.array(labels) == expecteds )\n    total   = len(expecteds)\n    return correct / total\n\ntrain_accuracy(train_df, 100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"def generate_submission():\n    submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)\n    lookup  = generate_lookup(train_df)\n    indexes = submission_df.index\n    labels  = Parallel(-1)(\n        delayed(extract_label)(json2text(index, \"test\"), lookup)\n        for index in indexes\n    )\n    submission_df['PredictionString'] = labels\n    return submission_df\n\nsubmission_df = generate_submission()\nsubmission_df.to_csv('submission.csv')\n!head submission.csv\nsubmission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}