{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport json\nimport re\nimport nltk\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom wordcloud import WordCloud, STOPWORDS\nfrom spacy.lang.en import English\nfrom collections import defaultdict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"EDA on the datalabels\n\nhttps://www.kaggle.com/narendra/datalabel-eda"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.display.max_rows=100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nlp=English()\nwordcloud=WordCloud(stopwords=STOPWORDS, \n                    width=600,\n                    height=300,\n                    background_color='white',\n                    #max_font_size=50,\n                    #max_words=100\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\npub_df=pd.read_csv('../input/publication-datasets/publication_dataset.csv')\npub_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Positive Samples-sentences on which the dataset labels match.\n\nNegative Samples-sentences on which the dataset labels did not match."},{"metadata":{},"cell_type":"markdown","source":"# lets see how many positive samples are there in publications"},{"metadata":{"trusted":true},"cell_type":"code","source":"pub_df.num_positive_samples.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"90% of the publications had {} sentences which mentions the datalabels\".format(pub_df.num_positive_samples.quantile(q=0.9)))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.xticks(rotation=45)\nsns.countplot(data=pub_df, x='num_positive_samples')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# let us see the the ratio of the postive to negative samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"pub_df['pos_neg_ratio']=pub_df.num_positive_samples.div(pub_df.num_negative_samples)\npub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pub_df[pub_df.num_negative_samples==0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"one publication had all the sentences as positive samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"pub_df[pub_df.num_negative_samples>0].pos_neg_ratio.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In most cases number of positive sentences is <1% compared to the negative sentences."},{"metadata":{"trusted":true},"cell_type":"code","source":"pub_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now that we had the positive samples; lets us see the words that appear in the context of the positive samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_sentences=[]\nfor sentences in pub_df.sentences.values:\n    sentences=eval(sentences)\n    pos_sents=sentences['pos_sents']\n    pos_sentences+=pos_sents\n    \nprint(len(pos_sentences))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nall_sentence_text=''\nfor sent in pos_sentences:\n    all_sentence_text+=sent.lower()+\" \"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wordcloud_image=wordcloud.generate(all_sentence_text)\nplt.imshow(wordcloud_image)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_sent_tokens(sent):\n    doc=nlp(sent)\n    tokens=[]\n    for token in doc:\n        if token.is_stop or token.is_punct or token.is_digit:\n            continue\n        token=token.lower_.strip()\n        if len(token)<=2:\n            continue\n        tokens.append(token)\n    return tokens","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_sent_df=pd.DataFrame.from_dict({'sentence': pos_sentences})\npos_sent_df['tokens']=pos_sent_df.sentence.apply(get_sent_tokens)\n\npos_sent_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Unigrams\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"unigrams=defaultdict(int)\nfor tokens in pos_sent_df.tokens:\n    for token in tokens:\n        unigrams[token]+=1\nunigram_df=pd.DataFrame.from_dict({\n    'word': list(unigrams.keys()),\n    'freq': list(unigrams.values())\n})\n\nunigram_df=unigram_df.sort_values('freq', ascending=False)\nunigram_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number Of Unigrams:\", len(unigrams))\nprint(\"Number Of Unigrams >2 freq:\", unigram_df[unigram_df.freq>2].shape[0])\nprint(\"Number Of Unigrams >10 freq:\", unigram_df[unigram_df.freq>10].shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unigram_df[unigram_df.freq>5].freq.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(unigram_df[unigram_df.freq>5].freq, bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# word freq > 10"},{"metadata":{"trusted":true},"cell_type":"code","source":"unigram_df=unigram_df[unigram_df.freq>10].copy()\nunigram_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unigram_df.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unigram_df.tail(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The above unigrams represents the topics generated in the sentences.\nbut seems noisy when considering the context.\n\nTo get the context lets mask the document labels from the sentences and the get the context words in the window."},{"metadata":{},"cell_type":"markdown","source":"# get only the context word around the datalabel(window size=5)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_context_tokens(sentence, direction, w=4):\n    sentence=sentence.strip()\n    tokens=[]\n    for token in nlp(sentence):\n        if not token.is_alpha:\n            continue\n        tokens.append(token.text)\n    if direction == -1:\n        return tokens[-w:]\n    return tokens[:w]\n\n\ndef get_context(row):\n    dataset_labels=eval(row['dataset_label'])\n    sentences=eval(row['sentences'])['pos_sents']\n    \n    context={\n        'left': [],\n        'right': []\n    }\n    for sentence in sentences:\n        for dl in dataset_labels:\n            for match in re.finditer(dl, sentence):\n                start=match.start()\n                end=match.end()\n                \n                left_sentence=sentence[:start]\n                right_sentence=sentence[end:]\n                \n                left_context=get_context_tokens(left_sentence, -1)\n                right_context=get_context_tokens(right_sentence, 1)\n                \n                context['left'].append(left_context)\n                context['right'].append(right_context)\n    return context","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pub_df['context']=pub_df.apply(get_context, axis=1)\ncontext_df=pd.DataFrame()\ncontext_df['left_context']=pub_df['context'].apply(lambda context: context['left'])\ncontext_df['right_context']=pub_df['context'].apply(lambda context: context['right'])\n\npub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ncontext_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"context_words=defaultdict(int)\nleft_context_words=defaultdict(int)\nright_context_words=defaultdict(int)\n\nall_context_text=''\nleft_context_text=''\nright_context_text=''\n\nfor left_context in context_df.left_context.values:\n    for words in left_context:\n        for word in words:\n            word=word.lower()\n            context_words[word]+=1\n            left_context_words[word]+=1\n            left_context_text+=word+\" \"\n            all_context_text+=word+\" \"\n\nfor right_context in context_df.right_context.values:\n    for words in right_context:\n        for word in words:\n            word=word.lower()\n            context_words[word]+=1\n            right_context_words[word]+=1\n            right_context_text+=word+\" \"\n            all_context_text+=word+\" \"\n\nprint(\"Number Of Context Words\", len(context_words))\nprint('Number Of Left Context Words:', len(left_context_words))\nprint('Number Of Right Context Words:', len(right_context_words))\n\ncontext_word_df=pd.DataFrame.from_dict({\n    'word': list(context_words.keys()),\n    'freq': list(context_words.values())\n})\n\nleft_context_word_df=pd.DataFrame.from_dict({\n    'word': list(left_context_words.keys()),\n    'freq': list(left_context_words.values())\n})\n\nright_context_word_df=pd.DataFrame.from_dict({\n    'word': list(right_context_words.keys()),\n    'freq': list(right_context_words.values())\n})\n\ncontext_word_df=context_word_df.sort_values('freq', ascending=False)\nleft_context_word_df=left_context_word_df.sort_values('freq', ascending=False)\nright_context_word_df=right_context_word_df.sort_values('freq', ascending=False)\ncontext_word_df.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"looks like the top unigrams have words related mostly the prepositions, and related areas of interst\n\nlets remove more frequent and less frequent unigrams\n>10 and <800"},{"metadata":{"trusted":true},"cell_type":"code","source":"context_word_df=context_word_df[(context_word_df.freq>10) & (context_word_df.freq<800)].copy()\ncontext_word_df.freq.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"context_word_df.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_context_wc=wordcloud.generate(all_context_text)\n\nplt.figure(figsize=(15, 4))\nplt.title('All Context')\nplt.imshow(all_context_wc)\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the above word cloud observed that most words are releavant to the seach of datasets but dominated by some of the words that appear most in publications\n\nDominated Words from publications:\n1. alzheimer\n2. education\n3. neuroimaging\n\nContext words\n1. dataset\n2. sample\n3. et al\n4. cohert\n5. database\n6. obtained, study, using etc."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_context_bigrams(context):\n    ctx_bigrams=[]\n    for ctx_list in context:\n        if len(ctx_list)<=1:\n            continue\n        for bg in nltk.bigrams(ctx_list):\n            ctx_bigrams.append( ' '.join(bg).lower() )\n    return ctx_bigrams","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"context_df['left_bigrams']=context_df.left_context.apply(get_context_bigrams)\ncontext_df['right_bigrams']=context_df.right_context.apply(get_context_bigrams)\ncontext_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"context_bigrams=defaultdict(int)\n\nfor bgs in context_df.left_bigrams.values:\n    for bg in bgs:\n        context_bigrams[bg]+=1\n\nfor bgs in context_df.right_bigrams.values:\n    for bg in bgs:\n        context_bigrams[bg]+=1\n\ncontext_bigrams_df=pd.DataFrame.from_dict({\n    'bigram': list(context_bigrams.keys()),\n    'freq': list(context_bigrams.values())\n})\n\ncontext_bigrams_df=context_bigrams_df.sort_values('freq', ascending=False)\nprint('Number Of Context Bigrams:', len(context_bigrams))\ncontext_bigrams_df.head(30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"context_bigrams_df[context_bigrams_df.bigram.apply(lambda x: 'sample' in x)].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"context_bigrams_df[context_bigrams_df.bigram.apply(lambda x: 'taken' in x)].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"taking a look at some common bigrams\n\n1. taken from (may be from left)\n2. were taken (may be from right)\n3. sampled from \n4. from the\n5. on the.\n6. et al, etc..."},{"metadata":{},"cell_type":"markdown","source":"we can see from the above that, we need to focus on the terms/phrases that are specific to datasets like\nextraction, sampling etc.\n\nwhere as some of the phrases are highly specific to the content of the publication."},{"metadata":{},"cell_type":"markdown","source":"By combining the Context-level features and lexical-level(word shapes, cap-letters etc.) we can understand get the candidate phrases of the dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}