{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ndf['title'] = df['pub_title']","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Preliminary Analysis**","metadata":{}},{"cell_type":"markdown","source":"**Number of characters present in the publication titles.**","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfig = sns.displot(x=df.title.str.len(), data=df, color='black', kde=False, height=6, aspect=3, kind='hist')\n\nprint(df.title.str.len().min())\nprint(df.title.str.len().max())\nprint(df.title.str.len().mean())","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the length of publications titles range 8 to 560 characters. On average, the publication title length is 96 characters.\n\n**Number of words in publication titles**","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\ntemp = df.title.str.split().map(lambda x: len(x))\n\nfig = sns.displot(x=temp, color='blue', kde=False, height=6, aspect=3, kind='hist')\n\nprint(temp.min())\nprint(temp.max())\nprint(temp.mean())","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the number of words in the publications titles range from 1 to 84. On average, we have 12 words in a publication title.\n\n**Word length in publication titles**","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\n\ntemp = df.title.str.split().apply(lambda x: [len(i) for i in x]).map(lambda x: np.mean(x))\n\nfig = sns.displot(x=temp, color='red', kde=False, height=6, aspect=2, kind='hist')\n\nprint(temp.min())\nprint(temp.max())\nprint(temp.mean())","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the number of characters in the words of publication titles vary from 2 to 16. On average the number of characters is 7.\n\n**Let's see the stopwords in the titles now.**","metadata":{}},{"cell_type":"code","source":"import nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\n\nstop = set(stopwords.words('english'))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus = []\ntitle = df.title.str.split()\ntitle = title.values.tolist()\ncorpus = [word for i in title for word in i]\n\nfrom collections import defaultdict\n\ndic = defaultdict(int)\n\nfor word in corpus:\n    if word in stop:\n        dic[word] += 1","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_dic = list(reversed(sorted(list(dic.items()), key=lambda x: x[1])))\n\nkeys = [i[0] for i in sorted_dic[:10]]\nvalues = [i[1] for i in sorted_dic[:10]]\n\nsns.set(rc={'figure.figsize':(10,10)})\n\nfig = sns.barplot(x=keys, y=values, palette='colorblind')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the top 10 stopwords used in the titles.\n\n**Most occuring words**","metadata":{}},{"cell_type":"code","source":"from collections import Counter\nfrom nltk.stem import PorterStemmer\n\nsns.set(rc={'figure.figsize':(15,15)})\n\nps = PorterStemmer()\ncounter = Counter(corpus)\nmost = counter.most_common()\n\nx, y = [], []\nlookup = []\nfor word,count in most[:120]:\n    if (word.lower() not in stop) and (ps.stem(word.lower()) not in lookup) and word.isalpha():\n        x.append(word)\n        y.append(count)\n        lookup.append(ps.stem(word.lower()))\n        \nsns.barplot(x=y,y=x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the most common words. From the words, we can infer that the dataset has primarily \npublications dealing with diseases such as Alzheimer and dementia. ","metadata":{}},{"cell_type":"markdown","source":"# **N-Gram Exploration**","metadata":{}},{"cell_type":"markdown","source":"**Most common bigrams**","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n\ndef get_top_ngram(corpus, n=None):\n    vec = CountVectorizer(ngram_range=(n, n)).fit(corpus)\n    bag_of_words = vec.transform(corpus)\n    sum_words = bag_of_words.sum(axis=0) \n    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]\n    fwords_freq = []\n    for i in words_freq:\n        temp = 0\n        for j in i[0].split():\n            if j in stop:\n                temp += 1\n        if temp != len(i[0].split()):\n            fwords_freq.append(i)\n    words_freq = fwords_freq\n    words_freq =sorted(words_freq, key=lambda x: x[1], reverse=True)\n    return words_freq[:100]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_n_bigrams = get_top_ngram(df.title, 2)[:20]\n\nx, y = map(list, zip(*top_n_bigrams)) \n\nsns.barplot(x=y, y=x, palette='hls')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looking at the top 20 bigrams, we can see the publications being focused on Alzheimer and related cognitive impairment in older adults. Also, we see the Covid 19 publications fighting against the present pandemic.\n\n**Most common trigrams**","metadata":{}},{"cell_type":"code","source":"top_n_trigrams = get_top_ngram(df.title, 3)[:20]\n\nx, y = map(list, zip(*top_n_trigrams)) \n\nsns.barplot(x=y, y=x, palette='coolwarm')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**We see the top 20 trigrams appearing publications.We observe that the publications are focused on Alzheimer disease and its effects in USA.**","metadata":{}},{"cell_type":"markdown","source":"# Topic Modelling","metadata":{}},{"cell_type":"code","source":"import nltk\nnltk.download('punkt')\nnltk.download('wordnet')\nfrom nltk.stem.wordnet import WordNetLemmatizer\nfrom nltk import word_tokenize\n\ndef preprocess_news(df):\n    corpus = []\n    stem = PorterStemmer()\n    lem = WordNetLemmatizer()\n    for news in df.title:\n        words = [w for w in word_tokenize(news) if (w.lower() not in stop and w.isalpha())]\n        words = [lem.lemmatize(w) for w in words if len(w) > 2]\n        corpus.append(words)\n    return corpus\n\ncorpus = preprocess_news(df)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gensim\n\ndic = gensim.corpora.Dictionary(corpus)\nbow_corpus = [dic.doc2bow(doc) for doc in corpus]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lda_model = gensim.models.LdaMulticore(bow_corpus, \n                                   num_topics = 5, \n                                   id2word = dic,                                    \n                                   passes = 10,\n                                   workers = 2)\nlda_model.show_topics()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pyLDAvis\nimport pyLDAvis.gensim_models\n\nLDAvis_prepared = pyLDAvis.gensim_models.prepare(lda_model, bow_corpus, dic)\npyLDAvis.display(LDAvis_prepared)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the topic of Alzheimer being the most occuring topic in the publications. But, we have a new entry of publications focused on Agriculture in topic 4.","metadata":{}},{"cell_type":"markdown","source":"# Wordcloud Analysis","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nstopwords = set(STOPWORDS)\n\ndef show_wordcloud(data):\n    wordcloud = WordCloud(\n        background_color=None,\n        stopwords=stopwords,\n        max_words=1000,\n        max_font_size=30,\n        scale=4,\n        random_state=42,\n        mode='RGBA',\n        colormap='plasma')\n   \n    wordcloud=wordcloud.generate(str(data))\n\n    fig = plt.figure(1, figsize=(15, 15))\n    plt.axis('off')\n\n    plt.imshow(wordcloud)\n    plt.show()\n\nshow_wordcloud(corpus)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see Alzheimer as the focus of most publications, followed by dementia and aging.","metadata":{}},{"cell_type":"markdown","source":"# Sentiment Analysis","metadata":{}},{"cell_type":"markdown","source":"**We'll first see the polarity of the publication titles.**","metadata":{}},{"cell_type":"code","source":"from textblob import TextBlob\n\nsns.set(rc={'figure.figsize':(6, 6)})\n\ndef polarity(text):\n    return TextBlob(text).sentiment.polarity\n\ndf.polarity_score = df.title.apply(lambda x : polarity(x))\ndf.polarity_score.hist(color='skyblue')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that majority of the publications have a neutral polarity.","metadata":{}},{"cell_type":"code","source":"def sentiment(x):\n    if x < 0:\n        return 'neg'\n    elif x == 0:\n        return 'neu'\n    else:\n        return 'pos'\n\nsns.set(rc={'figure.figsize':(6, 6)})\ndf.sentiment = df.polarity_score.map(lambda x: sentiment(x))\n\nsns.barplot(x=df.sentiment.value_counts().index, y=df.sentiment.value_counts(), palette='coolwarm')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the most prevalent publication titles are neutral followed by positive and then, negative.","metadata":{}},{"cell_type":"markdown","source":"# NER Analysis","metadata":{}},{"cell_type":"code","source":"! python -m spacy download en_core_web_sm","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\n\nnlp = spacy.load(\"en_core_web_sm\")","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ner(text):\n    doc = nlp(text)\n    return [X.label_ for X in doc.ents]\n\nent = df.title.apply(lambda x : ner(x))\nent = [x for sub in ent for x in sub]\ncounter = Counter(ent)\ncount = counter.most_common()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, y = map(list, zip(*count))\nsns.set(rc={'figure.figsize':(15, 15)})\nsns.barplot(x=y, y=x, palette='husl')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that ORG, GPE and DATE entities dominate the tally. Let's analyze them.\n\n**We'll first see ORG entities.**","metadata":{}},{"cell_type":"code","source":"def ner(text, ent=\"ORG\"):\n    doc = nlp(text)\n    return [X.text for X in doc.ents if X.label_ == ent]\n\norg = df.title.parallel_apply(lambda x: ner(x))\norg = [i for x in org for i in x]\ncounter = Counter(org)\n\nx, y = map(list,zip(*counter.most_common(20)))\nsns.barplot(y, x, palette='coolwarm')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see various organizations (National Center for Educational Statistics) and names of studies (Baltimore Longitudnal Study of Aging) popping up here. We also get various abbreviations here which may not be organizations such as STEM. ","metadata":{}},{"cell_type":"code","source":"! pip install pandarallel","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandarallel import pandarallel\n\npandarallel.initialize()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ner(text, ent=\"GPE\"):\n    doc = nlp(text)\n    return [X.text for X in doc.ents if X.label_ == ent]\n\nperson = df.title.apply(lambda x: ner(x))\nperson = [i for x in person for i in x]\ncounter = Counter(person)\n\nx,y=map(list,zip(*counter.most_common(20)))\nsns.barplot(y, x, palette='viridis')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see various countries and places mentioned. It gives us an insight into geographical distribution \\\nof the data. We see the most publications coming form US followed by China, India, South Korea, South Africa, Japan and Canada.","metadata":{}},{"cell_type":"code","source":"def ner(text, ent=\"DATE\"):\n    doc = nlp(text)\n    return [X.text for X in doc.ents if X.label_ == ent]\n\nperson = df.title.apply(lambda x: ner(x))\nperson = [i for x in person for i in x]\ncounter = Counter(person)\n\nx,y=map(list,zip(*counter.most_common(20)))\nsns.barplot(y, x, palette='viridis')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see '1988' at the top of the tally. Why? Because of National Education Longitudinal Study of 1988 on which many publications are based. Followed by it are papers published in the time 1992 - 2020. We see a huge volume of the papers published in years 2009 - 2020.","metadata":{}},{"cell_type":"markdown","source":"# POS Tagging\n\n**We'll now do Part-of-Speech Tagging.**\n\nHere's the list of tags:\n\nNoun (NN)- Joseph, London, table, cat, teacher, pen, city\n\nVerb (VB)- read, speak, run, eat, play, live, walk, have, like, are, is\n\nAdjective(JJ)- beautiful, happy, sad, young, fun, three\n\nAdverb(RB)- slowly, quietly, very, always, never, too, well, tomorrow\n\nPreposition (IN)- at, on, in, from, with, near, between, about, under\n\nConjunction (CC)- and, or, but, because, so, yet, unless, since, if\n\nPronoun(PRP)- I, you, we, they, he, she, it, me, us, them, him, her, this\n\nInterjection (INT)- Ouch! Wow! Great! Help! Oh! Hey! Hi!","metadata":{}},{"cell_type":"code","source":"def pos(text):\n    pos = nltk.pos_tag(word_tokenize(text))\n    pos = list(map(list,zip(*pos)))[1]\n    return pos\n\ntags = df.title.parallel_apply(lambda x : pos(x))\ntags = [x for l in tags for x in l]\ncounter = Counter(tags)\n\nx, y = list(map(list,zip(*counter.most_common(6))))\nsns.barplot(x=y, y=x, palette='coolwarm')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see proper nouns topping the list followed by nouns and prepositions.\n\n**Let's see the most prevalent plural nouns used.**","metadata":{}},{"cell_type":"code","source":"def get_nouns(text):\n    noun = []\n    pos = nltk.pos_tag(word_tokenize(text))\n    for word, tag in pos:\n        if tag == 'NNP' and word.isalpha():\n            noun.append(word)\n    return noun\n\nwords = df.title.parallel_apply(lambda x : get_nouns(x))\nwords = [x for l in words for x in l]\ncounter = Counter(words)\n\nx, y = list(map(list,zip(*counter.most_common(10))))\nsns.barplot(x=y, y=x, palette='magma')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We again see Alzheilmer topping the charts. Followed by it are its related words like Disease,Brain and Cognitive. \n\n**Let's the top nouns used.**","metadata":{}},{"cell_type":"code","source":"def get_nouns(text):\n    noun = []\n    pos = nltk.pos_tag(word_tokenize(text))\n    for word, tag in pos:\n        if tag == 'NN' and word.isalpha():\n            noun.append(word)\n    return noun\n\nwords = df.title.parallel_apply(lambda x : get_nouns(x))\nwords = [x for l in words for x in l]\ncounter = Counter(words)\n\nx, y = list(map(list,zip(*counter.most_common(10))))\nsns.barplot(x=y, y=x, palette='Accent')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the disease topping the chart followed by brain. The words revolve around the theme of brain and its impairment. For that purpose, many studies and analysis is done, which we see in the list.","metadata":{}},{"cell_type":"markdown","source":"# Text Complexity","metadata":{}},{"cell_type":"code","source":"! pip install textstat","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from textstat import flesch_reading_ease\n\ndf.title.parallel_apply(lambda x : flesch_reading_ease(x)).hist(color='black')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that the readibility score mostly falls after 50, which means most of the publication titles can be read easily.","metadata":{}},{"cell_type":"markdown","source":"**Let's also see publication titles with readibility score less than 5.**","metadata":{}},{"cell_type":"code","source":"df['reading'] = df.title.parallel_apply(lambda x : flesch_reading_ease(x))\n\ncnt = 0\nfor i in df[df.reading < 5].title:\n    print(i)\n    print()\n    cnt += 1\n    if cnt == 10:\n        break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that short publication titles have low readibility score. Titles which have less used words such as disidentification, postsecondary etc. have less readiblity score.","metadata":{}}]}