{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Coleridge Initiative EDA\n### Analysis of publication titles, including: \n1. Most frequent words in the dataset publication titles of lengths 3-10\n2. Bi, tri and quad-grams \n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\nimport math\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport json\nimport glob\nimport wordcloud\nimport string","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#force lower case and remove punctuation\ntrain.pub_title = train.pub_title.str.lower()\ntrain.pub_title = train.pub_title.str.translate(str.maketrans(\"\", \"\", string.punctuation+'’'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Find common words in the titles"},{"metadata":{"trusted":true},"cell_type":"code","source":"title_words = train.pub_title.str.lower().str.split(expand=True).stack().value_counts()\ntitle_words.sort_values(ascending=False,inplace=True)\ntitle_words = pd.DataFrame(title_words).reset_index()\ntitle_words.columns=['word','count']\ntitle_words","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create the word length datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"title_words['word_length']=title_words.word.str.len()\n\nall_top_words = title_words.iloc[:10]\n\nthree_letters_top_words = title_words[title_words['word_length']==3].iloc[:10]\nfour_letters_top_words = title_words[title_words['word_length']==4].iloc[:10]\nfive_letters_top_words = title_words[title_words['word_length']==5].iloc[:10]\nsix_letters_top_words = title_words[title_words['word_length']==6].iloc[:10]\nseven_letters_top_words = title_words[title_words['word_length']==7].iloc[:10]\neight_letters_top_words = title_words[title_words['word_length']==8].iloc[:10]\nnine_letters_top_words = title_words[title_words['word_length']==9].iloc[:10]\nten_letters_top_words = title_words[title_words['word_length']>=10].iloc[:10]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visuals - top words of n length"},{"metadata":{"trusted":true},"cell_type":"code","source":"palette = sns.color_palette(\"rocket\",10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,8))\nsns.barplot(x='count',y='word',data=all_top_words,palette=palette)\nplt.title('Top 10 Words in Publication Titles',fontsize=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_word_dfs = [three_letters_top_words,four_letters_top_words, five_letters_top_words,\n                six_letters_top_words, seven_letters_top_words, eight_letters_top_words,\n               nine_letters_top_words,ten_letters_top_words]\nn=2\n\nfor i in top_word_dfs:\n    plt.figure(figsize=(10,8))\n    n += 1\n    sns.barplot(x='count',y='word',data=i,palette=palette)\n    title = ('Top 10 Words in Publication Titles, of length ' + str(n))\n    if n==10: title = title + ' or more'\n    plt.title(title,fontsize=20)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Process data for n-grams"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create bi-grams\nbigrams = [a[0] + \"-\" + a[1] for b in train.pub_title for a in zip(b.split(\" \")[:-1], b.split(\" \")[1:])]\nbigrams = pd.DataFrame(bigrams,columns=['bigrams'])\nbigrams = bigrams.value_counts()\nbigrams.sort_values(ascending=False,inplace=True)\nbigrams = pd.DataFrame(bigrams).reset_index()\nbigrams.columns=['bigram','count']\nbigrams.sort_values(by='count',ascending=False,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create tri-grams\ntrigrams = [a[0] + \"-\" + a[1] + \"-\" + a[2] for b in train.pub_title for a in zip(b.split(\" \")[:-1],b.split(\" \")[1:], b.split(\" \")[2:])]\ntrigrams = pd.DataFrame(trigrams,columns=['trigrams'])\ntrigrams = trigrams.value_counts()\ntrigrams.sort_values(ascending=False,inplace=True)\ntrigrams = pd.DataFrame(trigrams).reset_index()\ntrigrams.columns=['trigram','count']\ntrigrams.sort_values(by='count',ascending=False,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create quad-grams\nquadgrams = [a[0] + \"-\" + a[1] + \"-\" + a[2] + \"-\" + a[3] for b in train.pub_title for a in zip(b.split(\" \")[:-1],b.split(\" \")[1:],b.split(\" \")[2:],b.split(\" \")[3:])]\nquadgrams = pd.DataFrame(quadgrams,columns=['quadgrams'])\nquadgrams = quadgrams.value_counts()\nquadgrams.sort_values(ascending=False,inplace=True)\nquadgrams = pd.DataFrame(quadgrams).reset_index()\nquadgrams.columns=['quadgram','count']\nquadgrams.sort_values(by='count',ascending=False,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visuals - top n-grams"},{"metadata":{"trusted":true},"cell_type":"code","source":"palette = sns.color_palette(\"mako\",10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,8))\nsns.barplot(x='count',y='bigram',data=bigrams.iloc[:10],palette=palette)\nplt.title('Top 10 Bi-Grams in Publication Titles',fontsize=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,8))\nsns.barplot(x='count',y='trigram',data=trigrams.iloc[:10],palette=palette)\nplt.title('Top 10 Tri-Grams in Publication Titles',fontsize=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,8))\nsns.barplot(x='count',y='quadgram',data=quadgrams.iloc[:10],palette=palette)\nplt.title('Top 10 Quad-Grams in Publication Titles',fontsize=20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## WIP\n"},{"metadata":{},"cell_type":"markdown","source":"### Please upvote if you enjoyed or want more..."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}