{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Coleridge Initiative - Show US the Data

\n

Let's See the Data🔥

","metadata":{}},{"cell_type":"markdown","source":"![](https://cusp.nyu.edu/wp-content/uploads/2018/09/CI_horizontal.png)","metadata":{}},{"cell_type":"markdown","source":"Import Packages","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport string\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport plotly.graph_objects as go\nimport plotly.express as px\n\nfrom wordcloud import WordCloud\nfrom collections import Counter\nfrom functools import partial\n\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom nltk.corpus import stopwords\nstoplist = stopwords.words('english')\n\npd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = \"../input/coleridgeinitiative-show-us-the-data/train\"\nTEST_DIR = \"../input/coleridgeinitiative-show-us-the-data/test\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load the Dataframe","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\nsample_sub = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Basic Exploration","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of Unique Publication Titles is {train['pub_title'].nunique()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of Unique Dataset Titles is {train['dataset_title'].nunique()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of Unique Dataset Labels is {train['dataset_label'].nunique()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of Unique Cleaned Dataset Labels is {train['cleaned_label'].nunique()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['publication_title_words'] = train['pub_title'].apply(lambda x: len(x.split()))\ntrain['dataset_title_words'] = train['dataset_title'].apply(lambda x: len(x.split()))\ntrain['dataset_label_words'] = train['dataset_label'].apply(lambda x: len(x.split()))\ntrain['cleaned_label_words'] = train['cleaned_label'].apply(lambda x: len(x.split()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's Check the Distribution of words","metadata":{}},{"cell_type":"code","source":"fig = px.histogram(train, x=\"publication_title_words\",\n marginal=\"box\") # or violin, rug)\nfig.update_layout(go.Layout(template= \"plotly_dark\",title = 'Length of Publication Titles' , xaxis = dict(title = 'Length'), yaxis = dict(title = 'Count')))\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(train, x=\"dataset_title_words\",\n marginal=\"box\") # or violin, rug)\nfig.update_layout(go.Layout(template= \"plotly_dark\",title = 'Length of Dataset Titles' , xaxis = dict(title = 'Length'), yaxis = dict(title = 'Count')))\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(train, x=\"dataset_label_words\",\n marginal=\"box\") # or violin, rug)\nfig.update_layout(go.Layout(template= \"plotly_dark\",title = 'Length of Dataset Labels' , xaxis = dict(title = 'Length'), yaxis = dict(title = 'Count')))\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's see the most frequent words","metadata":{}},{"cell_type":"markdown","source":"Publication Title","metadata":{}},{"cell_type":"code","source":"publication_title_list = []\nfor publication_title in train['pub_title'].tolist():\n words = publication_title.split()\n publication_title_list.extend(words)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"publication_title_word_freq = Counter(publication_title_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_word_freq = sorted(publication_title_word_freq.items(), key=lambda pair: pair[1], reverse=True)\nsorted_word_freq[:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It is quite indicative that we have a problem of stopwords here","metadata":{}},{"cell_type":"markdown","source":"Dataset Title","metadata":{}},{"cell_type":"code","source":"dataset_title_list = []\nfor dataset_title in train['dataset_title'].tolist():\n words = dataset_title.split()\n dataset_title_list.extend(words)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_title_word_freq = Counter(dataset_title_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_word_freq = sorted(dataset_title_word_freq.items(), key=lambda pair: pair[1], reverse=True)\nsorted_word_freq[:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset Label","metadata":{}},{"cell_type":"code","source":"dataset_label_list = []\nfor dataset_label in train['dataset_label'].tolist():\n words = dataset_label.split()\n dataset_label_list.extend(words)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_label_word_freq = Counter(dataset_label_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_word_freq = sorted(dataset_label_word_freq.items(), key=lambda pair: pair[1], reverse=True)\nsorted_word_freq[:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's Visualize using WordCloud","metadata":{}},{"cell_type":"markdown","source":"Helper Function","metadata":{}},{"cell_type":"code","source":"# Define a function to plot word cloud\ndef plot_cloud(wordcloud):\n # Set figure size\n plt.figure(figsize=(40, 30))\n # Display image\n plt.imshow(wordcloud) \n # No axis details\n plt.axis(\"off\");","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wordcloud = WordCloud(width = 1000, height = 500, random_state=1, colormap='twilight', \n font_path='../input/all-elon-musks-tweets/acetone_font.otf', collocations=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Publication Title","metadata":{}},{"cell_type":"code","source":"plot_cloud(wordcloud.generate_from_frequencies(publication_title_word_freq))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset Title","metadata":{}},{"cell_type":"code","source":"plot_cloud(wordcloud.generate_from_frequencies(dataset_title_word_freq))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset Label","metadata":{}},{"cell_type":"code","source":"plot_cloud(wordcloud.generate_from_frequencies(dataset_label_word_freq))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most Common Bigrams/Trigrams","metadata":{}},{"cell_type":"markdown","source":"Publication Title","metadata":{}},{"cell_type":"code","source":"c_vec = CountVectorizer(stop_words=stoplist, ngram_range=(2,3))\n# matrix of ngrams\nngrams = c_vec.fit_transform(train['pub_title'])\n# count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis=0)\n# list of ngrams\nvocab = c_vec.vocabulary_\ndf_ngram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse=True)\n ).rename(columns={0: 'frequency', 1:'bigram/trigram'})\n\n\ntop10_ngrams_freq = df_ngram.head(10)['frequency'].tolist()\ntop10_ngrams = df_ngram.head(10)['bigram/trigram'].tolist()\n\nfig = go.Figure(data=[go.Table(header=dict(values=['Bigram/Trigram', 'Count'], fill_color='yellow', line_color='darkslategray'),\n cells=dict(values=[top10_ngrams, top10_ngrams_freq], fill_color='lavender', line_color='darkslategray'))\n ])\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['rgb(160, 50, 168)']*len(top10_ngrams_freq)\n\ntrace = go.Bar(\n x = top10_ngrams_freq[::-1],\n y = top10_ngrams[::-1],\n marker = dict(color = colors,\n line=dict(color='rgb(0,0,0)',width=1.5)),\n text=top10_ngrams_freq[::-1], textposition='outside', orientation='h')\nlayout = go.Layout(template= \"plotly_dark\",title = 'TOP 10 BIGRAMS / TRIGRAMS IN PUBLICATION TITLE' , xaxis = dict(title = 'Count', automargin=True), yaxis = dict(title = 'Bigram/Trigram'))\nfig = go.Figure(data = [trace], layout = layout)\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset Title","metadata":{}},{"cell_type":"code","source":"c_vec = CountVectorizer(stop_words=stoplist, ngram_range=(2,3))\n# matrix of ngrams\nngrams = c_vec.fit_transform(train['dataset_title'])\n# count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis=0)\n# list of ngrams\nvocab = c_vec.vocabulary_\ndf_ngram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse=True)\n ).rename(columns={0: 'frequency', 1:'bigram/trigram'})\n\n\ntop10_ngrams_freq = df_ngram.head(10)['frequency'].tolist()\ntop10_ngrams = df_ngram.head(10)['bigram/trigram'].tolist()\n\nfig = go.Figure(data=[go.Table(header=dict(values=['Bigram/Trigram', 'Count'], fill_color='yellow', line_color='darkslategray'),\n cells=dict(values=[top10_ngrams, top10_ngrams_freq], fill_color='lavender', line_color='darkslategray'))\n ])\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['rgb(160, 50, 168)']*len(top10_ngrams_freq)\n\ntrace = go.Bar(\n x = top10_ngrams_freq[::-1],\n y = top10_ngrams[::-1],\n marker = dict(color = colors,\n line=dict(color='rgb(0,0,0)',width=1.5)),\n text=top10_ngrams_freq[::-1], textposition='outside', orientation='h')\nlayout = go.Layout(template= \"plotly_dark\",title = 'TOP 10 BIGRAMS / TRIGRAMS IN DATASET TITLE' , xaxis = dict(title = 'Count', automargin=True), yaxis = dict(title = 'Bigram/Trigram'))\nfig = go.Figure(data = [trace], layout = layout)\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset Label","metadata":{}},{"cell_type":"code","source":"c_vec = CountVectorizer(stop_words=stoplist, ngram_range=(2,3))\n# matrix of ngrams\nngrams = c_vec.fit_transform(train['dataset_label'])\n# count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis=0)\n# list of ngrams\nvocab = c_vec.vocabulary_\ndf_ngram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse=True)\n ).rename(columns={0: 'frequency', 1:'bigram/trigram'})\n\n\ntop10_ngrams_freq = df_ngram.head(10)['frequency'].tolist()\ntop10_ngrams = df_ngram.head(10)['bigram/trigram'].tolist()\n\nfig = go.Figure(data=[go.Table(header=dict(values=['Bigram/Trigram', 'Count'], fill_color='yellow', line_color='darkslategray'),\n cells=dict(values=[top10_ngrams, top10_ngrams_freq], fill_color='lavender', line_color='darkslategray'))\n ])\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['rgb(160, 50, 168)']*len(top10_ngrams_freq)\n\ntrace = go.Bar(\n x = top10_ngrams_freq[::-1],\n y = top10_ngrams[::-1],\n marker = dict(color = colors,\n line=dict(color='rgb(0,0,0)',width=1.5)),\n text=top10_ngrams_freq[::-1], textposition='outside', orientation='h')\nlayout = go.Layout(template= \"plotly_dark\",title = 'TOP 10 BIGRAMS / TRIGRAMS IN DATASET LABEL' , xaxis = dict(title = 'Count', automargin=True), yaxis = dict(title = 'Bigram/Trigram'))\nfig = go.Figure(data = [trace], layout = layout)\nfig.show()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Baseline Submission","metadata":{}},{"cell_type":"markdown","source":"Code taken from [https://www.kaggle.com/prashansdixit/coleridge-initiative-eda-baseline-model](https://www.kaggle.com/prashansdixit/coleridge-initiative-eda-baseline-model)","metadata":{}},{"cell_type":"markdown","source":"Helper Functions","metadata":{}},{"cell_type":"code","source":"def clean_text(txt):\n return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n '''\n Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n text - Sentence that needs to be cleaned\n '''\n text = ''.join([k for k in text if k not in string.punctuation])\n text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n \n return text","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=TRAIN_DIR, output='text'):\n \"\"\"\n Function to read json file and then return the text data from them and append to the dataframe\n \"\"\"\n json_path = os.path.join(train_files_path, (filename+'.json'))\n headings = []\n contents = []\n combined = []\n with open(json_path, 'r') as f:\n json_decode = json.load(f)\n for data in json_decode:\n headings.append(data.get('section_title'))\n contents.append(data.get('text'))\n combined.append(data.get('section_title'))\n combined.append(data.get('text'))\n \n all_headings = ' '.join(headings)\n all_contents = ' '.join(contents)\n all_data = '. '.join(combined)\n \n if output == 'text':\n return all_contents\n elif output == 'head':\n return all_headings\n else:\n return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['text'] = train['Id'].apply(read_append_return)\nsample_sub['text'] = sample_sub['Id'].apply(partial(read_append_return, train_files_path=TEST_DIR))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlables_list = []\nfor index, row in sample_sub.iterrows():\n sample_text = row['text']\n row_id = row['Id']\n temp_df = train[train['text'] == text_cleaning(sample_text)]\n cleaned_labels = temp_df['cleaned_label'].to_list()\n for known_label in existing_labels:\n if known_label in sample_text.lower():\n cleaned_labels.append(clean_text(known_label))\n cleaned_labels = [clean_text(x) for x in cleaned_labels]\n cleaned_labels = set(cleaned_labels)\n lables_list.append('|'.join(cleaned_labels))\n id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = id_list\nsubmission['PredictionString'] = lables_list\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![Upvote!](https://img.shields.io/badge/Upvote-If%20you%20like%20my%20work-07b3c8?style=for-the-badge&logo=kaggle)","metadata":{}}]}