{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Coleridge Initiative - Show US the Data","metadata":{}},{"cell_type":"markdown","source":"* The objective of the competition is to identify the mention of datasets within scientific publications.\n* Predictions that more accurately match the precise words used to identify the dataset within the publication will score higher.\n* Predictions should be cleaned using the clean_text function from the Evaluation page to ensure proper matching.\n* The goal in this competition is not just to match known dataset strings but to generalize to datasets that have never been seen before using NLP and statistical techniques.\n* The hidden test set has roughly ~8000 publications, many times the size of the public test set.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport plotly.express as px\nimport glob\n# Data Visualization\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style(\"darkgrid\")\n\nimport cv2\nfrom wordcloud import WordCloud, STOPWORDS\n#Text Processing\nimport re\nimport nltk\nnltk.download('popular')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2>Files</h2>\n<ul>\n<li><strong>train</strong> - the full text of the training set's publications in JSON format, broken into sections with section titles</li>\n<li><strong>test</strong> - the full text of the test set's publications in JSON format, broken into sections with section titles</li>\n<li><strong>train.csv</strong> - labels and metadata for the training set</li>\n<li><strong>sample_submission.csv</strong> - a sample submission file in the correct format</li>\n</ul>","metadata":{}},{"cell_type":"code","source":"submittion_csv = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ntrain_csv = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain_dir = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/train/*\")\ntest_dir = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/test/*\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h2>Columns</h2>\n<ul>\n<li><code>id</code> - publication <code>id</code> - note that there are multiple rows for some training documents, indicating multiple mentioned datasets</li>\n<li><code>pub_title</code>&nbsp;- title of the publication (a small number of publications have the same title)</li>\n<li><code>dataset_title</code> - the title of the dataset that is mentioned within the publication</li>\n<li><code>dataset_label</code> - a portion of the text that indicates the dataset</li>\n<li><code>cleaned_label</code> - the <code>dataset_label</code>, as passed through the <code>clean_text</code> function from the <a rel=\"nofollow\" href=\"https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation\">Evaluation page</a></li>\n</ul>","metadata":{}},{"cell_type":"code","source":"train_csv.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Total data: 19661\n#unique Id count: 14316\n#Ids are reused: 5345\nid_counts = pd.value_counts(train_csv[\"Id\"])\nfig = px.bar(x=id_counts.values[:20], y=id_counts.index[:20])\nx_axis = dict(tickmode = 'linear',\n    tick0 = 0,\n    dtick = 2)\ny_axis = dict(autorange=\"reversed\")\nfig.update_layout(\n    title=\"Id count\",\n    xaxis_title=\"Count\",\n    yaxis_title=\"Id\",\n    xaxis = x_axis,\n    yaxis = y_axis\n)\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Total data: 19661\n#unique Publication Titles: 14316\n#Publication Titles are reused: 5390\npub_counts = pd.value_counts(train_csv[\"pub_title\"])\nfig = px.bar(x=pub_counts.values[:20], y=pub_counts.index[:20])\nx_axis = dict(tickmode = 'linear',\n    tick0 = 0,\n    dtick = 2)\ny_axis = dict(autorange=\"reversed\",\n              tickmode=\"array\",\n              tickvals=list(range(len(pub_counts))),\n              ticktext = pub_counts.index[:20].map(lambda x: x[:40])\n    )\nfig.update_layout(\n    title=\"Pub Title Count\",\n    xaxis_title=\"Count\",\n    yaxis_title=\"Publication title\",\n    xaxis = x_axis,\n    yaxis = y_axis\n)\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Total data: 19661\n#unique Dataset Titles: 45\ndataset_title_counts = pd.value_counts(train_csv[\"dataset_title\"])\nfig = px.bar(x=dataset_title_counts.values[:20], y=dataset_title_counts.index[:20])\n\ny_axis = dict(autorange=\"reversed\",\n              tickmode=\"array\",\n              tickvals=list(range(len(pub_counts))),\n              ticktext = dataset_title_counts.index[:20].map(lambda x: x[:40])\n    )\nfig.update_layout(\n    title=\"Pub Title Count\",\n    xaxis_title=\"Count\",\n    yaxis_title=\"Publication title\",\n    yaxis = y_axis\n)\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Total data: 19661\n#unique Dataset Titles: 45\nclean_counts = pd.value_counts(train_csv[\"cleaned_label\"])\nfig = px.bar(x=clean_counts.values[:20], y=clean_counts.index[:20])\ny_axis = dict(autorange=\"reversed\",\n              tickmode=\"array\",\n              tickvals=list(range(len(clean_counts))),\n              ticktext = clean_counts.index[:20].map(lambda x: x[:40])\n    )\nfig.update_layout(\n    title=\"Pub Title Count\",\n    xaxis_title=\"Count\",\n    yaxis_title=\"Publication title\",\n    yaxis = y_axis\n)\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_wordcloud(column, title):\n    stopwords = set(STOPWORDS) \n    wordcloud = WordCloud(width = 800, \n                          height = 800,\n                          background_color ='white',\n                          min_font_size = 10,\n                          stopwords = stopwords).generate(' '.join(column)) \n    plt.figure(figsize = (8, 8), facecolor = None) \n    plt.imshow(wordcloud) \n    plt.axis(\"off\") \n    plt.tight_layout(pad = 0) \n    plt.title('Wordcloud: ' + title, fontsize = 20)\n    plt.show()  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_wordcloud(column = train_csv['pub_title'], title = 'Publication Title')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_text(text):\n    lst_stopwords = nltk.corpus.stopwords.words(\"english\")\n    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n    lst_text = text.split()\n    if lst_stopwords is not None:\n        lst_text = [word for word in lst_text if word not in \n                    lst_stopwords]\n\n    lem = nltk.stem.wordnet.WordNetLemmatizer()    \n    lst_text = [lem.lemmatize(word) for word in lst_text]\n\n    text = \" \".join(lst_text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv[\"clean_pub_title\"] = train_csv[\"pub_title\"].apply(lambda x: preprocess_text(x))\ntrain_csv[\"clean_pub_title_len\"] = train_csv[\"clean_pub_title\"].apply(lambda x: len(x))\ntrain_csv[\"clean_pub_title_word_count\"] =train_csv[\"clean_pub_title\"].apply(lambda x: len(str(x).split(\" \")))\ntrain_csv[\"clean_pub_title_char_count\"] = train_csv[\"clean_pub_title\"].apply(lambda x: sum(len(word) for word in str(x).split(\" \")))\ntrain_csv[\"clean_pub_title_avg_word_length\"] = train_csv[\"clean_pub_title_char_count\"] / train_csv[\"clean_pub_title_word_count\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_wordcloud(column = train_csv['clean_pub_title'], title = 'Publication Title')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_wordcloud(column = train_csv['dataset_title'], title = 'Dataset Title')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_wordcloud(column = train_csv['cleaned_label'], title = 'Cleaned Label')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_distribution(x, title):\n    fig = px.histogram(x)\n    fig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pub_title_list = [(\"clean_pub_title_len\", \"Publication Title: Length Distribution\"),\n                 (\"clean_pub_title_word_count\", \"Publication Title: Word Count Distribution\"),\n                 (\"clean_pub_title_char_count\", \"Publication Title: Character Count Distribution\"),\n                 (\"clean_pub_title_avg_word_length\", \"Publication Title: Average Word Length Distribution\")]\nfor i, j in pub_title_list:\n    plot_distribution(train_csv[i], j)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Refrences:https://www.kaggle.com/ishandutta/coleridge-complete-eda-in-one-notebook","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}