{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective: \n\nThe objective of the competition is to identify the mention of datasets within scientific publications. Your predictions will be short excerpts from the publications that appear to note a dataset. Predictions that more accurately match the precise words used to identify the dataset within the publication will score higher. \n\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom colorama import Fore, Back, Style\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n# Setting color palette.\npurple_black = [\n\"#9b59b6\", \"#3498db\", \"#95a5a6\", \"#e74c3c\", \"#34495e\", \"#2ecc71\"\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Familiarization","metadata":{}},{"cell_type":"code","source":"# load the meta data\ntrain_csv = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain_csv.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(Fore.BLUE + \"Metadata file has {} rows and {} columns\".format(train_csv.shape[0],train_csv.shape[1]),Style.RESET_ALL)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Let's check the data present in the metadata file","metadata":{}},{"cell_type":"code","source":"# Let's check publication ID\n# check the no. of unique publications present in the metadata\nprint(Fore.BLUE + \"No. of Unique Publications:\",train_csv.Id.nunique(),Style.RESET_ALL)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's check total no. of rows present in the train.csv\nprint(Fore.BLUE +\"Total no. of rows in the metadata file:\",train_csv.shape[0],Style.RESET_ALL)\n\n# train.csv has 19661 rows whereas there are only 14316 unique publications, which means there are multiple rows\n# for few publications, because they might have reffered to multiple datasets","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# No. of unique publications titles\nprint(Fore.BLUE +\"No. of unique publication titles:\",train_csv.pub_title.nunique(),Style.RESET_ALL)\n\n# There seems to be 14271 unique titles, whereas it should have been 14316, which means a small number of publications \n# have the same title","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# No. of unique dataset titles(title of the dataset that is mentioned within the publication)\nprint(Fore.BLUE +\"No. of unique dataset titles:\",train_csv.dataset_title.nunique(),Style.RESET_ALL)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# No. of unique dataset labels(a portion of the text that indicates the dataset) in the metadata \nprint(Fore.BLUE +\"No. of unique Labels in the meta:\",train_csv.dataset_label.nunique(),Style.RESET_ALL)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are only 45 unique titles whereas 130 labels, which means different variants of the dataset titles are used in the publications. let's verify.","metadata":{}},{"cell_type":"code","source":"# unique titles used\ncount = train_csv.dataset_title.value_counts()\n\nfig = go.Figure(data=[go.Table(\n columnwidth = [0.25, 2, 0.5],\n header=dict(\n values=[\"Rank\", \"Dataset Title\", \"Mentions\"],\n line_color='darkslategray',\n fill_color=\"green\",\n align='center',\n font=dict(color='white', size=12)\n ),\n cells=dict(\n values=np.array([np.array((str(i+1), \"\" + x + \"\", \"\" + str(y) + \"\", )) for i, (x, y) in enumerate(zip(count.index, count.values))]).T,\n line_color='darkslategray',\n # 2-D list of colors for alternating rows\n fill_color = [[\"white\",\"lavender\"]*25],\n align = 'center',\n font = dict(color = 'darkslategray', size = 11)\n ))\n])\n\nfig.update_layout(\n title={\"text\": \"Datasets Titles Mentions Counts\",\n \"x\": 0.5,\n \"xanchor\":\"center\",\n \"font_size\": 22},\n margin={\"r\":20, \"l\":20})\n\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's Visualize top 20 of the titles used \n\nfig = px.pie(count,\n values=count.values[:20],\n names=count.index[:20],\n color_discrete_sequence=purple_black,\n hole=.4,title=\"Top 20 Titles\")\nfig.update_traces(textinfo='percent', pull=0.05)\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique labels used\ncount = train_csv.dataset_label.value_counts()\n\nfig = go.Figure(data=[go.Table(\n columnwidth = [0.25, 2, 0.5],\n header=dict(\n values=[\"Rank\", \"Dataset Labels\", \"Mentions\"],\n line_color='darkslategray',\n fill_color=\"green\",\n align='center',\n font=dict(color='white', size=12)\n ),\n cells=dict(\n values=np.array([np.array((str(i+1), \"\" + x + \"\", \"\" + str(y) + \"\", )) for i, (x, y) in enumerate(zip(count.index, count.values))]).T,\n line_color='darkslategray',\n # 2-D list of colors for alternating rows\n fill_color = [[\"white\",\"lavender\"]*25],\n align = 'center',\n font = dict(color = 'darkslategray', size = 11)\n ))\n])\n\nfig.update_layout(\n title={\"text\": \"Datasets Labels Mentions Counts\",\n \"x\": 0.5,\n \"xanchor\":\"center\",\n \"font_size\": 22},\n margin={\"r\":20, \"l\":20})\n\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's Visualize top 20 of the labels used \n\nfig = px.pie(count,\n values=count.values[:20],\n names=count.index[:20],\n color_discrete_sequence=purple_black,\n hole=.4,title=\"Top 20 Labels\")\nfig.update_traces(textinfo='percent', pull=0.05)\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From above two results, we can confirm that different variants of the titles are used in the publications.\nfor example, \"ADNI\" & \"Alzheimer's Disease Neuroimaging Initiative (ADNI)\" have been used interchangably in the publications.","metadata":{}},{"cell_type":"markdown","source":"# Sanity Check","metadata":{}},{"cell_type":"code","source":"import os\npath = os.walk(\"../input/coleridgeinitiative-show-us-the-data/train\")\n\njson_list = []\n\nfor _,_,files in path:\n for file in files:\n #names.append(file[:-5])\n json_list.append(file)\n\nprint(Fore.BLUE + \"No. of Json Files in the training folder:\", len(json_list),Style.RESET_ALL)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets take first publication from train.csv and see if it is referred in the related publication in the train folder\nimport json\n \n# Opening JSON file\nf = open(\"../input/coleridgeinitiative-show-us-the-data/train\" + \"/\" + json_list[0])\n \n# returns JSON object as \n# a dictionary\ndata = json.load(f)\n \n# Iterating through the json list\nfor i in data:\n print(Fore.GREEN + \"First Section Title\",Style.RESET_ALL)\n print(i)\n break # break after printing first section_title\n# Closing file\nf.close()\n\n# we have publication for id d0fa7568-7d8e-4db9-870f-f9c6f668c17b in \"data\" variable\n# now we will check whether \"dataset title - National Education Longitudinal Study\" is present in the publication or not\n\nfor i in range(len(data)):\n if train_csv.loc[0]['dataset_title'] in data[i]['text']:\n print(Fore.BLUE +\"Title {}{}{} is Present in the given publication\".format(\"'\",train_csv.loc[0]['dataset_title'],\"'\"),Style.RESET_ALL)\n break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NER using SPACY","metadata":{}},{"cell_type":"markdown","source":"### SPACY Supports following entity types\n","metadata":{}},{"cell_type":"code","source":"import spacy\nfrom spacy import displacy\nfrom collections import Counter\nimport en_core_web_sm\nnlp = en_core_web_sm.load()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One of the nice things about Spacy is that we only need to apply nlp once, the entire background pipeline will return the objects.","metadata":{}},{"cell_type":"markdown","source":"### Entity","metadata":{}},{"cell_type":"code","source":"doc = nlp(str(data)) # \"data\" still has text from the publication we used earlier in this notebook \nprint([(X.text, X.label_) for X in doc.ents[0:20]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All the Entities seems to have tagged correctly!","metadata":{}},{"cell_type":"markdown","source":"### Token Level\nDuring the above example, we were working on entity level, in the following example, \nwe are demonstrating token-level entity annotation using the BILUO tagging scheme to describe the entity boundaries.\n","metadata":{}},{"cell_type":"code","source":"print([(X, X.ent_iob_, X.ent_type_) for X in doc[:20]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"B\" means the token begins an entity, \"I\" means it is inside an entity, \"O\" means it is outside an entity, and \"\" means no entity tag is set.","metadata":{}},{"cell_type":"code","source":"print(\"There are {} entities in the publication\".format(len(doc.ents)))\n\nlabels = [x.label_ for x in doc.ents]\nprint(\"\\nThese entities are represented by {} unique labels\".format(len(Counter(labels))))\n\nprint(\"\\nFollowing is the list of unique labels:\\n\")\nprint(Counter(labels))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Following are the 3 most common entities\")\nitems = [x.text for x in doc.ents]\nCounter(items).most_common(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let’s run displacy.render to generate the raw markup.\ndisplacy.render(nlp(str(data[0:1])), jupyter=True, style='ent')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using spaCy’s built-in displaCy visualizer, here’s what the above publication and its dependencies look like:\ndisplacy.render(nlp(str(doc[0:20])), style='dep', jupyter = True, options = {'distance': 120})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we verbatim, extract part-of-speech and lemmatize this publication.","metadata":{}},{"cell_type":"code","source":"[(x.orth_,x.pos_, x.lemma_) for x in [y \n for y\n in nlp(str(doc[0:100])) \n if not y.is_stop and y.pos_ != 'PUNCT']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict([(str(x), x.label_) for x in nlp(str(doc[0:200])).ents])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print([(x, x.ent_iob_, x.ent_type_) for x in doc[0:200]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Thank you all for your upvotes :) Please check my [NER MODEL](https://www.kaggle.com/jagdmir/spacy-ner-model) on model building for this competition","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}