{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Libraries\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport re\nimport json\nimport string\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom tqdm.autonotebook import tqdm\nfrom functools import partial\nfrom wordcloud import WordCloud, STOPWORDS\nimport nltk\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4000000\nfrom nltk.probability import FreqDist\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train Data Exploration","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Data Description

\n

\n","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we have no 'NULL' values in the train data","metadata":{}},{"cell_type":"code","source":"for col in train.columns:\n print(col + \":\" + str(len(train[col].unique())))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Inference

\n\n\n- The Training Dataset has 19,661 samples but only 14,316 unique IDs in the dataset. This means that some publications include a multitude of datasets. \n\n\n- The pub_title unique count is also less than the Id unique counts. This points to the precense of several occurences of having 2 separate publications, each with a unique ID, but sharing the exact same title.\n\n\n- Also, there are a total of 45 unique dataset_title and 130 unique dataset_label. It means that a single dataset could have multible labels throughout different publications.","metadata":{}},{"cell_type":"markdown","source":"Sample Submission","metadata":{}},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Data Description

\n

\n","metadata":{}},{"cell_type":"markdown","source":"

Data Processing

","metadata":{}},{"cell_type":"markdown","source":"Now we will create a function to get the text from the JSON file and append it to the new column in table","metadata":{}},{"cell_type":"code","source":"train_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def json_to_text(filename, train_files_path=train_files_path, output='text'):\n json_path = os.path.join(train_files_path, (filename+'.json'))\n headings = []\n contents = []\n combined = []\n with open(json_path, 'r') as f:\n json_decode = json.load(f)\n for data in json_decode:\n headings.append(data.get('section_title'))\n contents.append(data.get('text'))\n combined.append(data.get('section_title'))\n combined.append(data.get('text'))\n \n all_headings = ' '.join(headings)\n all_contents = ' '.join(contents)\n all_data = '. '.join(combined)\n \n if output == 'text':\n return all_contents\n elif output == 'head':\n return all_headings\n else:\n return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain['text'] = train['Id'].progress_apply(json_to_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see the Train Data now","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now apply the function to submission Data","metadata":{}},{"cell_type":"code","source":"tqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(json_to_text, train_files_path=test_files_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create a function to Preprocess the data using Basic NLP Filters (all text to lower case, Removes special charecters, emojis and multiple spaces)","metadata":{}},{"cell_type":"code","source":"def text_cleaning(text):\n text = ''.join([k for k in text if k not in string.punctuation])\n text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n text = re.sub(' +', ' ', text)\n emoji_pattern = re.compile(\"[\"\n u\"\\U0001F600-\\U0001F64F\" # emoticons\n u\"\\U0001F300-\\U0001F5FF\" # symbols & pictographs\n u\"\\U0001F680-\\U0001F6FF\" # transport & map symbols\n u\"\\U0001F1E0-\\U0001F1FF\" # flags (iOS)\n \"]+\", flags=re.UNICODE)\n text = emoji_pattern.sub(r'', text)\n return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain['text'] = train['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

EDA with Visualization

","metadata":{}},{"cell_type":"code","source":"ul = train['cleaned_label'].unique()\nul[0:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('There are {} unique cleaned labels'.format(len(ul)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Here is wordcloud based on texts from first unique label","metadata":{}},{"cell_type":"code","source":"text = ' '.join(train['text'][train['cleaned_label']==ul[0]].sample(frac=0.3))\nwordcloud = WordCloud(background_color='white', stopwords=STOPWORDS, width=2000, height=1200).generate(text)\nbarplot_dim = (15, 15)\nax = plt.subplots(figsize=barplot_dim, facecolor='w')\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.title('Top 100 Most Common Words in Publications Text', fontsize=50)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Here is wordcloud based on texts from 10th unique label","metadata":{}},{"cell_type":"code","source":"text = ' '.join(train['text'][train['cleaned_label']==ul[10]].sample(frac=0.3))\nwordcloud = WordCloud(background_color='white', stopwords=STOPWORDS, width=2000, height=1200).generate(text)\nbarplot_dim = (15, 15)\nax = plt.subplots(figsize=barplot_dim, facecolor='w')\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.title('Top 100 Most Common Words in Publications Text', fontsize=50)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Here is wordcloud based on texts from 100th unique label","metadata":{}},{"cell_type":"code","source":"text = ' '.join(train['text'][train['cleaned_label']==ul[100]].sample(frac=0.3))\nwordcloud = WordCloud(background_color='white', stopwords=STOPWORDS, width=2000, height=1200).generate(text)\nbarplot_dim = (15, 15)\nax = plt.subplots(figsize=barplot_dim, facecolor='w')\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.title('Top 100 Most Common Words in Publications Text', fontsize=50)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Some Similarity Measures

","metadata":{}},{"cell_type":"markdown","source":"### Cosine similarity function","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\n\ndef cosine(train, test):\n \"\"\"\n Enter text one from each train and test set for cosine similarity\n \"\"\"\n sw = stopwords.words('english')\n X_list = word_tokenize(train)\n Y_list = word_tokenize(test)\n l1 =[];l2 =[]\n # remove stop words from the string\n X_set = {w for w in X_list if not w in sw} \n Y_set = {w for w in Y_list if not w in sw}\n # form a set containing keywords of both strings \n rvector = X_set.union(Y_set) \n for w in rvector:\n if w in X_set: l1.append(1) # create a vector\n else: l1.append(0)\n if w in Y_set: l2.append(1)\n else: l2.append(0)\n c = 0\n # cosine formula \n for i in range(len(rvector)):\n c+= l1[i]*l2[i]\n cosine = c / float((sum(l1)*sum(l2))**0.5)\n return cosine","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(0,len(sample_sub['text'])):\n print('Similarity of first train text with text {} from test data'.format(j))\n print(cosine(train['text'][0], sample_sub.loc[j,'text']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(0,len(sample_sub['text'])):\n print('Similarity of second train text with text {} from test data'.format(j))\n print(cosine(train['text'][1], sample_sub.loc[j,'text']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Jaccard similarity","metadata":{}},{"cell_type":"code","source":"def jaccard_similarity(text_a, text_b):\n word_set_a, word_set_b = [set(text.split())\n for text in [text_a, text_b]]\n num_shared = len(word_set_a & word_set_b)\n num_total = len(word_set_a | word_set_b)\n return num_shared / num_total","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(0,len(sample_sub['text'])):\n print('Similarity of first train text with text {} from test data'.format(j))\n similarity = jaccard_similarity(sample_sub.loc[j,'text'], train['text'][0])\n print(similarity)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(0,len(sample_sub['text'])):\n print('Similarity of second train text with text {} from test data'.format(j))\n similarity = jaccard_similarity(sample_sub.loc[j,'text'], train['text'][1])\n print(similarity)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SequenceMatcher from difflib","metadata":{}},{"cell_type":"code","source":"import difflib\n\nfor j in range(0,len(sample_sub['text'])):\n print('Similarity of first train text with text {} from test data'.format(j))\n d = difflib.SequenceMatcher(None, sample_sub.loc[j,'text'], train['text'][0])\n similarity = d.ratio()*100\n print(similarity)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(0,len(sample_sub['text'])):\n print('Similarity of second train text with text {} from test data'.format(j))\n d = difflib.SequenceMatcher(None, sample_sub.loc[j,'text'], train['text'][1])\n similarity = d.ratio()*100\n print(similarity)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are several other distance/similarity measures. Many of them available in package like sklean, spacy, nltk etc.","metadata":{}},{"cell_type":"markdown","source":"# UPVOTE if you like this notebook.\n## Thanks","metadata":{}}]}