{"cells":[{"metadata":{"_uuid":"0b368b7f-19e9-47a3-bd4e-c05b210b1fed","_cell_guid":"48ac1e0c-e36b-4286-98bd-9e7440495691","trusted":true},"cell_type":"markdown","source":"##  Fetching Data and Data Cleaing"},{"metadata":{"_uuid":"a93d6980-832f-4096-84fb-fda41100109d","_cell_guid":"bf0fa0b4-f60a-4dce-b534-46325aac8b2f","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\n# Basic\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nimport json\nimport os\nimport random\nfrom tqdm.autonotebook import tqdm\nimport string\nimport re\nfrom functools import partial\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"214e97dd-e4fb-4be1-b289-1dbd1d420a8d","_cell_guid":"b11f9345-0e03-4e66-87f1-33c15b59bc6d","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"scrolled":true},"cell_type":"code","source":"import functools\nfrom IPython.core.display import display, HTML\npd.set_option('display.max_rows', None)\npd.set_option('display.max_columns', None)\npd.set_option('display.width', None)\npd.set_option('display.max_colwidth', None)\n#Metadata_df=pd.read_csv('/kaggle/input/CORD-19-research-challenge/2020-03-13/all_sources_metadata_2020-03-13.csv')\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'\n\n#Metadata_df = pd.read_csv('/kaggle/input/CORD-19-research-challenge/metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c430e73-3178-4ed9-a1c2-2ea00e7c8d46","_cell_guid":"a16d8399-27cc-441e-aae1-5e54a1dcf870","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"print('shape' ,train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d88bea41-517a-4cc4-b147-81e6b3f5f665","_cell_guid":"38c23682-0ca6-4d82-a452-7312e65802a4","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"print('info',train_df.info())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d47bb938-7dec-48d6-b8d2-0aef055a7c97","_cell_guid":"07b6b496-160e-4a2d-9852-4b3fb1896b6a","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d01a3d2d-75be-44d1-bf4d-5cd6588fbaa8","_cell_guid":"db3f6f84-de1b-4b14-9da4-b1ba9c68c317","trusted":true},"cell_type":"markdown","source":"# 1.Rendering Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef read_append_return(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tqdm.pandas()\ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_colwidth', None)\ntrain_df.text[train_df['Id']=='0007f880-0a9b-492d-9a58-76eb0b0e0bd7']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in train_df.columns:\n    print(f\"{col}: {len(train_df[col].unique())}\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"554058b2-78ff-4a16-b1d6-785ad3e390cc","_cell_guid":"70fb38f0-eed1-47e8-a05a-f8a9d5efedb5","trusted":true},"cell_type":"markdown","source":"# 2.Data Visualization"},{"metadata":{"_uuid":"f35d50bc-f393-43ee-af87-30c8117abe95","_cell_guid":"6e89927c-6d2f-454f-b7fb-9c00c3748412","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#textprops={'color':\"b\"}\nimport matplotlib.pyplot as plt\ntrain_df['pub_title'].value_counts().iloc[[0,1,2,3,4,5,6,7,8,9]].plot.pie(figsize = (10,10),autopct = '%.2f%%',\n                                                                           title = 'Top Ten Pub Title')\nplt.title(\"Top Ten Pub Title\", bbox={'facecolor':'0.8', 'pad':5})\nplt.savefig(\"Top Ten Pub Title\"+\".png\", bbox_inches='tight')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54d73d8d-ffaa-4f6f-887b-c55d6e0ef04e","_cell_guid":"03b411d1-57f4-424a-b65d-3a2f21e878bf","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"\ntrain_df['dataset_title'].value_counts().iloc[[0,1,2,3,4,5,6,7,8,9]].plot.pie(figsize = (10,10), autopct = '%.2f%%',\n                                                                           title = 'Top Ten Dataset title')\nplt.title(\"Top Ten Data set title\", bbox={'facecolor':'0.8', 'pad':5})\nplt.savefig(\"Top Ten Data set title\"+\".png\", bbox_inches='tight')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"309ffa28-4dcf-4f21-9037-a5534da13a79","_cell_guid":"1275a97c-da1e-4bca-b9f5-1984d5ecac68","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"\ntrain_df['dataset_label'].value_counts().iloc[[0,1,2,3,4,5,6,7,8,9]].plot.pie(figsize = (10,10), autopct = '%.2f%%',\n                                                                           title = 'Top Ten Dataset Label')\nplt.title(\"Top Ten Dataset Label\", bbox={'facecolor':'0.8', 'pad':5})\nplt.savefig(\"Top Ten Dataset Label\"+\".png\", bbox_inches='tight')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"duplicateRowsDF = train_df['Id'][train_df.duplicated(['Id'])]\nprint(\"Duplicate Rows except first occurrence based on all columns are :\")\nprint(duplicateRowsDF)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"dups_ID = train_df.pivot_table(index=['Id'], aggfunc='size')\nprint (dups_ID)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5dc744ab-6c22-4281-9256-b01ea824bcef","_cell_guid":"111c354d-f447-4a3d-ab94-c5d6a99e8909","trusted":true},"cell_type":"markdown","source":"# 3.Text Preprocessing"},{"metadata":{"_uuid":"ed40f11e-8af0-4e44-83ed-7431e326e09d","_cell_guid":"5e60a313-c18f-4c21-ac4b-42f242612d2b","trusted":true,"scrolled":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from gensim.parsing.preprocessing import remove_stopwords\nfrom wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\nfrom collections import Counter \nimport time\nimport re\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem.porter import PorterStemmer\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.decomposition import PCA\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\nfrom gensim import corpora,models,similarities\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem.porter import PorterStemmer\nfrom gensim.summarization import summarize\nfrom gensim.summarization import keywords\nporter = PorterStemmer()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"200d21c8-985b-42bb-b567-ed18fc3b94b4","_cell_guid":"9fea0807-a170-4ea9-9f4a-13762b37c884","trusted":true,"scrolled":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# getting the lower case , removing stopwords and gettting the word origin\ndef lower_case(input_str):\n    input_str = input_str.lower()\n    return input_str\ndef stemSentence(sentence):\n    token_words=word_tokenize(sentence)\n    token_words\n    stem_sentence=[]\n    for word in token_words:\n        stem_sentence.append(porter.stem(word))\n        stem_sentence.append(\" \")\n    return \"\".join(stem_sentence)\n\ndef text_preprocessing(text):\n    text= re.sub('[^a-zA-z0-9\\s]','',text)\n    text=lower_case(text)\n    text=remove_stopwords(text)\n    text=stemSentence(text)    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1043302c-de02-45ad-89ed-d95feb7b29c1","_cell_guid":"a1fa19cd-240e-4c3b-86b6-ea88ba3b5bc3","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"scrolled":true},"cell_type":"code","source":"'''Combined_data_processing=train_df.copy()\nCombined_data_processing['pub_title']=Combined_data_processing['pub_title'].apply(lambda x: text_preprocessing(x))\nCombined_data_processing['dataset_title']=Combined_data_processing['dataset_title'].apply(lambda x: text_preprocessing(x))\nCombined_data_processing['dataset_label']=Combined_data_processing['dataset_label'].apply(lambda x: text_preprocessing(x))\nCombined_data_processing['text']=Combined_data_processing['text'].apply(lambda x: text_preprocessing(x))'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Combined_data_processing.to_csv('processed_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Combined_data_processing=pd.read_csv('../input/coleridge-cosine-similarity-gensim/processed_train.csv')\nCombined_data_processing.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e8001f4-6f77-459a-b5c0-fc1015dd73a1","_cell_guid":"a3e9a7cb-6634-408c-9f39-3db50b93964b","trusted":true,"collapsed":true},"cell_type":"code","source":"'''KeyWords=[]\nfor i in range(0,len(Combined_data_processing)):\n    KeyWords.append(remove_stopwords(str(keywords(Combined_data_processing['text'][i]))))'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a6419d8-9a7b-4937-8179-c4f1e7763d01","_cell_guid":"fdabe7d0-6394-4e0d-87dd-8a3a319e49de","trusted":true,"scrolled":true},"cell_type":"code","source":"'''corpus_text = [] # intializing list of collection of  clean reviews\n\nfor i in range(0, 33384):\n    for word in str(KeyWords['KeyWords'][i]).split():\n        corpus_text.append(word)'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1c2da37-648e-430d-9fae-2814c2a343bf","_cell_guid":"9326b24a-1c58-41c7-94e8-96808bc90a1b","trusted":true,"scrolled":true},"cell_type":"code","source":"\n#len(KeyWords)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f2dfdf4-5b40-49fa-b13b-4c94bc5bcd5c","_cell_guid":"b0898dc8-0186-4e28-a51d-4934f32fcfd9","trusted":true,"scrolled":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":" #removing the words less than four chars \n#corpus_text_modified=[i for i in corpus_text if 5 <=  len(i)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"782f0e0e-ed66-4d1e-b45f-60def15c1e17","_cell_guid":"c13235a4-c268-490c-865e-ad0d89de50c4","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"'''unique_string=(\" \").join(KeyWords)\nwordcloud = WordCloud(width = 1000, height = 500 ,max_words=10000).generate(unique_string)\nplt.figure(figsize=(15,8))\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.savefig(\"your_file_name\"+\".png\", bbox_inches='tight')\nplt.show()\nplt.close()'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dccd6797-d161-4d48-bfa4-18764f03a7e6","_cell_guid":"f3f69322-0e52-42f6-9605-42f3bca36eba","trusted":true},"cell_type":"markdown","source":"# 4.Word Clouds"},{"metadata":{"_uuid":"00e32dd9-56ac-41ed-bdfe-92f6481dc3cc","_cell_guid":"a7aeb420-be8f-40dd-8a40-5ab52256af3b","trusted":true},"cell_type":"code","source":"corpus_pub_title = [] # intializing list of collection of  clean reviews\n\nfor i in range(0, len(Combined_data_processing)):\n    for word in str(Combined_data_processing['pub_title'][i]).split():\n        corpus_pub_title.append(word)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a40fea94-6020-4cac-8889-5767a41d87a7","_cell_guid":"617a12fa-4d5a-4c59-ba67-31d8a099dbbf","trusted":true},"cell_type":"code","source":"''''removed_words= ['nan', 'missing','miss']  \nfor word in list(corpus_pub_title):  # iterating on a copy since removing will mess things up\n    if word in removed_words:\n        corpus_pub_title.remove(word)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13afd469-ac9f-496f-a20a-7e16eba6c90d","_cell_guid":"f0125f77-d6cd-4432-a0df-b3c24fb9458c","trusted":true},"cell_type":"code","source":"corpus_pub_title=[i for i in corpus_pub_title if 3 <=  len(i)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2318bba9-c714-4359-af86-7a92d29bde28","_cell_guid":"210a77f0-017c-4e21-a87e-0c47e27c15ad","trusted":true},"cell_type":"code","source":"unique_string=(\" \").join(corpus_pub_title)\nwordcloud = WordCloud(background_color='white',width = 1000, height = 500 ,max_words=10000).generate(unique_string)\nplt.figure(figsize=(15,8))\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.savefig(\"Authors_Words\"+\".png\", bbox_inches='tight')\nplt.show()\nplt.close()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1364bfdb-f993-4527-ab73-a70feaa8a82e","_cell_guid":"34a2ff6d-477b-4cf8-ab83-95544896f675","trusted":true,"scrolled":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"corpus_dataset_title = [] # intializing list of collection of  clean reviews\n\nfor i in range(0, len(Combined_data_processing)):\n    for word in str(Combined_data_processing['dataset_title'][i]).split():\n        corpus_dataset_title.append(word)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eac035b4-fb37-4702-bf61-18f718a0e01e","_cell_guid":"680e68ee-c73c-40ce-a852-a08b7e22062f","trusted":true,"scrolled":true},"cell_type":"code","source":"unique_string=(\" \").join(corpus_dataset_title)\nwordcloud = WordCloud(background_color='white',width = 1000, height = 500 ,max_words=10000).generate(unique_string)\nplt.figure(figsize=(15,8))\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.savefig(\"Titles_Words\"+\".png\", bbox_inches='tight')\nplt.show()\nplt.close()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"937e58fb-ef9c-45a3-bd1a-539aacfc9e6d","_cell_guid":"24ccc5ce-58cd-4b43-be6d-68fa11543fb2","trusted":true,"scrolled":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"corpus_dataset_label = [] # intializing list of collection of  clean reviews\n\nfor i in range(0, len(Combined_data_processing)):\n    for word in str(Combined_data_processing['dataset_label'][i]).split():\n        corpus_dataset_label.append(word)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ed27080-1229-4649-8569-4abfdf034ba0","_cell_guid":"d97e4a2b-542b-4f47-92ad-b66ce253a782","trusted":true},"cell_type":"code","source":"unique_string=(\" \").join(corpus_dataset_label)\nwordcloud = WordCloud(background_color='white',width = 1000, height = 500 ,max_words=30000).generate(unique_string)\nplt.figure(figsize=(15,8))\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.savefig(\"abstract_words\"+\".png\", bbox_inches='tight')\nplt.show()\nplt.close()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20b0766f-efd4-4853-b1fc-8be4013c8851","_cell_guid":"02b4589a-52a7-4709-88c2-faf765fcaccf","trusted":true},"cell_type":"markdown","source":"# 5.  Most Frequent Words"},{"metadata":{"_uuid":"3c815d7b-c4d1-41d9-8599-ff560a28e498","_cell_guid":"f00831bc-a0c9-484e-bd4e-d570cbad66ce","trusted":true,"scrolled":false},"cell_type":"code","source":"title_font = {'fontname':'Arial', 'size':'18', 'color':'black', 'weight':'normal',\n              'verticalalignment':'bottom'} # Bottom vertical alignment for more space\naxis_font = {'fontname':'Arial', 'size':'14'}\n#corpus_title=[i for i in corpus_title if 4 <=  len(i)]\ncounter_title = Counter(corpus_dataset_title) \n  \n# most_common() produces k frequently encountered \n# input values and their respective counts. \nmost_occur = dict(counter_title.most_common(100)) \n  \nprint(most_occur) \n\nlabels, values = zip(*most_occur.items())\n\n# sort your values in descending order\nindSort = np.argsort(values)[::-1]\n\n# rearrange your data\nlabels = np.array(labels)[indSort]\nvalues = np.array(values)[indSort]\n\nindexes = np.arange(len(labels))\nplt.figure(figsize=(20,40))\nplt.barh(labels, values)\nplt.title('The most frequent words in Articles Titles',**title_font,bbox={'facecolor':'0.8', 'pad':5})\nplt.xlabel('Number of Words',**axis_font,color='black')\nplt.ylabel('Top 100 Words in Titles',**axis_font,color='black')\nplt.xticks(color='black')\nplt.yticks(color='black')\nplt.savefig(\"title most frequent Words\"+\".png\", bbox_inches='tight')\n# add labels\n#plt.xticks(indexes + bar_width, labels)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e270dcd-3968-4877-8c2f-56027ee26962","_cell_guid":"7b140d76-1434-49c5-ba2f-b6527d764305","trusted":true,"scrolled":false},"cell_type":"code","source":"counter_abstract = Counter(corpus_dataset_label) \n  \n# most_common() produces k frequently encountered \n# input values and their respective counts. \nmost_occur = dict(counter_abstract.most_common(100)) \n  \nprint(most_occur) \n\nlabels, values = zip(*most_occur.items())\n\n# sort your values in descending order\nindSort = np.argsort(values)[::-1]\n\n# rearrange your data\nlabels = np.array(labels)[indSort]\nvalues = np.array(values)[indSort]\n\nindexes = np.arange(len(labels))\nplt.figure(figsize=(20,40))\nplt.barh(labels, values)\nplt.title('The most frequent words in Articles Abstract',**title_font,bbox={'facecolor':'0.8', 'pad':5})\nplt.xlabel('Number of Words',**axis_font,color='black')\nplt.ylabel('Top 100 Words in Abstracts',**axis_font,color='black')\nplt.xticks(color='black')\nplt.yticks(color='black')\nplt.savefig(\"abstract most frequent Words\"+\".png\", bbox_inches='tight')\n# add labels\n#plt.xticks(indexes + bar_width, labels)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d0f58b7-c146-4585-8d99-4a451cb653dd","_cell_guid":"6a9e4715-ab76-47b9-9247-5472f9b962d6","trusted":true},"cell_type":"code","source":"title_font = {'fontname':'Arial', 'size':'18', 'color':'black', 'weight':'normal',\n              'verticalalignment':'bottom'} # Bottom vertical alignment for more space\naxis_font = {'fontname':'Arial', 'size':'14'}\n#corpus_authors=[i for i in corpus_authors if 2 <=  len(i)]\ncorpus_authors = Counter(corpus_pub_title) \n  \n# most_common() produces k frequently encountered \n# input values and their respective counts. \nmost_occur = dict(corpus_authors.most_common(100)) \n  \nprint(most_occur) \n\nlabels, values = zip(*most_occur.items())\n\n# sort your values in descending order\nindSort = np.argsort(values)[::-1]\n\n# rearrange your data\nlabels = np.array(labels)[indSort]\nvalues = np.array(values)[indSort]\n\nindexes = np.arange(len(labels))\nplt.figure(figsize=(20,40))\nplt.barh(labels, values)\nplt.title('The most frequent words in Articles Abstract',**title_font,bbox={'facecolor':'0.8', 'pad':5})\nplt.xlabel('Number of Words',**axis_font,color='black')\nplt.ylabel('Top 100 Words in Abstracts',**axis_font,color='black')\nplt.xticks(color='black')\nplt.yticks(color='black')\nplt.savefig(\"abstract most frequent Words\"+\".png\", bbox_inches='tight')\n# add labels\n#plt.xticks(indexes + bar_width, labels)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"49d82e35-7440-4f8e-b765-068f758d73d8","_cell_guid":"dc095cdf-40f7-4100-97dc-38a07ce0ed21","trusted":true},"cell_type":"markdown","source":"# 6. Modelling"},{"metadata":{"_uuid":"11ee5e64-6483-4eca-841c-56c73c03f86f","_cell_guid":"5d3e3029-5f99-408f-b7c9-af1b59e6dcfe","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"#Implementing TFIDF \nvectorizer = TfidfVectorizer(max_features=5000)\nX = vectorizer.fit_transform(Combined_data_processing['text'].values)\n\n#implementing K-Means\nk = 9\nkmeans = MiniBatchKMeans(n_clusters=k)\ny_pred = kmeans.fit_predict(X)\n\nfrom sklearn.decomposition import PCA\npca = PCA(n_components=3)\npca_result = pca.fit_transform(X.toarray())\n            \nprint(pca.components_)\nprint(pca.explained_variance_)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"384f7e04-40d5-477c-87b3-02d3a6ff5a69","_cell_guid":"e66722c7-360b-48fb-87ad-157e1c439e44","trusted":true,"scrolled":true},"cell_type":"code","source":"#implementing K-Means Clustering\ntitle_font = {'fontname':'Arial', 'size':'18', 'color':'black', 'weight':'normal',\n              'verticalalignment':'bottom'} # Bottom vertical alignment for more space\naxis_font = {'fontname':'Arial', 'size':'14'}\nimport seaborn as sns\nsns.set(rc={'figure.figsize':(15,15)})\n\n# colors\npalette = sns.color_palette(\"muted\", len(set(y_pred)))\n\n# plot\nsns.scatterplot(pca_result[:,0], pca_result[:,1], hue=y_pred,sizes =100, legend='full',palette=palette)\nplt.title(\"PCA - Clustered (K-Means)\",**title_font,bbox={'facecolor':'0.8', 'pad':5})\nplt.savefig(\"Kmeans\"+\".png\", bbox_inches='tight')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3afbe008-844c-42b6-bdd4-acd760f921ff","_cell_guid":"57636bf9-0aaa-4de3-ac89-40d729cd1cc4","trusted":true,"scrolled":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d import Axes3D\n\nax = plt.figure(figsize=(16,10)).gca(projection='3d')\nax.scatter(\n    xs=pca_result[:,0], \n    ys=pca_result[:,1], \n    zs=pca_result[:,2], \n    c=y_pred, \n    cmap='tab10'\n)\nax.set_xlabel('pca-1')\nax.set_ylabel('pca-2')\nax.set_zlabel('pca-3')\n\nplt.title(\"PCA - Clustered (K-Means)-3D\",**title_font,bbox={'facecolor':'0.9', 'pad':2})\nplt.savefig(\"Kmeans_3D\"+\".png\", bbox_inches=\"tight\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65ea1593-66ac-4e9c-b2bd-80ebc2e20d5b","_cell_guid":"c84e06a3-0e47-4b44-a068-eec2f168bab1","trusted":true},"cell_type":"markdown","source":"# 7. Model Evaluation"},{"metadata":{"_uuid":"af4d3aa0-61a9-43cd-a981-d09f4a83f19c","_cell_guid":"ca8c0985-d7cb-4049-8418-a693ea658fca","trusted":true},"cell_type":"code","source":"#Getting the best number of clusters using elbow method\nimport time\ndef elbow_plot(data, start_K, end_K, step):\n    '''\n    Generate an elbow plot to find optimal number of clusters\n    graphing K values from start_K to end_K every step value\n    \n    INPUT: \n        data: Demographics DataFrame\n        start_K: Inclusive starting value for cluster number\n        end_K: Exclusive stopping value for cluster number\n        step: Step value between start_K and end_K\n    OUTPUT: Trimmed and cleaned demographics DataFrame\n    '''\n    score_list = []\n\n    for i in range(start_K, end_K, step):\n        print(i)\n        start = time.time()\n        kmeans = MiniBatchKMeans(i)\n        model = kmeans.fit(data)\n        score = model.score(data)\n        score_list.append(abs(score))\n        end = time.time()\n        elapsed_time = end - start\n        print(elapsed_time)\n\n    plt.plot(range(start_K, end_K, step), \n    score_list, linestyle='--', marker='o', color='b');\n    plt.xlabel('# of clusters K');\n    plt.ylabel('Sum of squared errors');\n    plt.title('SSE vs. K');\n    plt.savefig('elbow_plot.png')\nelbow_plot(pca_result, 1, 20, 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b4d4151-54f3-4688-941e-f3ae58505713","_cell_guid":"a5361316-3b23-456a-b476-5272b6806e97","trusted":true},"cell_type":"markdown","source":"## Conclusion\n1. -This scatter plot is generated from Articles text , each article text is a feature. \n1. -usinig features vector TfidfVectorizer. \n1. -Dimensionality Reduction using PCA.\n1. -generating clustering using k-Means where k=9 (the best value as elbow plot).\n1. -Topic Modeling is done on each cluster to get the keywords per cluster."},{"metadata":{},"cell_type":"markdown","source":"# to be continued......"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}