{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from textblob import TextBlob\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nfrom nltk.stem import PorterStemmer\n!pip install textblob \nfrom nltk.corpus import stopwords\nimport nltk\nnltk.download()\nprint('Libraries Imported')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ndata","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (15,50)),\nsns.set_theme()\nsns.countplot(y=data[\"cleaned_label\"], order = data[\"cleaned_label\"].value_counts().index, palette = \"winter\")\nplt.ylabel(\"CLEANSED LABEL\",fontsize = 50)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords\nstoplist = stopwords.words('english') + ['though']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nc_vec = CountVectorizer(stop_words=stoplist, ngram_range=(2,2))\n# matrix of ngrams\nngrams = c_vec.fit_transform(data['cleaned_label'])\n# count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis=0)\n# list of ngrams\nvocab = c_vec.vocabulary_\ndf_bigram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse=True)\n            ).rename(columns={0: 'frequency', 1:'bigram'})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (15,80))\nsns.lineplot(y = df_bigram[\"bigram\"], x = df_bigram[\"frequency\"])\nplt.xticks(rotation = 90)\nplt.ylabel(\"BIGRAM\",fontsize = 50)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nc_vec = CountVectorizer(stop_words=stoplist, ngram_range=(3,3))\n# matrix of ngrams\nngrams = c_vec.fit_transform(data['cleaned_label'])\n# count frequency of ngrams\ncount_values = ngrams.toarray().sum(axis=0)\n# list of ngrams\nvocab = c_vec.vocabulary_\ndf_trigram = pd.DataFrame(sorted([(count_values[i],k) for k,i in vocab.items()], reverse=True)\n            ).rename(columns={0: 'frequency', 1:'trigram'})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_trigram[:30]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (15,80))\nsns.lineplot(y = df_trigram[\"trigram\"], x = df_trigram[\"frequency\"])\nplt.xticks(rotation = 90)\nplt.ylabel(\"TRIGRAM\",fontsize = 50)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}