{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport nltk\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom nltk.corpus import stopwords, inaugural","metadata":{"execution":{"iopub.status.busy":"2022-06-03T19:55:58.043720Z","iopub.execute_input":"2022-06-03T19:55:58.044963Z","iopub.status.idle":"2022-06-03T19:55:59.716941Z","shell.execute_reply.started":"2022-06-03T19:55:58.044815Z","shell.execute_reply":"2022-06-03T19:55:59.715926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('stopwords')\nnltk.download('punkt')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T19:55:59.718549Z","iopub.execute_input":"2022-06-03T19:55:59.718928Z","iopub.status.idle":"2022-06-03T19:56:00.108605Z","shell.execute_reply.started":"2022-06-03T19:55:59.718896Z","shell.execute_reply":"2022-06-03T19:56:00.107618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndf_train = pd.read_csv('../input/goodreads-books-reviews-290312/goodreads_train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T19:58:48.049766Z","iopub.execute_input":"2022-06-03T19:58:48.050546Z","iopub.status.idle":"2022-06-03T19:59:10.884798Z","shell.execute_reply.started":"2022-06-03T19:58:48.050498Z","shell.execute_reply":"2022-06-03T19:59:10.884116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train['review_text'][0])\nprint(df_train['rating'][0])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:01:05.991003Z","iopub.execute_input":"2022-06-03T20:01:05.991429Z","iopub.status.idle":"2022-06-03T20:01:05.998759Z","shell.execute_reply.started":"2022-06-03T20:01:05.991396Z","shell.execute_reply":"2022-06-03T20:01:05.997992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_rating = df_train[['review_text','rating']]","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:00:25.183571Z","iopub.execute_input":"2022-06-03T20:00:25.183999Z","iopub.status.idle":"2022-06-03T20:00:25.223013Z","shell.execute_reply.started":"2022-06-03T20:00:25.183964Z","shell.execute_reply":"2022-06-03T20:00:25.221984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We create a function to clean a document","metadata":{}},{"cell_type":"code","source":"def clean_text(text):\n    \"\"\"\n    Applies some pre-processing on the given text.\n    Steps :\n    - Removing HTML tags\n    - Removing punctuation\n    - Lowering text\n    \"\"\"\n\n    # remove HTML tags\n    text = re.sub(r'<.*?>', '', text)\n\n    # remove the characters [\\], ['] and [\"]\n    text = re.sub(r\"\\\\\", \"\", text)\n    #text = re.sub(r\"\\'\", \"\", text)\n    text = re.sub(r\"\\\"\", \"\", text)\n\n    # convert text to lowercase\n    text = text.strip().lower()\n\n    # remove all non-ASCII characters:\n    text = re.sub(r'[^\\x00-\\x7f]', r'', text)\n\n    # remove URL\n    text = re.sub(r\"http\\S+\", \"\", text)\n\n    # replace punctuation characters with spaces\n    filters = '!\"\\'#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n'\n    translate_dict = dict((c, \" \") for c in filters)\n    translate_map = str.maketrans(translate_dict)\n    text = text.translate(translate_map)\n    text = \" \".join(text.split())\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:01:26.733294Z","iopub.execute_input":"2022-06-03T20:01:26.734268Z","iopub.status.idle":"2022-06-03T20:01:26.746367Z","shell.execute_reply.started":"2022-06-03T20:01:26.734216Z","shell.execute_reply":"2022-06-03T20:01:26.745075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cleaning_data(X):\n    df_clean = X.apply(lambda x: re.sub(r'\\d+', '', str(x)))\n    return df_clean.apply(lambda x: clean_text(str(x)))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:01:23.649354Z","iopub.execute_input":"2022-06-03T20:01:23.649749Z","iopub.status.idle":"2022-06-03T20:01:23.655687Z","shell.execute_reply.started":"2022-06-03T20:01:23.649720Z","shell.execute_reply":"2022-06-03T20:01:23.654631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###  Visualize # of docs by classes","metadata":{}},{"cell_type":"code","source":"import pandas as pd\npd.options.display.max_columns = 30\npd.options.display.max_rows=200","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:01:58.127652Z","iopub.execute_input":"2022-06-03T20:01:58.128508Z","iopub.status.idle":"2022-06-03T20:01:58.133079Z","shell.execute_reply.started":"2022-06-03T20:01:58.128460Z","shell.execute_reply":"2022-06-03T20:01:58.132139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This shows the number of documents by rating","metadata":{}},{"cell_type":"code","source":"df_train_rating.groupby('rating').count(), df_train_rating.groupby('rating').count().plot.barh()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:02:20.260518Z","iopub.execute_input":"2022-06-03T20:02:20.260940Z","iopub.status.idle":"2022-06-03T20:02:20.934764Z","shell.execute_reply.started":"2022-06-03T20:02:20.260909Z","shell.execute_reply":"2022-06-03T20:02:20.933649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Review examples by ratings\n\nWe manually review texts for each class or rating","metadata":{}},{"cell_type":"code","source":"df = df_train_rating[df_train_rating['rating']==0].reset_index()\nprint(df['review_text'][3])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_train_rating[df_train_rating['rating']==1].reset_index()\nprint(df['review_text'][4])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_train_rating[df_train_rating['rating']==2].reset_index()\nprint(df['review_text'][4])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_train_rating[df_train_rating['rating']==3].reset_index()\nprint(df['review_text'][4])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_train_rating[df_train_rating['rating']==4].reset_index()\nprint(df['review_text'][3])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_train_rating[df_train_rating['rating']==5].reset_index()\nprint(df['review_text'][2])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clean and Tokenize","metadata":{}},{"cell_type":"markdown","source":"Function to extend contractions","metadata":{}},{"cell_type":"code","source":"#https://www.analyticsvidhya.com/blog/2020/04/beginners-guide-exploratory-data-analysis-text-data/\n# Dictionary of English Contractions\ncontractions_dict = { \"ain't\": \"are not\",\"'s\":\" is\",\"aren't\": \"are not\",\n                     \"can't\": \"cannot\",\"can't've\": \"cannot have\",\n                     \"'cause\": \"because\",\"could've\": \"could have\",\"couldn't\": \"could not\",\n                     \"couldn't've\": \"could not have\", \"didn't\": \"did not\",\"doesn't\": \"does not\",\n                     \"don't\": \"do not\",\"hadn't\": \"had not\",\"hadn't've\": \"had not have\",\n                     \"hasn't\": \"has not\",\"haven't\": \"have not\",\"he'd\": \"he would\",\n                     \"he'd've\": \"he would have\",\"he'll\": \"he will\", \"he'll've\": \"he will have\",\n                     \"how'd\": \"how did\",\"how'd'y\": \"how do you\",\"how'll\": \"how will\",\n                     \"I'd\": \"I would\", \"I'd've\": \"I would have\",\"I'll\": \"I will\",\n                     \"I'll've\": \"I will have\",\"I'm\": \"I am\",\"I've\": \"I have\", \"isn't\": \"is not\",\n                     \"it'd\": \"it would\",\"it'd've\": \"it would have\",\"it'll\": \"it will\",\n                     \"it'll've\": \"it will have\", \"let's\": \"let us\",\"ma'am\": \"madam\",\n                     \"mayn't\": \"may not\",\"might've\": \"might have\",\"mightn't\": \"might not\", \n                     \"mightn't've\": \"might not have\",\"must've\": \"must have\",\"mustn't\": \"must not\",\n                     \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n                     \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\"oughtn't\": \"ought not\",\n                     \"oughtn't've\": \"ought not have\",\"shan't\": \"shall not\",\"sha'n't\": \"shall not\",\n                     \"shan't've\": \"shall not have\",\"she'd\": \"she would\",\"she'd've\": \"she would have\",\n                     \"she'll\": \"she will\", \"she'll've\": \"she will have\",\"should've\": \"should have\",\n                     \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\"so've\": \"so have\",\n                     \"that'd\": \"that would\",\"that'd've\": \"that would have\", \"there'd\": \"there would\",\n                     \"there'd've\": \"there would have\", \"they'd\": \"they would\",\n                     \"they'd've\": \"they would have\",\"they'll\": \"they will\",\n                     \"they'll've\": \"they will have\", \"they're\": \"they are\",\"they've\": \"they have\",\n                     \"to've\": \"to have\",\"wasn't\": \"was not\",\"we'd\": \"we would\",\n                     \"we'd've\": \"we would have\",\"we'll\": \"we will\",\"we'll've\": \"we will have\",\n                     \"we're\": \"we are\",\"we've\": \"we have\", \"weren't\": \"were not\",\"what'll\": \"what will\",\n                     \"what'll've\": \"what will have\",\"what're\": \"what are\", \"what've\": \"what have\",\n                     \"when've\": \"when have\",\"where'd\": \"where did\", \"where've\": \"where have\",\n                     \"who'll\": \"who will\",\"who'll've\": \"who will have\",\"who've\": \"who have\",\n                     \"why've\": \"why have\",\"will've\": \"will have\",\"won't\": \"will not\",\n                     \"won't've\": \"will not have\", \"would've\": \"would have\",\"wouldn't\": \"would not\",\n                     \"wouldn't've\": \"would not have\",\"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n                     \"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\n                     \"y'all've\": \"you all have\", \"you'd\": \"you would\",\"you'd've\": \"you would have\",\n                     \"you'll\": \"you will\",\"you'll've\": \"you will have\", \"you're\": \"you are\",\n                     \"you've\": \"you have\",\n                     \"couldna\": \"could not have\"}\n\n# Regular expression for finding contractions\ncontractions_re=re.compile('(%s)' % '|'.join(contractions_dict.keys()))\n\n# Function for expanding contractions\ndef expand_contractions(text,contractions_dict=contractions_dict):\n  def replace(match):\n    return contractions_dict[match.group(0)]\n  return contractions_re.sub(replace, text)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:07:02.487441Z","iopub.execute_input":"2022-06-03T20:07:02.487817Z","iopub.status.idle":"2022-06-03T20:07:02.509596Z","shell.execute_reply.started":"2022-06-03T20:07:02.487788Z","shell.execute_reply":"2022-06-03T20:07:02.508569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extract data sample \n\nWe use for now a subset of the data to test our worflow .","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndf_train_samples, _ = train_test_split(df_train_rating,train_size=0.05,stratify=df_train_rating['rating'],random_state=42)\ndf_train_samples= df_train_samples.reset_index(drop=True)\nprint(len(df_train_samples))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:15:50.525085Z","iopub.execute_input":"2022-06-03T20:15:50.525502Z","iopub.status.idle":"2022-06-03T20:15:51.220280Z","shell.execute_reply.started":"2022-06-03T20:15:50.525466Z","shell.execute_reply":"2022-06-03T20:15:51.219300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_samples.groupby('rating').count()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:15:51.258133Z","iopub.execute_input":"2022-06-03T20:15:51.258539Z","iopub.status.idle":"2022-06-03T20:15:51.287958Z","shell.execute_reply.started":"2022-06-03T20:15:51.258506Z","shell.execute_reply":"2022-06-03T20:15:51.286917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data cleaning ","metadata":{}},{"cell_type":"code","source":"text = df_train_samples['review_text'][0]\n# print(text)\nprint(\"expand:\",expand_contractions(text=text))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:15:52.028180Z","iopub.execute_input":"2022-06-03T20:15:52.028588Z","iopub.status.idle":"2022-06-03T20:15:52.035860Z","shell.execute_reply.started":"2022-06-03T20:15:52.028554Z","shell.execute_reply":"2022-06-03T20:15:52.034948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The function below performed those transformations to each document:\n* remove numbers\n* clean text\n* expand contraction\n* tokenized\n* remove stopwords and puctuation ","metadata":{}},{"cell_type":"code","source":"import string\nfrom nltk import tokenize\n\nuseless_words = nltk.corpus.stopwords.words(\"english\") + list(string.punctuation)+['``',\"''\",'...','a',\"--\",\"***\",'http',\"**\"]\n\ndef review_tokenize(text):\n  return [word.lower() for word in word_tokenize(text)]\n\n\ndef remove_stopswords_punctuation(tokens):\n  return [w.lower() for w in tokens if w not in useless_words]\n\ndef tokenize_filtered(text):\n  x = re.sub(r'\\d+', '', str(text))\n  x = clean_text(x)\n  x = expand_contractions(x)\n  x = review_tokenize(x)\n  x = remove_stopswords_punctuation(x)\n\n  x = ' '.join(x)\n\n  return x\n\nt = tokenize_filtered(text)\nprint(t)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:15:52.694799Z","iopub.execute_input":"2022-06-03T20:15:52.695216Z","iopub.status.idle":"2022-06-03T20:15:52.718012Z","shell.execute_reply.started":"2022-06-03T20:15:52.695182Z","shell.execute_reply":"2022-06-03T20:15:52.717044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_train_samples['clean_text'] = df_train_samples['review_text'].apply(lambda x: tokenize_filtered(x))\ndf_train_samples","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:15:53.025164Z","iopub.execute_input":"2022-06-03T20:15:53.026116Z","iopub.status.idle":"2022-06-03T20:17:41.159674Z","shell.execute_reply.started":"2022-06-03T20:15:53.026074Z","shell.execute_reply":"2022-06-03T20:17:41.158568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len_seq = max([len(doc) for doc in df_train_samples['clean_text']])\nprint(\"The maximum document length is (# of characters in a document):\", max_len_seq)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:17:41.161364Z","iopub.execute_input":"2022-06-03T20:17:41.161717Z","iopub.status.idle":"2022-06-03T20:17:41.185771Z","shell.execute_reply.started":"2022-06-03T20:17:41.161686Z","shell.execute_reply":"2022-06-03T20:17:41.184760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# What to do with does ? Filter for now\ndf_train_samples = df_train_samples[df_train_samples['clean_text'] !=''].copy()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:17:41.186968Z","iopub.execute_input":"2022-06-03T20:17:41.187780Z","iopub.status.idle":"2022-06-03T20:17:41.228744Z","shell.execute_reply.started":"2022-06-03T20:17:41.187741Z","shell.execute_reply":"2022-06-03T20:17:41.227617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Document statistics\n\n* review length \n* word counts\n* average word length \n* sentiment polarity ","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nfrom textblob import TextBlob\n\ndf_train_samples['review_len'] = df_train_samples['clean_text'].str.len()\ndf_train_samples['word_cnt'] = df_train_samples['clean_text'].apply(lambda x: len(word_tokenize(x)))\ndf_train_samples['polarity'] = df_train_samples['clean_text'].apply(lambda x: TextBlob(x).sentiment.polarity)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:17:41.230836Z","iopub.execute_input":"2022-06-03T20:17:41.231197Z","iopub.status.idle":"2022-06-03T20:19:05.267961Z","shell.execute_reply.started":"2022-06-03T20:17:41.231168Z","shell.execute_reply":"2022-06-03T20:19:05.267149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def average_word_len(x):\n  word_len = [len(word) for word in x]\n  return sum(word_len)/len(word_len)\n\ndf_train_samples['average_word_len'] = df_train_samples['clean_text'].apply(lambda x: average_word_len(word_tokenize(x)))\ndf_train_samples.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:05.268981Z","iopub.execute_input":"2022-06-03T20:19:05.269847Z","iopub.status.idle":"2022-06-03T20:19:50.830112Z","shell.execute_reply.started":"2022-06-03T20:19:05.269811Z","shell.execute_reply":"2022-06-03T20:19:50.829005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Histogram review length","metadata":{}},{"cell_type":"code","source":"g = sns.FacetGrid(df_train_samples, col=\"rating\")\ng.map(sns.histplot, \"review_len\")","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:50.832572Z","iopub.execute_input":"2022-06-03T20:19:50.832954Z","iopub.status.idle":"2022-06-03T20:19:53.021825Z","shell.execute_reply.started":"2022-06-03T20:19:50.832914Z","shell.execute_reply":"2022-06-03T20:19:53.020942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Boxplot review length","metadata":{}},{"cell_type":"code","source":"sns.boxplot(data=df_train_samples,y='review_len',x='rating')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:53.023093Z","iopub.execute_input":"2022-06-03T20:19:53.023396Z","iopub.status.idle":"2022-06-03T20:19:53.269752Z","shell.execute_reply.started":"2022-06-03T20:19:53.023369Z","shell.execute_reply":"2022-06-03T20:19:53.268792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Boxplot words count length","metadata":{}},{"cell_type":"code","source":"sns.boxplot(data=df_train_samples,y='word_cnt',x='rating')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:53.270995Z","iopub.execute_input":"2022-06-03T20:19:53.271331Z","iopub.status.idle":"2022-06-03T20:19:53.526555Z","shell.execute_reply.started":"2022-06-03T20:19:53.271303Z","shell.execute_reply":"2022-06-03T20:19:53.525670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Boxplot average word length","metadata":{}},{"cell_type":"code","source":"sns.boxplot(data=df_train_samples,y='average_word_len',x='rating')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:53.528067Z","iopub.execute_input":"2022-06-03T20:19:53.528707Z","iopub.status.idle":"2022-06-03T20:19:53.772030Z","shell.execute_reply.started":"2022-06-03T20:19:53.528676Z","shell.execute_reply":"2022-06-03T20:19:53.771053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### Boxplot Text polarity","metadata":{}},{"cell_type":"code","source":"sns.boxplot(data=df_train_samples,y='polarity',x='rating')","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:53.775026Z","iopub.execute_input":"2022-06-03T20:19:53.775797Z","iopub.status.idle":"2022-06-03T20:19:54.043974Z","shell.execute_reply.started":"2022-06-03T20:19:53.775763Z","shell.execute_reply":"2022-06-03T20:19:54.043169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n#tokenize_filtered\ndef get_top_ngrams(corpus,n_gram=1,rating=None):\n  if rating is not None:\n    corpus = corpus[corpus['rating']==rating]\n  #print(corpus.head())\n  vectorizer = CountVectorizer(stop_words='english',ngram_range=(n_gram,n_gram)).fit(corpus['clean_text'])\n  bag_of_words = vectorizer.transform(corpus['clean_text'])\n  sum_words = bag_of_words.sum(axis=0)\n  word_freq = [(word,sum_words[0,idx]) for word,idx in vectorizer.vocabulary_.items()]\n  word_freq = sorted(word_freq,key= lambda x:x[1],reverse=True)\n\n  return word_freq\n","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:54.045190Z","iopub.execute_input":"2022-06-03T20:19:54.046290Z","iopub.status.idle":"2022-06-03T20:19:54.054497Z","shell.execute_reply.started":"2022-06-03T20:19:54.046252Z","shell.execute_reply":"2022-06-03T20:19:54.053468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\nsns.set_theme(style=\"whitegrid\")\n#sns.set(rc={\"figure.figsize\":(10, 8)})\n\ndef plot_top_ngrams(bow,top_n=20):\n  plt.figure(figsize=(6,6))\n  df_word_cnt = pd.DataFrame(data=bow,columns=['word', 'cnt'])\n  return sns.barplot(x='cnt',y='word',data=df_word_cnt[:top_n],palette=\"Blues_r\")\n\n\ndef plot_top_ngrams_by_rating(data,n_gram=1,top_n=10):\n  fig , axes  = plt.subplots(2,3, figsize=(20,10))\n  for i in range(6):\n    ax = plt.subplot(2, 3, i + 1)\n    ax.set_title(f\"rating {i}\")\n    ax.set_ylabel(\"\")\n    bow = get_top_ngrams(data,n_gram=n_gram,rating=i)\n    df_word_cnt = pd.DataFrame(data=bow,columns=['word', 'cnt'])\n    sns.barplot(ax=ax,x='cnt',y='word',data=df_word_cnt[:top_n],palette=\"Blues_r\")","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:54.055706Z","iopub.execute_input":"2022-06-03T20:19:54.056054Z","iopub.status.idle":"2022-06-03T20:19:54.070195Z","shell.execute_reply.started":"2022-06-03T20:19:54.056019Z","shell.execute_reply":"2022-06-03T20:19:54.069243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Top 20 Unigrams","metadata":{}},{"cell_type":"code","source":"plot_top_ngrams(get_top_ngrams(df_train_samples))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:19:54.071656Z","iopub.execute_input":"2022-06-03T20:19:54.072169Z","iopub.status.idle":"2022-06-03T20:20:05.199104Z","shell.execute_reply.started":"2022-06-03T20:19:54.072124Z","shell.execute_reply":"2022-06-03T20:20:05.198225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_top_ngrams_by_rating(df_train_samples,top_n=20)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:20:05.200204Z","iopub.execute_input":"2022-06-03T20:20:05.200503Z","iopub.status.idle":"2022-06-03T20:20:18.233775Z","shell.execute_reply.started":"2022-06-03T20:20:05.200476Z","shell.execute_reply":"2022-06-03T20:20:18.233093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Top 20 Bigrams","metadata":{}},{"cell_type":"code","source":"plot_top_ngrams( get_top_ngrams(df_train_samples,n_gram=2))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:21:34.872858Z","iopub.execute_input":"2022-06-03T20:21:34.873375Z","iopub.status.idle":"2022-06-03T20:22:10.344829Z","shell.execute_reply.started":"2022-06-03T20:21:34.873341Z","shell.execute_reply":"2022-06-03T20:22:10.343484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_top_ngrams_by_rating(df_train_samples,n_gram=2,top_n=20)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:22:10.347198Z","iopub.execute_input":"2022-06-03T20:22:10.347548Z","iopub.status.idle":"2022-06-03T20:22:45.738526Z","shell.execute_reply.started":"2022-06-03T20:22:10.347512Z","shell.execute_reply":"2022-06-03T20:22:45.737346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Top 20 Trigrams","metadata":{}},{"cell_type":"code","source":"plot_top_ngrams( get_top_ngrams(df_train_samples,n_gram=3))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:23:31.433921Z","iopub.execute_input":"2022-06-03T20:23:31.434243Z","iopub.status.idle":"2022-06-03T20:24:16.633171Z","shell.execute_reply.started":"2022-06-03T20:23:31.434215Z","shell.execute_reply":"2022-06-03T20:24:16.632129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_top_ngrams_by_rating(df_train_samples,n_gram=3,top_n=20)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:24:59.384839Z","iopub.execute_input":"2022-06-03T20:24:59.385264Z","iopub.status.idle":"2022-06-03T20:25:41.979205Z","shell.execute_reply.started":"2022-06-03T20:24:59.385231Z","shell.execute_reply":"2022-06-03T20:25:41.978161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize TFIDF documents by rating","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nvectorizer = TfidfVectorizer(stop_words='english',ngram_range=(1,3)).fit(df_train_samples['clean_text'])\nbag_of_words = vectorizer.transform(df_train_samples['clean_text'])","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:26:50.872431Z","iopub.execute_input":"2022-06-03T20:26:50.872762Z","iopub.status.idle":"2022-06-03T20:28:00.964404Z","shell.execute_reply.started":"2022-06-03T20:26:50.872734Z","shell.execute_reply":"2022-06-03T20:28:00.963325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA, TruncatedSVD\nimport matplotlib.patches as mpatches\nimport matplotlib\n\ndef plot_vectorizer(data, labels, savepath=\"PCA_demo.csv\", plot=True):\n  lsa = TruncatedSVD(n_components=2)\n  lsa.fit(data)\n  lsa_scores = lsa.transform(data)\n  color_mapper = {label:idx for idx,label in enumerate(set(labels))}\n  color_column = [color_mapper[label] for label in labels]\n  #print(color_column)\n  colors = ['yellow','orange','blue','green','brown','red']\n  #colors = ['orange','blue']\n  if plot:\n    plt.scatter(lsa_scores[:,0], lsa_scores[:,1], s=8, alpha=.8, c=labels, cmap=matplotlib.colors.ListedColormap(colors))\n    yellow_patch = mpatches.Patch(color='yellow', label='0')\n    orange_patch = mpatches.Patch(color='orange', label='1')\n    blue_patch = mpatches.Patch(color='blue', label='2')\n    green_patch = mpatches.Patch(color='green', label='3')\n    brown_patch = mpatches.Patch(color='brown', label='4')\n    red_patch = mpatches.Patch(color='red', label='5')\n    #black_patch = mpatches.Patch(color='black', label='6')\n    plt.legend(handles=[yellow_patch,orange_patch, blue_patch,green_patch,brown_patch,red_patch], prop={'size': 15})\n    #plt.legend(handles=[orange_patch, blue_patch], prop={'size': 30})","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:28:00.965725Z","iopub.execute_input":"2022-06-03T20:28:00.966087Z","iopub.status.idle":"2022-06-03T20:28:01.007516Z","shell.execute_reply.started":"2022-06-03T20:28:00.966056Z","shell.execute_reply":"2022-06-03T20:28:01.006640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(9, 9))          \nplot_vectorizer(bag_of_words, df_train_samples['rating'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:28:01.009901Z","iopub.execute_input":"2022-06-03T20:28:01.010266Z","iopub.status.idle":"2022-06-03T20:28:26.621860Z","shell.execute_reply.started":"2022-06-03T20:28:01.010234Z","shell.execute_reply":"2022-06-03T20:28:26.620865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Embeddings\n\n* https://www.kaggle.com/code/christofhenkel/how-to-preprocessing-when-using-embeddings/notebook","metadata":{}},{"cell_type":"markdown","source":"### Embeddings Coverage","metadata":{}},{"cell_type":"code","source":"vectorizer = CountVectorizer(stop_words='english',ngram_range=(1,1)).fit(df_train_samples['clean_text'])\nbag_of_words = vectorizer.transform(df_train_samples['clean_text'])\nvocabs = vectorizer.vocabulary_","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:31:23.492134Z","iopub.execute_input":"2022-06-03T20:31:23.492624Z","iopub.status.idle":"2022-06-03T20:31:34.050654Z","shell.execute_reply.started":"2022-06-03T20:31:23.492588Z","shell.execute_reply":"2022-06-03T20:31:34.049607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.models import KeyedVectors\nimport gensim.downloader as api\n\nprint(list(api.info()['models'].keys()))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:31:34.052637Z","iopub.execute_input":"2022-06-03T20:31:34.053139Z","iopub.status.idle":"2022-06-03T20:31:34.637045Z","shell.execute_reply.started":"2022-06-03T20:31:34.053094Z","shell.execute_reply":"2022-06-03T20:31:34.636030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_vectors = api.load(\"glove-wiki-gigaword-50\") \n#word_vectors = api.load(\"word2vec-google-news-300\") \nglove_index = dict(zip(word_vectors.index_to_key, range(len(word_vectors.index_to_key,))))","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:32:46.909605Z","iopub.execute_input":"2022-06-03T20:32:46.910205Z","iopub.status.idle":"2022-06-03T20:33:09.536403Z","shell.execute_reply.started":"2022-06-03T20:32:46.910164Z","shell.execute_reply":"2022-06-03T20:33:09.535397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def embedding_coverage(vocabs,embeddings_index):\n  cnt = 0\n  oov = {}\n  for word, index in vocabs.items():\n    if word in embeddings_index:\n      cnt+=1\n    else:\n      oov[word] = index\n  \n  print('Found embeddings for {:.2%} of vocab'.format(cnt/len(vocabs)))\n  #oov = sorted(oov.items(), key=lambda x: x[1],reverse=False)\n\n  return oov\n\noov = embedding_coverage(vocabs,glove_index)","metadata":{"execution":{"iopub.status.busy":"2022-06-03T20:33:15.457815Z","iopub.execute_input":"2022-06-03T20:33:15.458700Z","iopub.status.idle":"2022-06-03T20:33:15.514768Z","shell.execute_reply.started":"2022-06-03T20:33:15.458649Z","shell.execute_reply":"2022-06-03T20:33:15.513765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That's it for now, we plan to share more later. Please share any comments, suggestions or feedback!","metadata":{}},{"cell_type":"markdown","source":"### References","metadata":{}},{"cell_type":"markdown","source":"* https://www.kaggle.com/code/theoviel/improve-your-score-with-some-text-preprocessing/notebook\n* https://www.kaggle.com/code/christofhenkel/how-to-preprocessing-when-using-embeddings/notebook\n* https://www.kaggle.com/code/wil2210/eda-nlp-ml/notebook\n* https://www.kaggle.com/code/vbmokin/nlp-eda-bag-of-words-tf-idf-glove-bert/notebook\n* https://www.kaggle.com/code/parulpandey/eda-and-preprocessing-for-bert/notebook\n","metadata":{}}]}