{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom keras.layers import Dense, SimpleRNN, BatchNormalization\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn import metrics\nfrom sklearn.metrics import accuracy_score\nfrom mlxtend.plotting import plot_confusion_matrix\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport nltk\nimport re\nfrom nltk.tokenize import word_tokenize\nimport string\nimport re\nimport nltk\nnltk.download('stopwords')\nnltk.download('wordnet')\nnltk.download('omw-1.4')\nnltk.download('punkt')\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-19T21:13:10.090830Z","iopub.execute_input":"2022-10-19T21:13:10.091962Z","iopub.status.idle":"2022-10-19T21:13:21.492504Z","shell.execute_reply.started":"2022-10-19T21:13:10.091823Z","shell.execute_reply":"2022-10-19T21:13:21.490026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Reading and checking dataset**","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/goodreads-books-reviews-290312/goodreads_train.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:14:04.009930Z","iopub.execute_input":"2022-10-19T21:14:04.010285Z","iopub.status.idle":"2022-10-19T21:14:30.048687Z","shell.execute_reply.started":"2022-10-19T21:14:04.010260Z","shell.execute_reply":"2022-10-19T21:14:30.047078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:32:02.705697Z","iopub.execute_input":"2022-10-19T21:32:02.706144Z","iopub.status.idle":"2022-10-19T21:32:03.058460Z","shell.execute_reply.started":"2022-10-19T21:32:02.706111Z","shell.execute_reply":"2022-10-19T21:32:03.056948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Resuming dataset**\n    Taking only the columns that interest us. The review and ratings.","metadata":{}},{"cell_type":"code","source":"df = df[[\"rating\", \"review_text\"]]\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:32:05.926326Z","iopub.execute_input":"2022-10-19T21:32:05.926664Z","iopub.status.idle":"2022-10-19T21:32:05.966812Z","shell.execute_reply.started":"2022-10-19T21:32:05.926630Z","shell.execute_reply":"2022-10-19T21:32:05.965248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['rating'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:32:08.129600Z","iopub.execute_input":"2022-10-19T21:32:08.130078Z","iopub.status.idle":"2022-10-19T21:32:08.151467Z","shell.execute_reply.started":"2022-10-19T21:32:08.130054Z","shell.execute_reply":"2022-10-19T21:32:08.149906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# # **Balancing our dataset**\n    I took the amount from the review with less quantity, then I made all the other have the same amount.","metadata":{}},{"cell_type":"code","source":"df = df.groupby('rating').head(28718)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:32:10.347249Z","iopub.execute_input":"2022-10-19T21:32:10.347565Z","iopub.status.idle":"2022-10-19T21:32:10.418341Z","shell.execute_reply.started":"2022-10-19T21:32:10.347540Z","shell.execute_reply":"2022-10-19T21:32:10.417700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['rating'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:32:12.477516Z","iopub.execute_input":"2022-10-19T21:32:12.477851Z","iopub.status.idle":"2022-10-19T21:32:12.492534Z","shell.execute_reply.started":"2022-10-19T21:32:12.477827Z","shell.execute_reply":"2022-10-19T21:32:12.491390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Processing the text**\n    Now we cleaned the reviews from symbols, HTML, URL, punctuation, numbers, emojis and capital letter.  All things that do not influence our interpretation of the data for sentiment analysis.","metadata":{}},{"cell_type":"code","source":"def clean_data(data):\n  #Remove HTML\n  data = re.sub(r'<.*?>', '',  data)\n  #Remove URL\n  data = re.sub(r'http\\S+', '', data)\n  #Remove punctuation\n  data = re.sub(r'[^\\w\\s]', '', data)\n  #Remove numbers\n  data = re.sub(r'\\b\\d+\\b', '', data)\n  #Create a list os emojis\n  emoji_pattern = re.compile(\"[\"\n        u\"\\U0001F600-\\U0001F64F\"\n        u\"\\U0001F300-\\U0001F5FF\"\n        u\"\\U0001F680-\\U0001F6FF\"\n        u\"\\U0001F1E0-\\U0001F1FF\"\n        \"]+\", flags=re.UNICODE)\n  #Remove emojis\n  data = emoji_pattern.sub(r'', data)\n  #Make all letters lower case\n  data = data.lower()\n  return data","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:32:20.750413Z","iopub.execute_input":"2022-10-19T21:32:20.750758Z","iopub.status.idle":"2022-10-19T21:32:20.757187Z","shell.execute_reply.started":"2022-10-19T21:32:20.750724Z","shell.execute_reply":"2022-10-19T21:32:20.756361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['review_text'] = df['review_text'].apply(lambda z: clean_data(z))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:32:23.111171Z","iopub.execute_input":"2022-10-19T21:32:23.111502Z","iopub.status.idle":"2022-10-19T21:32:35.013572Z","shell.execute_reply.started":"2022-10-19T21:32:23.111477Z","shell.execute_reply":"2022-10-19T21:32:35.011721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"       Now i split each review from a big string to a vector of strings for each word.","metadata":{}},{"cell_type":"code","source":"df['review_text'] = df.apply(lambda row: nltk.word_tokenize(row['review_text']), axis=1)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:32:46.030398Z","iopub.execute_input":"2022-10-19T21:32:46.030707Z","iopub.status.idle":"2022-10-19T21:34:24.868556Z","shell.execute_reply.started":"2022-10-19T21:32:46.030684Z","shell.execute_reply":"2022-10-19T21:34:24.866949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now i removed all the stopwords from the english language","metadata":{}},{"cell_type":"code","source":"stop = stopwords.words('english')\n\ndf['review_text'] = df['review_text'].apply(lambda x: [word for word in x if word not in (stop)])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:35:15.873063Z","iopub.execute_input":"2022-10-19T21:35:15.873454Z","iopub.status.idle":"2022-10-19T21:35:54.095862Z","shell.execute_reply.started":"2022-10-19T21:35:15.873431Z","shell.execute_reply":"2022-10-19T21:35:54.094906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  **Lemmatization**\n\n    Lemmatization is basically replacing different forms of the same word for a single one. \n    Ex:\n        words = [\"connects\", \"connected\", \"strange\", \"am\", \"is\"]\n        lemmatized = [\"connect\", \"connect\", \"strange\", \"be\", \"be\"]\n    ","metadata":{}},{"cell_type":"code","source":"nltk.download('averaged_perceptron_tagger')","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:35:54.141822Z","iopub.execute_input":"2022-10-19T21:35:54.142299Z","iopub.status.idle":"2022-10-19T21:35:54.151434Z","shell.execute_reply.started":"2022-10-19T21:35:54.142272Z","shell.execute_reply":"2022-10-19T21:35:54.149064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"    I did a function to get the lemmatized version of each word based on if it is a Adjective, Noun, Verb or Adverb. \n    \n    You may get a different result if you don't take this into account","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import wordnet\n\ndef get_wordnet_pos(word):\n    tag = nltk.pos_tag([word])[0][1][0].upper()\n    tag_dict = {\"J\": wordnet.ADJ,\n                \"N\": wordnet.NOUN,\n                \"V\": wordnet.VERB,\n                \"R\": wordnet.ADV}\n\n    return tag_dict.get(tag, wordnet.NOUN)\n\nfrom nltk.corpus import wordnet\n\ndef lemma_traincorpus(data):\n    lemmatizer=WordNetLemmatizer()\n    out_data= []\n    for words in data:\n        out_data.append(lemmatizer.lemmatize(words, get_wordnet_pos(words)))\n    return out_data","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:35:54.153614Z","iopub.execute_input":"2022-10-19T21:35:54.154006Z","iopub.status.idle":"2022-10-19T21:35:54.168363Z","shell.execute_reply.started":"2022-10-19T21:35:54.153973Z","shell.execute_reply":"2022-10-19T21:35:54.166410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['review_text']=df['review_text'].apply(lambda z: lemma_traincorpus(z))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T21:35:54.169609Z","iopub.execute_input":"2022-10-19T21:35:54.169908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Now you can save this version of the data to run in your own model**\n    Is good to now that if you save the data using to_csv the vector of words can be turn into a string. This can be solved by applaying eval() on the review_text column after re reading the data.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}