{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-12T07:43:29.409261Z","iopub.execute_input":"2022-12-12T07:43:29.410173Z","iopub.status.idle":"2022-12-12T07:43:29.428891Z","shell.execute_reply.started":"2022-12-12T07:43:29.410090Z","shell.execute_reply":"2022-12-12T07:43:29.427523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"user_id - Id of user\n\nbook_id - Id of Book\n\nreview_id - Id of review\n\nrating - rating from 0 to 5\n\nreview_text - review text\n\ndate_added - date added\n\ndate_updated - date updated\n\nread_at - read at\n\nstarted_at - started at\n\nn_votes - no. of votes\n\nn_comments - no. of comments","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"*  Import required libraries\n*  load the data\n*  Perform EDA\n*  Remove stop words and  apply stemming\n*  Convert the text data into vectors\n*  Split data into training and testing\n*  Define our model\n*  Test the model on testing data\n*  Check the accuracy\n*  Test on the custom data","metadata":{}},{"cell_type":"markdown","source":"Import required library for use","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import WordNetLemmatizer\nimport re\nfrom bs4 import BeautifulSoup\nimport re, string, unicodedata","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:43:29.431668Z","iopub.execute_input":"2022-12-12T07:43:29.433300Z","iopub.status.idle":"2022-12-12T07:43:29.443915Z","shell.execute_reply.started":"2022-12-12T07:43:29.433242Z","shell.execute_reply":"2022-12-12T07:43:29.442435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Training data preprocessing**","metadata":{}},{"cell_type":"code","source":"train_book=pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:43:29.446064Z","iopub.execute_input":"2022-12-12T07:43:29.447044Z","iopub.status.idle":"2022-12-12T07:43:43.196039Z","shell.execute_reply.started":"2022-12-12T07:43:29.446990Z","shell.execute_reply":"2022-12-12T07:43:43.194653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_book=pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:43:43.198621Z","iopub.execute_input":"2022-12-12T07:43:43.198979Z","iopub.status.idle":"2022-12-12T07:44:01.564790Z","shell.execute_reply.started":"2022-12-12T07:43:43.198945Z","shell.execute_reply":"2022-12-12T07:44:01.563910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.566250Z","iopub.execute_input":"2022-12-12T07:44:01.566841Z","iopub.status.idle":"2022-12-12T07:44:01.588707Z","shell.execute_reply.started":"2022-12-12T07:44:01.566806Z","shell.execute_reply":"2022-12-12T07:44:01.587576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will predict the best book or review on basis of rating and other important features.","metadata":{}},{"cell_type":"code","source":"train_book.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.590663Z","iopub.execute_input":"2022-12-12T07:44:01.591031Z","iopub.status.idle":"2022-12-12T07:44:01.601917Z","shell.execute_reply.started":"2022-12-12T07:44:01.590999Z","shell.execute_reply":"2022-12-12T07:44:01.600799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book.columns","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.603662Z","iopub.execute_input":"2022-12-12T07:44:01.604344Z","iopub.status.idle":"2022-12-12T07:44:01.614324Z","shell.execute_reply.started":"2022-12-12T07:44:01.604301Z","shell.execute_reply":"2022-12-12T07:44:01.613155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book.corr()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.615742Z","iopub.execute_input":"2022-12-12T07:44:01.616885Z","iopub.status.idle":"2022-12-12T07:44:01.737470Z","shell.execute_reply.started":"2022-12-12T07:44:01.616852Z","shell.execute_reply":"2022-12-12T07:44:01.736463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book['review_text']","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.738744Z","iopub.execute_input":"2022-12-12T07:44:01.739606Z","iopub.status.idle":"2022-12-12T07:44:01.749643Z","shell.execute_reply.started":"2022-12-12T07:44:01.739573Z","shell.execute_reply":"2022-12-12T07:44:01.748398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book.drop(['user_id','review_id','date_added','date_updated','read_at','started_at'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.754289Z","iopub.execute_input":"2022-12-12T07:44:01.754654Z","iopub.status.idle":"2022-12-12T07:44:01.808955Z","shell.execute_reply.started":"2022-12-12T07:44:01.754622Z","shell.execute_reply":"2022-12-12T07:44:01.808160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.810197Z","iopub.execute_input":"2022-12-12T07:44:01.811081Z","iopub.status.idle":"2022-12-12T07:44:01.827306Z","shell.execute_reply.started":"2022-12-12T07:44:01.811043Z","shell.execute_reply":"2022-12-12T07:44:01.826008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book.corr()#find the correlation between the respective columns to be involved in prediction.","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.829401Z","iopub.execute_input":"2022-12-12T07:44:01.829865Z","iopub.status.idle":"2022-12-12T07:44:01.940661Z","shell.execute_reply.started":"2022-12-12T07:44:01.829822Z","shell.execute_reply":"2022-12-12T07:44:01.939498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique ratings\npd.unique(train_book['rating'])","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.942302Z","iopub.execute_input":"2022-12-12T07:44:01.943365Z","iopub.status.idle":"2022-12-12T07:44:01.960228Z","shell.execute_reply.started":"2022-12-12T07:44:01.943317Z","shell.execute_reply":"2022-12-12T07:44:01.959156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book['rating'].value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.962787Z","iopub.execute_input":"2022-12-12T07:44:01.965476Z","iopub.status.idle":"2022-12-12T07:44:01.984495Z","shell.execute_reply.started":"2022-12-12T07:44:01.965429Z","shell.execute_reply":"2022-12-12T07:44:01.982958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.unique(train_book['n_comments'])","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:01.985958Z","iopub.execute_input":"2022-12-12T07:44:01.986897Z","iopub.status.idle":"2022-12-12T07:44:02.000908Z","shell.execute_reply.started":"2022-12-12T07:44:01.986863Z","shell.execute_reply":"2022-12-12T07:44:02.000126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Categorize good book and average and not recommeded book on basis of reviews andratings.","metadata":{}},{"cell_type":"code","source":"sns.countplot(data=train_book,\n              x='rating',\n              order=train_book.rating.value_counts().index).set(title='Ratings of all books')\n","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:02.002142Z","iopub.execute_input":"2022-12-12T07:44:02.002464Z","iopub.status.idle":"2022-12-12T07:44:02.365347Z","shell.execute_reply.started":"2022-12-12T07:44:02.002435Z","shell.execute_reply":"2022-12-12T07:44:02.364150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#importing nlp library\nimport nltk\n#Stop words present in the library\nstopwords = nltk.corpus.stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:02.366934Z","iopub.execute_input":"2022-12-12T07:44:02.367857Z","iopub.status.idle":"2022-12-12T07:44:02.375183Z","shell.execute_reply.started":"2022-12-12T07:44:02.367821Z","shell.execute_reply":"2022-12-12T07:44:02.374108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install wordcloud  ","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:02.376657Z","iopub.execute_input":"2022-12-12T07:44:02.377462Z","iopub.status.idle":"2022-12-12T07:44:15.723580Z","shell.execute_reply.started":"2022-12-12T07:44:02.377429Z","shell.execute_reply":"2022-12-12T07:44:15.722194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport nltk.corpus\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nstop_words = stopwords.words('english')\ntrain_book['review_text']=train_book['review_text'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop_words)]))","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:44:15.725309Z","iopub.execute_input":"2022-12-12T07:44:15.725700Z","iopub.status.idle":"2022-12-12T07:51:01.653578Z","shell.execute_reply.started":"2022-12-12T07:44:15.725662Z","shell.execute_reply":"2022-12-12T07:51:01.652326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Make text data as lower**","metadata":{}},{"cell_type":"code","source":"train_book['review_text']= train_book['review_text'].apply(lambda x: x.lower())","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:51:01.655351Z","iopub.execute_input":"2022-12-12T07:51:01.655696Z","iopub.status.idle":"2022-12-12T07:51:03.133783Z","shell.execute_reply.started":"2022-12-12T07:51:01.655664Z","shell.execute_reply":"2022-12-12T07:51:03.132549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:51:03.135430Z","iopub.execute_input":"2022-12-12T07:51:03.135866Z","iopub.status.idle":"2022-12-12T07:51:03.153650Z","shell.execute_reply.started":"2022-12-12T07:51:03.135833Z","shell.execute_reply":"2022-12-12T07:51:03.152493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Import punctuation library**","metadata":{}},{"cell_type":"code","source":"#library that contains punctuation\nimport string\nstring.punctuation","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:51:03.155396Z","iopub.execute_input":"2022-12-12T07:51:03.155732Z","iopub.status.idle":"2022-12-12T07:51:03.162611Z","shell.execute_reply.started":"2022-12-12T07:51:03.155702Z","shell.execute_reply":"2022-12-12T07:51:03.161297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Remove Punctuation**","metadata":{}},{"cell_type":"code","source":"# Define the function to remove the punctuation\ndef remove_punctuations(text):\n    for punctuation in string.punctuation:\n        text = text.replace(punctuation, '')\n    return text\n# Apply to the DF series\ntrain_book['review_text'] =train_book['review_text'].apply(remove_punctuations)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:51:03.164263Z","iopub.execute_input":"2022-12-12T07:51:03.165185Z","iopub.status.idle":"2022-12-12T07:51:34.175764Z","shell.execute_reply.started":"2022-12-12T07:51:03.165140Z","shell.execute_reply":"2022-12-12T07:51:34.174574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Remove the urls**","metadata":{}},{"cell_type":"code","source":"train_book['review_text'] =train_book['review_text'].replace(r'http\\S+', '', regex=True).replace(r'www\\S+', '', regex=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:51:34.177028Z","iopub.execute_input":"2022-12-12T07:51:34.177390Z","iopub.status.idle":"2022-12-12T07:51:38.086217Z","shell.execute_reply.started":"2022-12-12T07:51:34.177359Z","shell.execute_reply":"2022-12-12T07:51:38.085330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Remove the numbers from the text data**","metadata":{}},{"cell_type":"code","source":"train_book['review_text'] =train_book['review_text'].str.replace('\\d+', '')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:51:38.089304Z","iopub.execute_input":"2022-12-12T07:51:38.090091Z","iopub.status.idle":"2022-12-12T07:51:51.799848Z","shell.execute_reply.started":"2022-12-12T07:51:38.090043Z","shell.execute_reply":"2022-12-12T07:51:51.798859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Do the tokenization operation on text data**","metadata":{}},{"cell_type":"code","source":"\ndef tokenize_data(text):\n    tokens=re.split('w+',text)\n    return tokens\ntrain_book['review_text']=train_book['review_text'].apply(lambda x : tokenize_data(x))","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:51:51.801696Z","iopub.execute_input":"2022-12-12T07:51:51.802155Z","iopub.status.idle":"2022-12-12T07:52:06.983584Z","shell.execute_reply.started":"2022-12-12T07:51:51.802124Z","shell.execute_reply":"2022-12-12T07:52:06.982606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#importing the Stemming function from nltk library\nfrom nltk.stem.porter import PorterStemmer\n#defining the object for stemming\nporter_stemmer = PorterStemmer()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:52:06.993596Z","iopub.execute_input":"2022-12-12T07:52:06.994026Z","iopub.status.idle":"2022-12-12T07:52:07.024662Z","shell.execute_reply.started":"2022-12-12T07:52:06.993993Z","shell.execute_reply":"2022-12-12T07:52:07.023141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stemmer(text):\n    data_steemer=[porter_stemmer.stem(word)for word in text]\n    return data_steemer\ntrain_book['review_text']=train_book['review_text'].apply(lambda x:stemmer(x))","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:52:07.030273Z","iopub.execute_input":"2022-12-12T07:52:07.031337Z","iopub.status.idle":"2022-12-12T07:56:59.156288Z","shell.execute_reply.started":"2022-12-12T07:52:07.031298Z","shell.execute_reply":"2022-12-12T07:56:59.155441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('wordnet')\nnltk.download('omw-1.4')","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:56:59.162288Z","iopub.execute_input":"2022-12-12T07:56:59.162948Z","iopub.status.idle":"2022-12-12T07:56:59.628350Z","shell.execute_reply.started":"2022-12-12T07:56:59.162911Z","shell.execute_reply":"2022-12-12T07:56:59.627231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book['review_text'] = [','.join(map(str, l)) for l in train_book['review_text']]","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:56:59.629563Z","iopub.execute_input":"2022-12-12T07:56:59.629890Z","iopub.status.idle":"2022-12-12T07:57:02.174857Z","shell.execute_reply.started":"2022-12-12T07:56:59.629860Z","shell.execute_reply":"2022-12-12T07:57:02.173774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:57:02.177804Z","iopub.execute_input":"2022-12-12T07:57:02.178157Z","iopub.status.idle":"2022-12-12T07:57:02.193434Z","shell.execute_reply.started":"2022-12-12T07:57:02.178127Z","shell.execute_reply":"2022-12-12T07:57:02.192604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Sentiment Analysis using VADER library**\n","metadata":{}},{"cell_type":"code","source":"from nltk.sentiment.vader import SentimentIntensityAnalyzer","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:57:02.194667Z","iopub.execute_input":"2022-12-12T07:57:02.194966Z","iopub.status.idle":"2022-12-12T07:57:02.204598Z","shell.execute_reply.started":"2022-12-12T07:57:02.194939Z","shell.execute_reply":"2022-12-12T07:57:02.203466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"analyzer = SentimentIntensityAnalyzer()","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:57:02.206034Z","iopub.execute_input":"2022-12-12T07:57:02.206415Z","iopub.status.idle":"2022-12-12T07:57:02.228117Z","shell.execute_reply.started":"2022-12-12T07:57:02.206360Z","shell.execute_reply":"2022-12-12T07:57:02.227243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****","metadata":{}},{"cell_type":"markdown","source":"**Create negative polarity score for each row**","metadata":{}},{"cell_type":"code","source":"train_book['neg'] = [analyzer.polarity_scores(x)['neg'] for x in train_book['review_text']]","metadata":{"execution":{"iopub.status.busy":"2022-12-12T07:57:02.229350Z","iopub.execute_input":"2022-12-12T07:57:02.229831Z","iopub.status.idle":"2022-12-12T08:23:14.270761Z","shell.execute_reply.started":"2022-12-12T07:57:02.229801Z","shell.execute_reply":"2022-12-12T08:23:14.269604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book['pos'] = [analyzer.polarity_scores(x)['pos'] for x in train_book['review_text']]","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:23:14.272243Z","iopub.execute_input":"2022-12-12T08:23:14.272678Z","iopub.status.idle":"2022-12-12T08:49:28.539290Z","shell.execute_reply.started":"2022-12-12T08:23:14.272632Z","shell.execute_reply":"2022-12-12T08:49:28.538161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book['neutral'] = [analyzer.polarity_scores(x)['neu'] for x in train_book['review_text']]","metadata":{"execution":{"iopub.status.busy":"2022-12-12T08:49:28.540942Z","iopub.execute_input":"2022-12-12T08:49:28.541399Z","iopub.status.idle":"2022-12-12T09:15:41.651476Z","shell.execute_reply.started":"2022-12-12T08:49:28.541356Z","shell.execute_reply":"2022-12-12T09:15:41.650278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_book['polarity'] =train_book['review_text'].apply(lambda x:analyzer.polarity_scores(x))\n","metadata":{"execution":{"iopub.status.busy":"2022-12-12T09:15:41.653176Z","iopub.execute_input":"2022-12-12T09:15:41.653582Z","iopub.status.idle":"2022-12-12T09:15:41.659648Z","shell.execute_reply.started":"2022-12-12T09:15:41.653544Z","shell.execute_reply":"2022-12-12T09:15:41.658367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_book","metadata":{"execution":{"iopub.status.busy":"2022-12-12T09:23:01.695987Z","iopub.execute_input":"2022-12-12T09:23:01.696447Z","iopub.status.idle":"2022-12-12T09:23:01.717627Z","shell.execute_reply.started":"2022-12-12T09:23:01.696412Z","shell.execute_reply":"2022-12-12T09:23:01.716161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}