{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-09-20T07:41:06.961649Z","iopub.execute_input":"2022-09-20T07:41:06.962034Z","iopub.status.idle":"2022-09-20T07:41:06.976401Z","shell.execute_reply.started":"2022-09-20T07:41:06.962004Z","shell.execute_reply":"2022-09-20T07:41:06.975409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install essential libraries\n! pip install tensorflow-data-validation\n! python -m nltk.downloader all","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"markdown","source":"### Import libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\nfrom tqdm import tqdm\ntqdm.pandas()\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-09-20T06:49:10.106831Z","iopub.execute_input":"2022-09-20T06:49:10.107433Z","iopub.status.idle":"2022-09-20T06:49:10.117337Z","shell.execute_reply.started":"2022-09-20T06:49:10.107389Z","shell.execute_reply":"2022-09-20T06:49:10.116425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Importing and analyzing the datasets\n","metadata":{}},{"cell_type":"code","source":"goodreads_train = pd.read_csv(\"../input/goodreads-books-reviews-290312/goodreads_train.csv\") \ngoodreads_test = pd.read_csv(\"../input/goodreads-books-reviews-290312/goodreads_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:52:57.064150Z","iopub.execute_input":"2022-09-20T04:52:57.064468Z","iopub.status.idle":"2022-09-20T04:53:37.009979Z","shell.execute_reply.started":"2022-09-20T04:52:57.064442Z","shell.execute_reply":"2022-09-20T04:53:37.008972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"goodreads_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:37.011649Z","iopub.execute_input":"2022-09-20T04:53:37.012084Z","iopub.status.idle":"2022-09-20T04:53:37.022811Z","shell.execute_reply.started":"2022-09-20T04:53:37.011995Z","shell.execute_reply":"2022-09-20T04:53:37.021665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"goodreads_train.shape ","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:37.026044Z","iopub.execute_input":"2022-09-20T04:53:37.027030Z","iopub.status.idle":"2022-09-20T04:53:37.035010Z","shell.execute_reply.started":"2022-09-20T04:53:37.026992Z","shell.execute_reply":"2022-09-20T04:53:37.034095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"goodreads_train.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:37.038016Z","iopub.execute_input":"2022-09-20T04:53:37.038756Z","iopub.status.idle":"2022-09-20T04:53:37.085496Z","shell.execute_reply.started":"2022-09-20T04:53:37.038703Z","shell.execute_reply":"2022-09-20T04:53:37.084438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"goodreads_train.set_index(\"review_id\", inplace=True)\ngoodreads_test.set_index(\"review_id\", inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:37.087035Z","iopub.execute_input":"2022-09-20T04:53:37.087507Z","iopub.status.idle":"2022-09-20T04:53:37.098322Z","shell.execute_reply.started":"2022-09-20T04:53:37.087471Z","shell.execute_reply":"2022-09-20T04:53:37.097262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Concating the train and test sets to EDA","metadata":{}},{"cell_type":"code","source":"books = pd.concat([goodreads_train.drop('rating', axis=1), goodreads_test], axis=0)\nbooks_rates = goodreads_train['rating']\n\nbooks.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:37.108753Z","iopub.execute_input":"2022-09-20T04:53:37.109315Z","iopub.status.idle":"2022-09-20T04:53:37.793045Z","shell.execute_reply.started":"2022-09-20T04:53:37.109280Z","shell.execute_reply":"2022-09-20T04:53:37.791947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:37.796761Z","iopub.execute_input":"2022-09-20T04:53:37.797369Z","iopub.status.idle":"2022-09-20T04:53:38.270020Z","shell.execute_reply.started":"2022-09-20T04:53:37.797332Z","shell.execute_reply":"2022-09-20T04:53:38.268917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow_data_validation as tfdv\n# books_stats = tfdv.generate_statistics_from_dataframe(books)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T07:16:48.809867Z","iopub.execute_input":"2022-09-20T07:16:48.810241Z","iopub.status.idle":"2022-09-20T07:16:48.815733Z","shell.execute_reply.started":"2022-09-20T07:16:48.810210Z","shell.execute_reply":"2022-09-20T07:16:48.814385Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tfdv.visualize_statistics(books_stats)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:38.282945Z","iopub.execute_input":"2022-09-20T04:53:38.283327Z","iopub.status.idle":"2022-09-20T04:53:38.290942Z","shell.execute_reply.started":"2022-09-20T04:53:38.283293Z","shell.execute_reply":"2022-09-20T04:53:38.289983Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Droping some non-essential columns\nbooks.drop(columns=['date_updated', 'read_at', 'started_at'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:38.293956Z","iopub.execute_input":"2022-09-20T04:53:38.294905Z","iopub.status.idle":"2022-09-20T04:53:38.421751Z","shell.execute_reply.started":"2022-09-20T04:53:38.294857Z","shell.execute_reply":"2022-09-20T04:53:38.420758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing rows with negative n_votes or negative n_comments \nbooks = books[(books['n_votes'] >= 0) & (books['n_comments'] >= 0)]","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:38.423183Z","iopub.execute_input":"2022-09-20T04:53:38.423542Z","iopub.status.idle":"2022-09-20T04:53:38.638694Z","shell.execute_reply.started":"2022-09-20T04:53:38.423505Z","shell.execute_reply":"2022-09-20T04:53:38.637727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking duplications\nsum(books.duplicated())","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:38.640053Z","iopub.execute_input":"2022-09-20T04:53:38.640891Z","iopub.status.idle":"2022-09-20T04:53:42.937558Z","shell.execute_reply.started":"2022-09-20T04:53:38.640854Z","shell.execute_reply":"2022-09-20T04:53:42.936658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(books.duplicated(subset=['review_text']))","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:42.939218Z","iopub.execute_input":"2022-09-20T04:53:42.939609Z","iopub.status.idle":"2022-09-20T04:53:46.622281Z","shell.execute_reply.started":"2022-09-20T04:53:42.939573Z","shell.execute_reply":"2022-09-20T04:53:46.621206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books.drop_duplicates(subset=['review_text'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:46.623802Z","iopub.execute_input":"2022-09-20T04:53:46.624478Z","iopub.status.idle":"2022-09-20T04:53:50.236095Z","shell.execute_reply.started":"2022-09-20T04:53:46.624441Z","shell.execute_reply":"2022-09-20T04:53:50.234832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finding unappropriate texts\nunapp_texts = books['review_text'].apply(lambda txt: np.mean(list(map(len, txt.split())))) > 25\nunapp_texts.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:53:50.244346Z","iopub.execute_input":"2022-09-20T04:53:50.244734Z","iopub.status.idle":"2022-09-20T04:54:45.221926Z","shell.execute_reply.started":"2022-09-20T04:53:50.244682Z","shell.execute_reply":"2022-09-20T04:54:45.220935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books[unapp_texts].head()","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:54:45.223480Z","iopub.execute_input":"2022-09-20T04:54:45.224124Z","iopub.status.idle":"2022-09-20T04:54:45.238723Z","shell.execute_reply.started":"2022-09-20T04:54:45.224085Z","shell.execute_reply":"2022-09-20T04:54:45.237746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books.drop(books[unapp_texts].index, axis=0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:54:45.240138Z","iopub.execute_input":"2022-09-20T04:54:45.240737Z","iopub.status.idle":"2022-09-20T04:54:45.954215Z","shell.execute_reply.started":"2022-09-20T04:54:45.240687Z","shell.execute_reply":"2022-09-20T04:54:45.953211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spoil_statement = 'spoiler alert'\nbooks['spoil'] = [True if spoil_statement in txt else False for txt in books['review_text']]","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:54:45.955612Z","iopub.execute_input":"2022-09-20T04:54:45.956004Z","iopub.status.idle":"2022-09-20T04:54:47.655083Z","shell.execute_reply.started":"2022-09-20T04:54:45.955957Z","shell.execute_reply":"2022-09-20T04:54:47.654051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books[books['spoil'] == True]","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:54:47.656469Z","iopub.execute_input":"2022-09-20T04:54:47.657598Z","iopub.status.idle":"2022-09-20T04:54:47.688749Z","shell.execute_reply.started":"2022-09-20T04:54:47.657549Z","shell.execute_reply":"2022-09-20T04:54:47.687725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessing the review texts","metadata":{}},{"cell_type":"code","source":"import nltk\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.corpus import stopwords","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:54:47.690273Z","iopub.execute_input":"2022-09-20T04:54:47.690686Z","iopub.status.idle":"2022-09-20T04:54:48.230774Z","shell.execute_reply.started":"2022-09-20T04:54:47.690651Z","shell.execute_reply":"2022-09-20T04:54:48.229764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing punctuation and converting the characters to lowercase\n# NOTE! : It's need considerable time about 18 min --> It will improve\n\n\n## Way 1\nimport string\nexclist = string.punctuation + string.digits\nlamitizer = WordNetLemmatizer()\nstopwords_list = stopwords.words('english')\n\ndef preprocessing_text(txt):\n    words = txt.translate(str.maketrans(exclist, ' '*len(exclist))).lower().split()\n\n    return (' '.join([lamitizer.lemmatize(word) for word in words if word not in stopwords_list])).strip()\n\n\nbooks['review_text_tokens'] = books['review_text'].progress_apply(preprocessing_text)\n\nbooks['review_text_tokens'].sample(5)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:24:11.379960Z","iopub.execute_input":"2022-09-20T05:24:11.380328Z","iopub.status.idle":"2022-09-20T05:42:36.919539Z","shell.execute_reply.started":"2022-09-20T05:24:11.380299Z","shell.execute_reply":"2022-09-20T05:42:36.918630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books.sample(4)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:46:00.519481Z","iopub.execute_input":"2022-09-20T05:46:00.520205Z","iopub.status.idle":"2022-09-20T05:46:00.572249Z","shell.execute_reply.started":"2022-09-20T05:46:00.520168Z","shell.execute_reply":"2022-09-20T05:46:00.571019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datetime_format = '%a %b %d %H:%M:%S %z %Y'\nbooks['date_added'] = pd.to_datetime(books['date_added'], format=datetime_format, utc=True)\n\nbooks.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:55:25.448377Z","iopub.execute_input":"2022-09-20T04:55:25.448734Z","iopub.status.idle":"2022-09-20T04:56:00.918539Z","shell.execute_reply.started":"2022-09-20T04:55:25.448684Z","shell.execute_reply":"2022-09-20T04:56:00.917600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis\n","metadata":{}},{"cell_type":"code","source":"books_dates_added = books['date_added'].dt.date\nbooks_dates_added.value_counts()[:10].plot(kind='bar', \n                                            figsize=(10, 8),\n                                            title='10 Date with most reviews',\n                                            xlabel='Dates',\n                                            ylabel='Count')","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:00.919918Z","iopub.execute_input":"2022-09-20T04:56:00.920557Z","iopub.status.idle":"2022-09-20T04:56:01.683652Z","shell.execute_reply.started":"2022-09-20T04:56:00.920520Z","shell.execute_reply":"2022-09-20T04:56:01.682613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.title(\"Number of rating action in days of all weeks\")\nplt.xlabel(\"Day of week\")\nplt.ylabel(\"Count\")\nsns.histplot(books['date_added'].dt.day_name())","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:01.685184Z","iopub.execute_input":"2022-09-20T04:56:01.685533Z","iopub.status.idle":"2022-09-20T04:56:04.119732Z","shell.execute_reply.started":"2022-09-20T04:56:01.685497Z","shell.execute_reply":"2022-09-20T04:56:04.118761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(13, 8))\nplt.title(\"Number of rating action in months of all years\")\nplt.xlabel(\"Month of week\")\nplt.ylabel(\"Count\")\nsns.histplot(books['date_added'].dt.month_name())","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:04.121265Z","iopub.execute_input":"2022-09-20T04:56:04.121620Z","iopub.status.idle":"2022-09-20T04:56:06.756626Z","shell.execute_reply.started":"2022-09-20T04:56:04.121584Z","shell.execute_reply":"2022-09-20T04:56:06.755684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books.groupby('book_id').count() \\\n                .sort_values(by='user_id', ascending=False)['user_id'][:10] \\\n                .plot(kind='bar', \n                        figsize=(10, 8),\n                        title='10 books with the most reviews',\n                        xlabel='Book id',\n                        ylabel='Number of reviews')","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:06.761062Z","iopub.execute_input":"2022-09-20T04:56:06.768013Z","iopub.status.idle":"2022-09-20T04:56:07.761613Z","shell.execute_reply.started":"2022-09-20T04:56:06.767974Z","shell.execute_reply":"2022-09-20T04:56:07.760625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books.groupby('user_id').count() \\\n        .sort_values(by='book_id', ascending=False)['book_id'][:10] \\\n        .plot(kind='bar',\n                figsize=(10, 8),\n                title='10 users with the most reviews',\n                xlabel='User id',\n                ylabel='Number of reviews')","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:07.763057Z","iopub.execute_input":"2022-09-20T04:56:07.763608Z","iopub.status.idle":"2022-09-20T04:56:08.862570Z","shell.execute_reply.started":"2022-09-20T04:56:07.763571Z","shell.execute_reply":"2022-09-20T04:56:08.861632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books['spoil'].value_counts().plot(kind='bar', \n                                    figsize=(10, 8),\n                                    title='Number of Spoilers Vs. NO Spoilers',\n                                    xlabel='Tags',\n                                    ylabel='Count')","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:08.864522Z","iopub.execute_input":"2022-09-20T04:56:08.865342Z","iopub.status.idle":"2022-09-20T04:56:09.078756Z","shell.execute_reply.started":"2022-09-20T04:56:08.865298Z","shell.execute_reply":"2022-09-20T04:56:09.077649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books_rates.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:09.080074Z","iopub.execute_input":"2022-09-20T04:56:09.080462Z","iopub.status.idle":"2022-09-20T04:56:09.111827Z","shell.execute_reply.started":"2022-09-20T04:56:09.080420Z","shell.execute_reply":"2022-09-20T04:56:09.110770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_books_rate = books.merge(books_rates, how='inner', on='review_id')\nclean_books_rate.sample(3)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:46:28.770616Z","iopub.execute_input":"2022-09-20T05:46:28.771334Z","iopub.status.idle":"2022-09-20T05:46:30.030738Z","shell.execute_reply.started":"2022-09-20T05:46:28.771298Z","shell.execute_reply":"2022-09-20T05:46:30.029640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_books_rate['rating'].value_counts().plot(kind='bar', \n                                                figsize=(10, 8),\n                                                title='The Number of each rate',\n                                                xlabel='Rate',\n                                                ylabel='Count')","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:10.661838Z","iopub.execute_input":"2022-09-20T04:56:10.662291Z","iopub.status.idle":"2022-09-20T04:56:10.886659Z","shell.execute_reply.started":"2022-09-20T04:56:10.662234Z","shell.execute_reply":"2022-09-20T04:56:10.885628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_books_rate.groupby(by='book_id').mean()['rating'] \\\n                    .plot(kind='hist', \n                            figsize=(10, 8),\n                            title='The Mean of rateing per book',\n                            xlabel='Value',\n                            bins=100, \n                            legend=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:10.888078Z","iopub.execute_input":"2022-09-20T04:56:10.889116Z","iopub.status.idle":"2022-09-20T04:56:11.464461Z","shell.execute_reply.started":"2022-09-20T04:56:10.889077Z","shell.execute_reply":"2022-09-20T04:56:11.463485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_books_rate.groupby(by='book_id').std()['rating'] \\\n                    .plot(kind='hist', \n                            figsize=(10, 8),\n                            title='The STD of rateing per book',\n                            xlabel='Value',\n                            bins=100, \n                            legend=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:11.465891Z","iopub.execute_input":"2022-09-20T04:56:11.466283Z","iopub.status.idle":"2022-09-20T04:56:12.046882Z","shell.execute_reply.started":"2022-09-20T04:56:11.466245Z","shell.execute_reply":"2022-09-20T04:56:12.044876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from collections import Counter\n# list1=['apple','egg','apple','banana','egg','apple']\n# counts = Counter(list1)\n# print(counts)\n# # Counter({'apple': 3, 'egg': 2, 'banana': 1})\n\n# # initialising dictionaries\n# ini_dictionary1 = Counter({'nikhil': 1, 'akash' : 5,\n#                      'manjeet' : 10, 'akshat' : 15})\n# ini_dictionary2 = Counter({'akash' : 7, 'akshat' : 5,\n#                                           'm' : 15})\n  \n# # printing initial dictionaries\n# print (\"initial 1st dictionary\", str(ini_dictionary1))\n# print (\"initial 2nd dictionary\", str(ini_dictionary2))\n  \n# # combining dictionaries\n# # using Counter\n# final_dictionary = ini_dictionary1 + ini_dictionary2\n  \n# # printing final result\n# print (\"final dictionary\", str(final_dictionary))","metadata":{"execution":{"iopub.status.busy":"2022-09-20T04:56:12.048807Z","iopub.execute_input":"2022-09-20T04:56:12.049415Z","iopub.status.idle":"2022-09-20T04:56:12.054563Z","shell.execute_reply.started":"2022-09-20T04:56:12.049375Z","shell.execute_reply":"2022-09-20T04:56:12.053615Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"markdown","source":"## Machine Learning Methods","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression, SGDClassifier\nfrom sklearn.naive_bayes import MultinomialNB\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics import classification_report, confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2022-09-20T06:31:59.008694Z","iopub.execute_input":"2022-09-20T06:31:59.009577Z","iopub.status.idle":"2022-09-20T06:31:59.115473Z","shell.execute_reply.started":"2022-09-20T06:31:59.009538Z","shell.execute_reply":"2022-09-20T06:31:59.114539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = clean_books_rate[['review_text_tokens', 'rating']]","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:46:43.797476Z","iopub.execute_input":"2022-09-20T05:46:43.798049Z","iopub.status.idle":"2022-09-20T05:46:43.838456Z","shell.execute_reply.started":"2022-09-20T05:46:43.798006Z","shell.execute_reply":"2022-09-20T05:46:43.837582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sample(4)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:46:45.490607Z","iopub.execute_input":"2022-09-20T05:46:45.491566Z","iopub.status.idle":"2022-09-20T05:46:45.525406Z","shell.execute_reply.started":"2022-09-20T05:46:45.491526Z","shell.execute_reply":"2022-09-20T05:46:45.524378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.sample(10000).to_csv('mini_clean_train_goodreads.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:47:09.256105Z","iopub.execute_input":"2022-09-20T05:47:09.256456Z","iopub.status.idle":"2022-09-20T05:47:09.412006Z","shell.execute_reply.started":"2022-09-20T05:47:09.256427Z","shell.execute_reply":"2022-09-20T05:47:09.410619Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mini_df = df.sample(10000)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:48:36.113314Z","iopub.execute_input":"2022-09-20T05:48:36.113683Z","iopub.status.idle":"2022-09-20T05:48:36.144784Z","shell.execute_reply.started":"2022-09-20T05:48:36.113649Z","shell.execute_reply":"2022-09-20T05:48:36.143659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = mini_df['review_text_tokens']\ny = mini_df['rating']","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:48:49.470127Z","iopub.execute_input":"2022-09-20T05:48:49.470955Z","iopub.status.idle":"2022-09-20T05:48:49.476229Z","shell.execute_reply.started":"2022-09-20T05:48:49.470914Z","shell.execute_reply":"2022-09-20T05:48:49.475176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, shuffle=True, random_state=404)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:48:51.879320Z","iopub.execute_input":"2022-09-20T05:48:51.879667Z","iopub.status.idle":"2022-09-20T05:48:51.891436Z","shell.execute_reply.started":"2022-09-20T05:48:51.879635Z","shell.execute_reply":"2022-09-20T05:48:51.890485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"X Train set size: \", X_train.shape)\nprint(\"X Validation set size: \", X_val.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:48:59.756525Z","iopub.execute_input":"2022-09-20T05:48:59.756904Z","iopub.status.idle":"2022-09-20T05:48:59.762641Z","shell.execute_reply.started":"2022-09-20T05:48:59.756870Z","shell.execute_reply":"2022-09-20T05:48:59.761625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Y Train set size: \", y_train.shape)\nprint(\"Y Validation set size: \", y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:49:03.377185Z","iopub.execute_input":"2022-09-20T05:49:03.377797Z","iopub.status.idle":"2022-09-20T05:49:03.383565Z","shell.execute_reply.started":"2022-09-20T05:49:03.377759Z","shell.execute_reply":"2022-09-20T05:49:03.382518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfidf_vectorizer = TfidfVectorizer(use_idf=True)\n\nX_train_vectors = tfidf_vectorizer.fit_transform(X_train) \nX_val_vectors = tfidf_vectorizer.transform(X_val)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:49:07.118378Z","iopub.execute_input":"2022-09-20T05:49:07.118761Z","iopub.status.idle":"2022-09-20T05:49:07.869348Z","shell.execute_reply.started":"2022-09-20T05:49:07.118721Z","shell.execute_reply":"2022-09-20T05:49:07.868264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train_vectors.shape)\nprint(X_val_vectors.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:49:13.467281Z","iopub.execute_input":"2022-09-20T05:49:13.467639Z","iopub.status.idle":"2022-09-20T05:49:13.473554Z","shell.execute_reply.started":"2022-09-20T05:49:13.467607Z","shell.execute_reply":"2022-09-20T05:49:13.472373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## * Logistic Regression","metadata":{}},{"cell_type":"code","source":"lr_model=LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')\nlr_model.fit(X_train_vectors, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:49:26.014549Z","iopub.execute_input":"2022-09-20T05:49:26.015436Z","iopub.status.idle":"2022-09-20T05:49:27.881436Z","shell.execute_reply.started":"2022-09-20T05:49:26.015396Z","shell.execute_reply":"2022-09-20T05:49:27.880228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predict y value for test dataset\ny_predict = lr_model.predict(X_val_vectors)\ny_prob = lr_model.predict_proba(X_val_vectors)[:,1]\n\nprint(\"test accuracy\", str(np.mean(y_val == y_predict)))\nprint(\"-\"*60)\nprint(classification_report(y_val,y_predict))\nprint(\"-\"*60)\nprint('Confusion Matrix:',confusion_matrix(y_val, y_predict))","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:52:25.429340Z","iopub.execute_input":"2022-09-20T05:52:25.429855Z","iopub.status.idle":"2022-09-20T05:52:25.459833Z","shell.execute_reply.started":"2022-09-20T05:52:25.429808Z","shell.execute_reply":"2022-09-20T05:52:25.458770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## * Naive Bayes","metadata":{}},{"cell_type":"code","source":"multiNB_model = MultinomialNB()\nmultiNB_model.fit(X_train_vectors, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T05:53:10.709213Z","iopub.execute_input":"2022-09-20T05:53:10.709580Z","iopub.status.idle":"2022-09-20T05:53:10.727692Z","shell.execute_reply.started":"2022-09-20T05:53:10.709542Z","shell.execute_reply":"2022-09-20T05:53:10.726831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predict y value for test dataset\ny_predict = multiNB_model.predict(X_val_vectors)\ny_prob = multiNB_model.predict_proba(X_val_vectors)[:,1]\n\nprint(\"test accuracy\", str(np.mean(y_val == y_predict)))\nprint(\"-\"*60)\nprint(classification_report(y_val,y_predict))\nprint(\"-\"*60)\nprint('Confusion Matrix:',confusion_matrix(y_val, y_predict))","metadata":{"execution":{"iopub.status.busy":"2022-09-20T06:49:28.444075Z","iopub.execute_input":"2022-09-20T06:49:28.444459Z","iopub.status.idle":"2022-09-20T06:49:28.467717Z","shell.execute_reply.started":"2022-09-20T06:49:28.444427Z","shell.execute_reply":"2022-09-20T06:49:28.466604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## * SGD","metadata":{}},{"cell_type":"code","source":"sgd_model = SGDClassifier(loss='hinge', penalty='l2', alpha=1e-3, random_state=404)\nsgd_model.fit(X_train_vectors, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T06:00:28.912617Z","iopub.execute_input":"2022-09-20T06:00:28.913431Z","iopub.status.idle":"2022-09-20T06:00:29.156950Z","shell.execute_reply.started":"2022-09-20T06:00:28.913383Z","shell.execute_reply":"2022-09-20T06:00:29.155960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predict y value for test dataset\ny_predict = sgd_model.predict(X_val_vectors)\n# y_prob = sgd_model.predict_proba(X_val_vectors)[:,1] # Is not available for loss='hinge'\n\nprint(\"test accuracy\", str(np.mean(y_val == y_predict)))\nprint(\"-\"*60)\nprint(classification_report(y_val,y_predict))\nprint(\"-\"*60)\nprint('Confusion Matrix:',confusion_matrix(y_val, y_predict))","metadata":{"execution":{"iopub.status.busy":"2022-09-20T06:01:07.264329Z","iopub.execute_input":"2022-09-20T06:01:07.264701Z","iopub.status.idle":"2022-09-20T06:01:07.284156Z","shell.execute_reply.started":"2022-09-20T06:01:07.264668Z","shell.execute_reply":"2022-09-20T06:01:07.283042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## * XGBoost","metadata":{}},{"cell_type":"code","source":"xgb_model = XGBClassifier()\nxgb_model.fit(X_train_vectors, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-20T06:32:57.311128Z","iopub.execute_input":"2022-09-20T06:32:57.312260Z","iopub.status.idle":"2022-09-20T06:34:52.040641Z","shell.execute_reply.started":"2022-09-20T06:32:57.312216Z","shell.execute_reply":"2022-09-20T06:34:52.039626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predict y value for test dataset\ny_predict = xgb_model.predict(X_val_vectors)\ny_prob = xgb_model.predict_proba(X_val_vectors)[:,1] # Is not available for loss='hinge'\n\nprint(\"test accuracy\", str(np.mean(y_val == y_predict)))\nprint(\"-\"*60)\nprint(classification_report(y_val,y_predict))\nprint(\"-\"*60)\nprint('Confusion Matrix:',confusion_matrix(y_val, y_predict))","metadata":{"execution":{"iopub.status.busy":"2022-09-20T07:39:51.004799Z","iopub.execute_input":"2022-09-20T07:39:51.005591Z","iopub.status.idle":"2022-09-20T07:39:51.213835Z","shell.execute_reply.started":"2022-09-20T07:39:51.005540Z","shell.execute_reply":"2022-09-20T07:39:51.212781Z"},"trusted":true},"execution_count":null,"outputs":[]}]}