{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-21T06:06:17.991118Z","iopub.execute_input":"2023-05-21T06:06:17.991485Z","iopub.status.idle":"2023-05-21T06:06:18.011707Z","shell.execute_reply.started":"2023-05-21T06:06:17.991454Z","shell.execute_reply":"2023-05-21T06:06:18.010687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install nltk","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:06:18.032952Z","iopub.execute_input":"2023-05-21T06:06:18.033337Z","iopub.status.idle":"2023-05-21T06:06:31.748215Z","shell.execute_reply.started":"2023-05-21T06:06:18.033299Z","shell.execute_reply":"2023-05-21T06:06:31.746775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install textblob","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:06:31.751116Z","iopub.execute_input":"2023-05-21T06:06:31.751594Z","iopub.status.idle":"2023-05-21T06:06:43.080783Z","shell.execute_reply.started":"2023-05-21T06:06:31.751553Z","shell.execute_reply":"2023-05-21T06:06:43.079338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nimport re\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\nfrom textblob import TextBlob","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:06:43.083009Z","iopub.execute_input":"2023-05-21T06:06:43.083380Z","iopub.status.idle":"2023-05-21T06:06:44.948933Z","shell.execute_reply.started":"2023-05-21T06:06:43.083341Z","shell.execute_reply":"2023-05-21T06:06:44.947890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:06:44.950535Z","iopub.execute_input":"2023-05-21T06:06:44.951216Z","iopub.status.idle":"2023-05-21T06:07:07.039163Z","shell.execute_reply.started":"2023-05-21T06:06:44.951182Z","shell.execute_reply":"2023-05-21T06:07:07.038078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count the number of occurrences of each user rating in the 'rating' column\n\nrating_counts = df_train['rating'].value_counts()\n\nprint(rating_counts)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:07.042444Z","iopub.execute_input":"2023-05-21T06:07:07.042755Z","iopub.status.idle":"2023-05-21T06:07:07.066451Z","shell.execute_reply.started":"2023-05-21T06:07:07.042730Z","shell.execute_reply":"2023-05-21T06:07:07.065555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating Sample Data\n\n* *Group the data by rating category and take a random sample of 1000 reviews from each category*","metadata":{}},{"cell_type":"code","source":"sample_data = df_train.groupby('rating').apply(lambda x: x.sample(n=1000))\nsample_data = sample_data.reset_index(drop=True)\nsample_data","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:07.067823Z","iopub.execute_input":"2023-05-21T06:07:07.068418Z","iopub.status.idle":"2023-05-21T06:07:07.443185Z","shell.execute_reply.started":"2023-05-21T06:07:07.068372Z","shell.execute_reply":"2023-05-21T06:07:07.441979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rating_counts = sample_data['rating'].value_counts()\nprint(rating_counts)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:07.444931Z","iopub.execute_input":"2023-05-21T06:07:07.445356Z","iopub.status.idle":"2023-05-21T06:07:07.453761Z","shell.execute_reply.started":"2023-05-21T06:07:07.445318Z","shell.execute_reply":"2023-05-21T06:07:07.451653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample_size = 10000\n#sampled_df = df_train.sample(n=sample_size, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:07.455378Z","iopub.execute_input":"2023-05-21T06:07:07.456528Z","iopub.status.idle":"2023-05-21T06:07:07.462904Z","shell.execute_reply.started":"2023-05-21T06:07:07.456492Z","shell.execute_reply":"2023-05-21T06:07:07.461545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\nfrom sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(sample_data['review_text'], sample_data['rating'], test_size=0.2)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:07.464593Z","iopub.execute_input":"2023-05-21T06:07:07.465783Z","iopub.status.idle":"2023-05-21T06:07:07.686091Z","shell.execute_reply.started":"2023-05-21T06:07:07.465751Z","shell.execute_reply":"2023-05-21T06:07:07.685168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cleaning and Preprocessing the Reviews...\n* *Removing special characters, punctuation, and numericals.*\n* *Preprocessing the text to convert it into a format that can be used by the machine learning algorithm. This could involve techniques such as tokenization, stemming, and removing stop words.*","metadata":{}},{"cell_type":"code","source":"# a function that removes numericals and special characters\n\ndef clean_text(text):\n    # remove numericals\n    text = re.sub(r'\\d+', '', text)\n    # remove special characters\n    text = re.sub(r'[^\\w\\s]', '', text)\n    \n    return text\n\n# apply the function to the 'review_text' column\nsample_data['processed_text'] = sample_data['review_text'].apply(clean_text)\n\nsample_data['processed_text']","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:07.689408Z","iopub.execute_input":"2023-05-21T06:07:07.690334Z","iopub.status.idle":"2023-05-21T06:07:08.172397Z","shell.execute_reply.started":"2023-05-21T06:07:07.690302Z","shell.execute_reply":"2023-05-21T06:07:08.171487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Using a contraction expansion function:\n\n* *Define a function that expands contractions using regular expressions or a predefined mapping. Here's an example using regular expressions:*","metadata":{}},{"cell_type":"code","source":"def expand_contractions(text):\n    contraction_patterns = re.compile(r\"(?i)(\\w+)(n't)\")\n    return contraction_patterns.sub(lambda match: match.group(1) + \" not\", text)\n\n# Apply the function to the DataFrame column using the apply() method:\n\nsample_data['processed_text'] = sample_data['processed_text'].apply(expand_contractions)\nsample_data['processed_text']","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:08.173988Z","iopub.execute_input":"2023-05-21T06:07:08.174329Z","iopub.status.idle":"2023-05-21T06:07:09.631572Z","shell.execute_reply.started":"2023-05-21T06:07:08.174296Z","shell.execute_reply":"2023-05-21T06:07:09.630672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Download NLTK's required resources\nnltk.download('punkt')\nnltk.download('stopwords')\n\n# Define a stemmer and set of stop words\nstemmer = SnowballStemmer('english')\nstop_words = set(stopwords.words('english'))","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:09.632856Z","iopub.execute_input":"2023-05-21T06:07:09.633714Z","iopub.status.idle":"2023-05-21T06:07:09.905084Z","shell.execute_reply.started":"2023-05-21T06:07:09.633676Z","shell.execute_reply":"2023-05-21T06:07:09.904050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the batch size\nbatch_size = 500\n\n# Function to tokenize, stem, and remove stop words from a sentence\ndef process_sentence(sentence):\n    for i in range(0, len(sample_data), batch_size):\n    #for i in range(0, len(sample_data)):\n\n    # Tokenize the sentence\n        tokens = word_tokenize(sentence.lower())\n    # Remove stop words\n        tokens = [token for token in tokens if token not in stop_words]\n    # Stem the tokens\n        tokens = [stemmer.stem(token) for token in tokens]\n    # Join the stemmed tokens into a string\n        processed_sentence = ' '.join(tokens)\n    return processed_sentence","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:09.906740Z","iopub.execute_input":"2023-05-21T06:07:09.907103Z","iopub.status.idle":"2023-05-21T06:07:09.916294Z","shell.execute_reply.started":"2023-05-21T06:07:09.907069Z","shell.execute_reply":"2023-05-21T06:07:09.915317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# apply the function to the 'processed_text' column\nsample_data['processed_text'] = sample_data['processed_text'].apply(process_sentence)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:07:09.923062Z","iopub.execute_input":"2023-05-21T06:07:09.923376Z","iopub.status.idle":"2023-05-21T06:10:52.312544Z","shell.execute_reply.started":"2023-05-21T06:07:09.923351Z","shell.execute_reply":"2023-05-21T06:10:52.311606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_data['processed_text']","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:10:52.314120Z","iopub.execute_input":"2023-05-21T06:10:52.314486Z","iopub.status.idle":"2023-05-21T06:10:52.323002Z","shell.execute_reply.started":"2023-05-21T06:10:52.314453Z","shell.execute_reply":"2023-05-21T06:10:52.322038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample_data[['rating','processed_text']]","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:10:52.324232Z","iopub.execute_input":"2023-05-21T06:10:52.325381Z","iopub.status.idle":"2023-05-21T06:10:52.335145Z","shell.execute_reply.started":"2023-05-21T06:10:52.325343Z","shell.execute_reply":"2023-05-21T06:10:52.334240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Initialize the SentimentIntensityAnalyzer:\n* *The sentiment scores returned by the SentimentIntensityAnalyzer are in the form of a dictionary, where the keys represent different aspects of sentiment analysis:*\n\n* *'neg': The negative sentiment score, representing the intensity of negative sentiment in the text. Higher values indicate more negativity.*\n\n* *'neu': The neutral sentiment score, representing the intensity of neutral sentiment in the text. Higher values indicate a more neutral tone.*\n\n* *'pos': The positive sentiment score, representing the intensity of positive sentiment in the text. Higher values indicate more positivity.*\n\n* *'compound': The compound sentiment score, which is a normalized score that combines the values of 'neg', 'neu', and 'pos'. It ranges from -1 (extremely negative) to +1 (extremely positive). The compound score can be useful when you want a single, overall sentiment score for a piece of text.*","metadata":{}},{"cell_type":"code","source":"# download NLTK resources\nnltk.download('vader_lexicon')\n\n# create an instance of the SentimentIntensityAnalyzer\nsia = SentimentIntensityAnalyzer()","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:10:52.336683Z","iopub.execute_input":"2023-05-21T06:10:52.337021Z","iopub.status.idle":"2023-05-21T06:10:52.365634Z","shell.execute_reply.started":"2023-05-21T06:10:52.336989Z","shell.execute_reply":"2023-05-21T06:10:52.364611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define a function to apply the SentimentIntensityAnalyzer to each text string\ndef get_sentiment_scores(text):\n    scores = sia.polarity_scores(text)\n    return scores\n\n# apply the get_sentiment_scores function to the 'comment' column\nsample_data['sentiment_scores'] = sample_data['processed_text'].apply(get_sentiment_scores)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:10:52.366952Z","iopub.execute_input":"2023-05-21T06:10:52.367262Z","iopub.status.idle":"2023-05-21T06:11:01.397742Z","shell.execute_reply.started":"2023-05-21T06:10:52.367233Z","shell.execute_reply":"2023-05-21T06:11:01.396756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(sample_data[['processed_text', 'sentiment_scores']])\nsample_data['compound'] = sample_data['sentiment_scores'].apply(lambda x: x['compound'])\nprint(sample_data[['processed_text', 'compound', 'rating']])","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:11:01.398954Z","iopub.execute_input":"2023-05-21T06:11:01.399284Z","iopub.status.idle":"2023-05-21T06:11:01.419778Z","shell.execute_reply.started":"2023-05-21T06:11:01.399254Z","shell.execute_reply":"2023-05-21T06:11:01.418720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sentiment Analysis with TextBlob library\n* *Used the TextBlob library to analyze the sentiment polarity of each sentence in the 'processed_text' column of the dataframe.* \n* *Defined the get_sentiment() function that takes a text string, creates a TextBlob object, and returns the sentiment polarity using the sentiment.polarity attribute.* \n* *Then applied this function to the 'processed_text' column of the dataframe using the apply() method and store the sentiment polarity in a new column called 'blob_sentiment_scores'.*","metadata":{}},{"cell_type":"code","source":"def get_sentiment(text):\n    \"\"\"Returns sentiment polarity of a text\"\"\"\n    blob = TextBlob(text)\n    return blob.sentiment.polarity\n\n# apply the function to the 'comment' column\nsample_data['blob_sentiment_scores'] = sample_data['processed_text'].apply(get_sentiment)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:11:01.421183Z","iopub.execute_input":"2023-05-21T06:11:01.422263Z","iopub.status.idle":"2023-05-21T06:11:05.760122Z","shell.execute_reply.started":"2023-05-21T06:11:01.422226Z","shell.execute_reply":"2023-05-21T06:11:05.759149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_data[['processed_text','blob_sentiment_scores','rating']]","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:11:05.761334Z","iopub.execute_input":"2023-05-21T06:11:05.761696Z","iopub.status.idle":"2023-05-21T06:11:05.778940Z","shell.execute_reply.started":"2023-05-21T06:11:05.761665Z","shell.execute_reply":"2023-05-21T06:11:05.777852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Relating sentiment analysis data with the rating\n*Calculated the average sentiment score for each rating category, then grouped the reviews by rating category.*","metadata":{}},{"cell_type":"code","source":"# Group the reviews by rating category and calculate the average sentiment score for each group\ngrouped_data = sample_data.groupby('rating').agg({'blob_sentiment_scores': ['mean', 'std']})\ngrouped_data.columns = ['mean', 'std']\ngrouped_data = grouped_data.reset_index()\ngrouped_data","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:24:36.820706Z","iopub.execute_input":"2023-05-21T06:24:36.821065Z","iopub.status.idle":"2023-05-21T06:24:36.839093Z","shell.execute_reply.started":"2023-05-21T06:24:36.821036Z","shell.execute_reply":"2023-05-21T06:24:36.837927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confidence Intervals:\n* *These confidence intervals indicate the range within which the true population mean sentiment score is likely to fall with the specified confidence level.*","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport scipy.stats as stats\n\n# Assuming you have a DataFrame named 'df' with columns 'rating', 'mean', and 'std'\n\n# Set the desired confidence level\nconfidence_level = 0.95\n\n# Calculate the sample size\nsample_size = len(sample_data)\n\n# Calculate the z-score based on the desired confidence level\nz_score = stats.norm.ppf(1 - (1 - confidence_level) / 2)\n\n# Create empty lists to store the confidence intervals\nlower_ci = []\nupper_ci = []\n\n# Iterate over each row in the DataFrame\nfor _, row in grouped_data.iterrows():\n    rating = row['rating']\n    mean = row['mean']\n    std = row['std']\n    \n    # Calculate the margin of error\n    margin_of_error = z_score * (std / np.sqrt(sample_size))\n    \n    # Calculate the confidence interval\n    lower_bound = mean - margin_of_error\n    upper_bound = mean + margin_of_error\n    \n    # Append the confidence interval values to the lists\n    lower_ci.append(lower_bound)\n    upper_ci.append(upper_bound)\n\n# Add the confidence interval columns to the DataFrame\ngrouped_data['lower_ci'] = lower_ci\ngrouped_data['upper_ci'] = upper_ci\n\ngrouped_data","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:28:30.880828Z","iopub.execute_input":"2023-05-21T07:28:30.881190Z","iopub.status.idle":"2023-05-21T07:28:30.903140Z","shell.execute_reply.started":"2023-05-21T07:28:30.881162Z","shell.execute_reply":"2023-05-21T07:28:30.901767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group the reviews by rating category and calculate the average sentiment score for each group\ngrouped_data_2 = sample_data.groupby('rating').agg({'compound': 'mean'})\ngrouped_data_2","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:19:45.205313Z","iopub.execute_input":"2023-05-21T06:19:45.205690Z","iopub.status.idle":"2023-05-21T06:19:45.220528Z","shell.execute_reply.started":"2023-05-21T06:19:45.205656Z","shell.execute_reply":"2023-05-21T06:19:45.219366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Visualization...\n*Plotted the average sentiment scores against the rating using a scatter plot to Analyze if there is a correlation between the sentiment scores and the ratings.*","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Create a scatter plot of the average sentiment scores and the rating categories\nplt.scatter(grouped_data['rating'], grouped_data['mean'])\n\n# Add labels and title to the plot\nplt.xlabel('Rating')\nplt.ylabel('Average Sentiment Score')\nplt.title('Sentiment Scores by Rating')\n\n# Show the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-21T06:27:39.340660Z","iopub.execute_input":"2023-05-21T06:27:39.341035Z","iopub.status.idle":"2023-05-21T06:27:39.625723Z","shell.execute_reply.started":"2023-05-21T06:27:39.341005Z","shell.execute_reply":"2023-05-21T06:27:39.624780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pearson correlation coefficient (between the sentiment scores and the ratings)\n\n*The correlation coefficient ranges from -1 to 1, where -1 indicates a perfect negative correlation, 0 indicates no correlation, and 1 indicates a perfect positive correlation.*","metadata":{}},{"cell_type":"code","source":"# Select the columns with the variables of interest\nrating = sample_data['rating']\nblob_sentiment_scores = sample_data['blob_sentiment_scores']\n\n# Calculate the correlation coefficient\ncorrelation = blob_sentiment_scores.corr(rating)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:42:43.250724Z","iopub.execute_input":"2023-05-21T07:42:43.251103Z","iopub.status.idle":"2023-05-21T07:42:43.258969Z","shell.execute_reply.started":"2023-05-21T07:42:43.251073Z","shell.execute_reply":"2023-05-21T07:42:43.257892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Correlation between sentiment scores and product ratings:', correlation)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:42:43.260960Z","iopub.execute_input":"2023-05-21T07:42:43.261287Z","iopub.status.idle":"2023-05-21T07:42:43.274740Z","shell.execute_reply.started":"2023-05-21T07:42:43.261256Z","shell.execute_reply":"2023-05-21T07:42:43.273684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*It is suggested that there is a considerable correlation between the reviews and the ratings given by the users. Although the sentiment score for zero rating is quite misleading and can not be consider a good indicator for training a model, for the rest (1-5) ratings it can be obseerved from the scatter plot that a relation does exists and a model based on ratings and reviews can be trained.*","metadata":{}},{"cell_type":"markdown","source":"### Train the SVR Model: \n* *Fit the SVR model on the training data, using the sentiment analysis score and the corresponding ratings.*","metadata":{}},{"cell_type":"code","source":"# Convert the Series to a NumPy array\nX_array = sample_data['blob_sentiment_scores'].values\n\n# Reshape the NumPy array when it has a single feature\nX_reshaped = X_array.reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:42:43.278501Z","iopub.execute_input":"2023-05-21T07:42:43.278798Z","iopub.status.idle":"2023-05-21T07:42:43.286078Z","shell.execute_reply.started":"2023-05-21T07:42:43.278773Z","shell.execute_reply":"2023-05-21T07:42:43.285160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.svm import SVR\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:42:43.287414Z","iopub.execute_input":"2023-05-21T07:42:43.287897Z","iopub.status.idle":"2023-05-21T07:42:43.297701Z","shell.execute_reply.started":"2023-05-21T07:42:43.287864Z","shell.execute_reply":"2023-05-21T07:42:43.296857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X contains the sentiment analysis scores and y contains the book ratings\n\n# Split the data into training and test sets\nX_train, X_test, y_train, y_test = train_test_split(X_reshaped, sample_data['rating'], test_size=0.2, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:42:43.309191Z","iopub.execute_input":"2023-05-21T07:42:43.309662Z","iopub.status.idle":"2023-05-21T07:42:43.317631Z","shell.execute_reply.started":"2023-05-21T07:42:43.309612Z","shell.execute_reply":"2023-05-21T07:42:43.316782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create an instance of SVR with desired parameters\nsvr = SVR(kernel='rbf', C=1.0, epsilon=0.1)\n\n# Train the SVR model\nsvr.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:42:43.411609Z","iopub.execute_input":"2023-05-21T07:42:43.412679Z","iopub.status.idle":"2023-05-21T07:42:44.257667Z","shell.execute_reply.started":"2023-05-21T07:42:43.412611Z","shell.execute_reply":"2023-05-21T07:42:44.256574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions on the test set\ny_pred = svr.predict(X_test)\nrounded_predictions = [round(pred) for pred in y_pred]","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:46:40.314203Z","iopub.execute_input":"2023-05-21T07:46:40.314572Z","iopub.status.idle":"2023-05-21T07:46:40.508765Z","shell.execute_reply.started":"2023-05-21T07:46:40.314541Z","shell.execute_reply":"2023-05-21T07:46:40.507855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate the model using mean squared error\nmse = mean_squared_error(y_test, rounded_predictions)\nprint(\"Mean Squared Error:\", mse)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:46:49.275545Z","iopub.execute_input":"2023-05-21T07:46:49.276289Z","iopub.status.idle":"2023-05-21T07:46:49.284878Z","shell.execute_reply.started":"2023-05-21T07:46:49.276254Z","shell.execute_reply":"2023-05-21T07:46:49.283653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*In the context of SVR model, an MSE of 2.622 suggests that, on average, the predicted ratings deviate from the actual ratings by approximately 2.622 units squared. Since MSE is calculated by squaring the differences, **it penalizes larger deviations more than smaller ones**.*","metadata":{}},{"cell_type":"markdown","source":"### Initialize and train a Linear regression model:\n*Created a linear regression model using LinearRegression from sklearn.linear_model and fit the model to the training data using fit(). Afterward, we make predictions on the testing data using predict() and calculate the mean squared error (MSE) as an evaluation metric.*","metadata":{"execution":{"iopub.status.busy":"2023-05-18T04:51:39.346175Z","iopub.execute_input":"2023-05-18T04:51:39.346811Z","iopub.status.idle":"2023-05-18T04:51:39.355455Z","shell.execute_reply.started":"2023-05-18T04:51:39.346775Z","shell.execute_reply":"2023-05-18T04:51:39.353378Z"}}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nregression_model = LinearRegression()\nregression_model.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:47:44.790556Z","iopub.execute_input":"2023-05-21T07:47:44.790936Z","iopub.status.idle":"2023-05-21T07:47:44.801607Z","shell.execute_reply.started":"2023-05-21T07:47:44.790905Z","shell.execute_reply":"2023-05-21T07:47:44.800436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = regression_model.predict(X_test)\nrounded_predictions_2 = [round(pred) for pred in y_pred]","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:47:47.554834Z","iopub.execute_input":"2023-05-21T07:47:47.555211Z","iopub.status.idle":"2023-05-21T07:47:47.566414Z","shell.execute_reply.started":"2023-05-21T07:47:47.555179Z","shell.execute_reply":"2023-05-21T07:47:47.563209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mse = mean_squared_error(y_test, rounded_predictions_2)\nprint(\"Mean Squared Error:\", mse)","metadata":{"execution":{"iopub.status.busy":"2023-05-21T07:47:57.285213Z","iopub.execute_input":"2023-05-21T07:47:57.285617Z","iopub.status.idle":"2023-05-21T07:47:57.293595Z","shell.execute_reply.started":"2023-05-21T07:47:57.285587Z","shell.execute_reply":"2023-05-21T07:47:57.292669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Not much difference between two models used for predicting the ratings of a book!***","metadata":{}}]}