{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Data Preprocessing**","metadata":{"id":"6dtGEB89OIFn"}},{"cell_type":"code","source":"# Basic Imports\nimport pandas as pd\nimport numpy as np\n\n# Ignore Warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"id":"ff5dn320OaZe","execution":{"iopub.status.busy":"2023-01-01T11:49:29.078051Z","iopub.execute_input":"2023-01-01T11:49:29.078642Z","iopub.status.idle":"2023-01-01T11:49:29.106232Z","shell.execute_reply.started":"2023-01-01T11:49:29.078516Z","shell.execute_reply":"2023-01-01T11:49:29.104829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load and view the training data\ntrain_df = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv\")\ntrain_df.sample(5, random_state = 42)","metadata":{"id":"HKz1uSsCOuv5","outputId":"79ad4e27-13f0-48d9-c1e3-4c4e570c7418","execution":{"iopub.status.busy":"2023-01-01T11:49:29.266275Z","iopub.execute_input":"2023-01-01T11:49:29.266784Z","iopub.status.idle":"2023-01-01T11:50:00.636841Z","shell.execute_reply.started":"2023-01-01T11:49:29.266717Z","shell.execute_reply":"2023-01-01T11:50:00.635419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load and view the testing data\ntest_df = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv\")\ntest_df.sample(5, random_state = 42)","metadata":{"id":"wZIbXQopPJbl","outputId":"f7318750-2e87-406e-cebf-9955800282ff","execution":{"iopub.status.busy":"2023-01-01T11:50:00.641615Z","iopub.execute_input":"2023-01-01T11:50:00.642043Z","iopub.status.idle":"2023-01-01T11:50:17.549916Z","shell.execute_reply.started":"2023-01-01T11:50:00.642004Z","shell.execute_reply":"2023-01-01T11:50:17.548567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View the shape of data\ntrain_df.shape, test_df.shape","metadata":{"id":"xLaV9fQJPQhs","outputId":"7e0fcb34-40c3-4ba9-f2b4-601d4262cc14","execution":{"iopub.status.busy":"2023-01-01T11:50:17.552221Z","iopub.execute_input":"2023-01-01T11:50:17.552689Z","iopub.status.idle":"2023-01-01T11:50:17.560959Z","shell.execute_reply.started":"2023-01-01T11:50:17.552649Z","shell.execute_reply":"2023-01-01T11:50:17.559726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Count the unique users in train and test data\ntrain_df[\"user_id\"].nunique(), test_df[\"user_id\"].nunique()","metadata":{"id":"EpUgpRkaPmOf","outputId":"e397c9c2-c3b8-402a-d490-abc5c149acbf","execution":{"iopub.status.busy":"2023-01-01T11:50:17.564362Z","iopub.execute_input":"2023-01-01T11:50:17.565239Z","iopub.status.idle":"2023-01-01T11:50:17.744534Z","shell.execute_reply.started":"2023-01-01T11:50:17.565195Z","shell.execute_reply":"2023-01-01T11:50:17.743319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Finding common Users from train and test set**","metadata":{}},{"cell_type":"code","source":"# Check for common users in train and test data\nnp.intersect1d(train_df[\"user_id\"].unique(), test_df[\"user_id\"].unique())","metadata":{"id":"ytwY23BUP0VQ","outputId":"dc82af98-d046-403c-de30-86ad7fd5b286","execution":{"iopub.status.busy":"2023-01-01T11:50:17.746095Z","iopub.execute_input":"2023-01-01T11:50:17.746630Z","iopub.status.idle":"2023-01-01T11:50:17.942874Z","shell.execute_reply.started":"2023-01-01T11:50:17.746582Z","shell.execute_reply":"2023-01-01T11:50:17.941493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Finding unique books in train and test set**","metadata":{}},{"cell_type":"code","source":"# Count the unique books in train and test data\ntrain_df[\"book_id\"].nunique(), test_df[\"book_id\"].nunique()","metadata":{"id":"zKQ4zYLeQegK","outputId":"9d61be0e-108b-4279-e7fe-36d2671d7ef2","execution":{"iopub.status.busy":"2023-01-01T11:50:17.944531Z","iopub.execute_input":"2023-01-01T11:50:17.945063Z","iopub.status.idle":"2023-01-01T11:50:17.987988Z","shell.execute_reply.started":"2023-01-01T11:50:17.945027Z","shell.execute_reply":"2023-01-01T11:50:17.986791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Finding common books from train and test set**","metadata":{}},{"cell_type":"code","source":"# Check for common books in train and test data\nnp.intersect1d(train_df[\"book_id\"].unique(), test_df[\"book_id\"].unique()).size","metadata":{"id":"4JRETQypRBvk","outputId":"205b201d-2b24-47de-e874-d9408c515054","execution":{"iopub.status.busy":"2023-01-01T11:50:17.989341Z","iopub.execute_input":"2023-01-01T11:50:17.989724Z","iopub.status.idle":"2023-01-01T11:50:18.043065Z","shell.execute_reply.started":"2023-01-01T11:50:17.989683Z","shell.execute_reply":"2023-01-01T11:50:18.042011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dropping the columns of no use\ntrain_df.drop(columns = [\"book_id\", \"review_id\", \"date_added\", \"date_updated\", \"read_at\", \"started_at\", \"user_id\", \"n_votes\", \"n_comments\"], inplace = True)\ntest_df.drop(columns = [\"book_id\", \"review_id\", \"date_added\", \"date_updated\", \"read_at\", \"started_at\", \"user_id\", \"n_votes\", \"n_comments\"], inplace = True)","metadata":{"id":"5XR4X_zCRFbc","execution":{"iopub.status.busy":"2023-01-01T11:50:18.044795Z","iopub.execute_input":"2023-01-01T11:50:18.045606Z","iopub.status.idle":"2023-01-01T11:50:18.253459Z","shell.execute_reply.started":"2023-01-01T11:50:18.045541Z","shell.execute_reply":"2023-01-01T11:50:18.252242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View the new train_df\ntrain_df.sample(5, random_state = 42)","metadata":{"id":"nUbHyn-CRhti","outputId":"73ea9588-7f92-4f72-cdce-ac2ee055db7c","execution":{"iopub.status.busy":"2023-01-01T11:50:18.255127Z","iopub.execute_input":"2023-01-01T11:50:18.255798Z","iopub.status.idle":"2023-01-01T11:50:18.310888Z","shell.execute_reply.started":"2023-01-01T11:50:18.255756Z","shell.execute_reply":"2023-01-01T11:50:18.309534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View the new test_df\ntest_df.sample(5, random_state = 42)","metadata":{"id":"CV56i7IJRmOZ","outputId":"6c75de69-cf44-4113-8e99-fb2619c0db7c","execution":{"iopub.status.busy":"2023-01-01T11:50:18.315593Z","iopub.execute_input":"2023-01-01T11:50:18.316049Z","iopub.status.idle":"2023-01-01T11:50:18.346932Z","shell.execute_reply.started":"2023-01-01T11:50:18.316011Z","shell.execute_reply":"2023-01-01T11:50:18.345566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check forl null values in our training data\ntrain_df.isnull().sum()","metadata":{"id":"IeU8ClfwSA9h","outputId":"d467f180-809d-4584-c59b-b908ea95b59c","execution":{"iopub.status.busy":"2023-01-01T11:50:18.348386Z","iopub.execute_input":"2023-01-01T11:50:18.348853Z","iopub.status.idle":"2023-01-01T11:50:18.494153Z","shell.execute_reply.started":"2023-01-01T11:50:18.348820Z","shell.execute_reply":"2023-01-01T11:50:18.492965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check forl null values in our testing data\ntest_df.isnull().sum()","metadata":{"id":"hzXMtxyxSFbF","outputId":"199c7565-b303-4c79-f126-2705f8a5e6db","execution":{"iopub.status.busy":"2023-01-01T11:50:18.495791Z","iopub.execute_input":"2023-01-01T11:50:18.496343Z","iopub.status.idle":"2023-01-01T11:50:18.576671Z","shell.execute_reply.started":"2023-01-01T11:50:18.496303Z","shell.execute_reply":"2023-01-01T11:50:18.575080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View train_df\ntrain_df.sample(5, random_state = 42)","metadata":{"id":"Bj-we6hBSOhu","outputId":"6aa6b2b2-db61-4ef7-b486-b0717e7a0e4f","execution":{"iopub.status.busy":"2023-01-01T11:50:18.578621Z","iopub.execute_input":"2023-01-01T11:50:18.579588Z","iopub.status.idle":"2023-01-01T11:50:18.635641Z","shell.execute_reply.started":"2023-01-01T11:50:18.579537Z","shell.execute_reply":"2023-01-01T11:50:18.634257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View test_df\ntest_df.sample(5, random_state = 42)","metadata":{"id":"KC4WMogrSTgF","outputId":"e054ee80-e051-402e-bd1e-8b4f51ed9d07","execution":{"iopub.status.busy":"2023-01-01T11:50:18.637297Z","iopub.execute_input":"2023-01-01T11:50:18.637742Z","iopub.status.idle":"2023-01-01T11:50:18.666841Z","shell.execute_reply.started":"2023-01-01T11:50:18.637691Z","shell.execute_reply":"2023-01-01T11:50:18.665341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Cleaning the Review Text**","metadata":{"id":"CFRhAbFySIpT"}},{"cell_type":"markdown","source":"**Lowercase**","metadata":{}},{"cell_type":"code","source":"# Lowercase the review text\ntrain_df[\"review_text\"] = train_df[\"review_text\"].apply(lambda x: str(x).lower())\ntest_df[\"review_text\"] = test_df[\"review_text\"].apply(lambda x: str(x).lower())","metadata":{"id":"moyZdtj2SHt-","execution":{"iopub.status.busy":"2023-01-01T11:50:18.668363Z","iopub.execute_input":"2023-01-01T11:50:18.669251Z","iopub.status.idle":"2023-01-01T11:50:21.905627Z","shell.execute_reply.started":"2023-01-01T11:50:18.669211Z","shell.execute_reply":"2023-01-01T11:50:21.904142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Line Breaks and Extra Spcaces**","metadata":{}},{"cell_type":"code","source":"# Remove the line breaks and extra spaces\ntrain_df[\"review_text\"] = train_df[\"review_text\"].apply(lambda x: \" \".join(x.split()))\ntest_df[\"review_text\"] = test_df[\"review_text\"].apply(lambda x: \" \".join(x.split()))","metadata":{"id":"onLtqNtKSsRI","execution":{"iopub.status.busy":"2023-01-01T11:50:21.907369Z","iopub.execute_input":"2023-01-01T11:50:21.907808Z","iopub.status.idle":"2023-01-01T11:50:42.530740Z","shell.execute_reply.started":"2023-01-01T11:50:21.907771Z","shell.execute_reply":"2023-01-01T11:50:42.529353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Links and URL's**","metadata":{}},{"cell_type":"code","source":"# re for matching and replacing patterns in string\nimport re\n\n# Removing https links from the text\ntrain_df[\"review_text\"] = train_df[\"review_text\"].apply(\n    lambda x: re.sub(\n        r'(https?:\\/\\/)(\\s)*(www\\.)?(\\s)*((\\w|\\s)+\\.)*([\\w\\-\\s]+\\/)*([\\w\\-]+)((\\?)?[\\w\\s]*=\\s*[\\w\\%&]*)*', '',\n        x, flags=re.MULTILINE))\n\ntest_df[\"review_text\"] = test_df[\"review_text\"].apply(\n    lambda x: re.sub(\n        r'(https?:\\/\\/)(\\s)*(www\\.)?(\\s)*((\\w|\\s)+\\.)*([\\w\\-\\s]+\\/)*([\\w\\-]+)((\\?)?[\\w\\s]*=\\s*[\\w\\%&]*)*', '',\n        x, flags=re.MULTILINE))","metadata":{"id":"VEfyZP5x6FXr","execution":{"iopub.status.busy":"2023-01-01T11:50:42.532853Z","iopub.execute_input":"2023-01-01T11:50:42.533597Z","iopub.status.idle":"2023-01-01T11:50:47.457453Z","shell.execute_reply.started":"2023-01-01T11:50:42.533502Z","shell.execute_reply":"2023-01-01T11:50:47.455897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Special Characters**","metadata":{}},{"cell_type":"code","source":"# Remove special charaters from the review text\ntrain_df[\"review_text\"] = train_df[\"review_text\"].apply(lambda x: re.sub('\\W+',' ', x))\ntest_df[\"review_text\"] = test_df[\"review_text\"].apply(lambda x: re.sub('\\W+',' ', x))","metadata":{"id":"4Hc7qmklTtug","execution":{"iopub.status.busy":"2023-01-01T11:50:47.460846Z","iopub.execute_input":"2023-01-01T11:50:47.461320Z","iopub.status.idle":"2023-01-01T11:52:30.633842Z","shell.execute_reply.started":"2023-01-01T11:50:47.461275Z","shell.execute_reply":"2023-01-01T11:52:30.632143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import nltk and download stopwords\nimport nltk\nnltk.download('stopwords')\n\nfrom nltk.corpus import stopwords\n\n# Load the stop words\nstop_words = list(stopwords.words('english'))\n\n# View the count of stop_words\nlen(stop_words)","metadata":{"id":"80WIo6hnUDuo","execution":{"iopub.status.busy":"2023-01-01T11:52:30.636523Z","iopub.execute_input":"2023-01-01T11:52:30.636985Z","iopub.status.idle":"2023-01-01T11:52:32.227503Z","shell.execute_reply.started":"2023-01-01T11:52:30.636944Z","shell.execute_reply":"2023-01-01T11:52:32.225754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove stop_words from reviews\ntrain_df[\"review_text\"] = train_df[\"review_text\"].apply(lambda x: ' '.join([word for word in x.split() if word not in stop_words]))\ntest_df[\"review_text\"] = test_df[\"review_text\"].apply(lambda x: ' '.join([word for word in x.split() if word not in stop_words]))","metadata":{"id":"lzBo0dYXWAyG","execution":{"iopub.status.busy":"2023-01-01T11:52:32.229475Z","iopub.execute_input":"2023-01-01T11:52:32.230841Z","iopub.status.idle":"2023-01-01T12:02:19.911460Z","shell.execute_reply.started":"2023-01-01T11:52:32.230788Z","shell.execute_reply":"2023-01-01T12:02:19.909569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View train_df\ntrain_df.sample(5, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:02:19.913811Z","iopub.execute_input":"2023-01-01T12:02:19.914307Z","iopub.status.idle":"2023-01-01T12:02:19.971771Z","shell.execute_reply.started":"2023-01-01T12:02:19.914262Z","shell.execute_reply":"2023-01-01T12:02:19.970417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# View test_df\ntest_df.sample(5, random_state = 42)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:02:19.973449Z","iopub.execute_input":"2023-01-01T12:02:19.973871Z","iopub.status.idle":"2023-01-01T12:02:20.003695Z","shell.execute_reply.started":"2023-01-01T12:02:19.973837Z","shell.execute_reply":"2023-01-01T12:02:20.001907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Visualization**","metadata":{"id":"V29SmUBvgdoF"}},{"cell_type":"code","source":"# Wordcloud for text visualization - most used words throughout all reviews\nfrom wordcloud import WordCloud\n\n# Matplotlib for plotting\nimport matplotlib.pyplot as plt\n\ndef show_wordcloud(data, title = None):\n    # Initialize the wordcloud\n    wordcloud = WordCloud(\n        colormap            = \"Spectral\",\n        scale               = 3,\n        random_state        = 1\n    ).generate(str(data))\n\n    # Customize the wordcloud plot\n    fig = plt.figure(1, figsize = (16, 16))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize = 20)\n        fig.subplots_adjust(top = 2.3)\n\n    # Plot and display the wordcloud\n    plt.imshow(wordcloud)\n    plt.show()","metadata":{"id":"KQdpIjlGWQsv","execution":{"iopub.status.busy":"2023-01-01T12:02:20.005701Z","iopub.execute_input":"2023-01-01T12:02:20.007050Z","iopub.status.idle":"2023-01-01T12:02:20.066003Z","shell.execute_reply.started":"2023-01-01T12:02:20.006882Z","shell.execute_reply":"2023-01-01T12:02:20.064598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data word cloud\nshow_wordcloud(train_df[\"review_text\"], title = \"Train Data Wordcloud\")","metadata":{"id":"m3V3GHkfXXSJ","execution":{"iopub.status.busy":"2023-01-01T12:02:20.068224Z","iopub.execute_input":"2023-01-01T12:02:20.068855Z","iopub.status.idle":"2023-01-01T12:02:20.795094Z","shell.execute_reply.started":"2023-01-01T12:02:20.068799Z","shell.execute_reply":"2023-01-01T12:02:20.793372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test data word cloud\nshow_wordcloud(test_df[\"review_text\"], title = \"Test Data Wordcloud\")","metadata":{"id":"r6mb9jmYXfHK","execution":{"iopub.status.busy":"2023-01-01T12:02:20.797415Z","iopub.execute_input":"2023-01-01T12:02:20.798037Z","iopub.status.idle":"2023-01-01T12:02:21.467386Z","shell.execute_reply.started":"2023-01-01T12:02:20.797982Z","shell.execute_reply":"2023-01-01T12:02:21.466300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import Plotly for plots\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n# HTML to display the plotly chart\nfrom IPython.core.display import HTML","metadata":{"id":"JfdA49Z2bNsG","execution":{"iopub.status.busy":"2023-01-01T12:02:21.469292Z","iopub.execute_input":"2023-01-01T12:02:21.470503Z","iopub.status.idle":"2023-01-01T12:02:22.753127Z","shell.execute_reply.started":"2023-01-01T12:02:21.470441Z","shell.execute_reply":"2023-01-01T12:02:22.751936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing the CountVectorizer\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n# Function to get frequency of words\ndef get_top_n_words(corpus, n = None, ngram_range = (1, 1)):\n\n    # CountVectorize the text\n    vec = CountVectorizer(stop_words = 'english', ngram_range = ngram_range).fit(corpus)\n\n    # Create the bag_of_words \n    bag_of_words = vec.transform(corpus)\n\n    # Get the sum of the count of words\n    sum_words = bag_of_words.sum(axis = 0)\n\n    # Get the word frequency of each word\n    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]\n\n    # Sort the words by the frequencey\n    words_freq = sorted(words_freq, key = lambda x: x[1], reverse=True)\n\n    # Return first n words\n    return words_freq[:n]","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:02:22.754983Z","iopub.execute_input":"2023-01-01T12:02:22.755818Z","iopub.status.idle":"2023-01-01T12:02:22.767453Z","shell.execute_reply.started":"2023-01-01T12:02:22.755766Z","shell.execute_reply":"2023-01-01T12:02:22.766100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to plot the top n words by frequency\ndef get_frequency_count_plot(df, title, n = 20, ngram_range = (1, 1)):\n    \n    # Get top 20 words from data\n    common_words = get_top_n_words(df['review_text'], n, ngram_range)\n\n    # Create a dataframe of top n words\n    temp_df = pd.DataFrame(common_words, columns = ['review_text' , 'count'])\n    temp_df = temp_df.groupby('review_text').sum()['count'].sort_values(ascending = False)\n        \n    # Create plotly figure bar plot to display data\n    fig = go.Figure([\n        go.Bar(x = temp_df.keys(), y = temp_df.values)\n    ])\n    \n    # Customize the figure\n    fig.update_layout(\n        title = title,\n        xaxis_title = \"Word\",\n        yaxis_title = \"Count\",\n        width = 1000,\n        height = 600,\n    )\n    fig.update_xaxes(tickangle = 90)\n    \n    return fig","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:02:22.772759Z","iopub.execute_input":"2023-01-01T12:02:22.773623Z","iopub.status.idle":"2023-01-01T12:02:22.783011Z","shell.execute_reply.started":"2023-01-01T12:02:22.773575Z","shell.execute_reply":"2023-01-01T12:02:22.781773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 20 words - unigrams from train data\nget_frequency_count_plot(df = train_df, title = \"Top 20 words - unigrams - Train Data\", n = 20, ngram_range = (1, 1))","metadata":{"id":"SMZkdAdxhF6J","execution":{"iopub.status.busy":"2023-01-01T12:03:13.830675Z","iopub.execute_input":"2023-01-01T12:03:13.831153Z","iopub.status.idle":"2023-01-01T12:06:07.870372Z","shell.execute_reply.started":"2023-01-01T12:03:13.831118Z","shell.execute_reply":"2023-01-01T12:06:07.868880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 20 words - unigrams from test data\nget_frequency_count_plot(df = test_df, title = \"Top 20 words - unigrams - Test Data\", n = 20, ngram_range = (1, 1))","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:06:13.555342Z","iopub.execute_input":"2023-01-01T12:06:13.555837Z","iopub.status.idle":"2023-01-01T12:07:44.538095Z","shell.execute_reply.started":"2023-01-01T12:06:13.555795Z","shell.execute_reply":"2023-01-01T12:07:44.536587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 20 words - bigram from train data\nget_frequency_count_plot(df = train_df, title = \"Top 20 words - bigram - Train Data\", n = 20, ngram_range = (2, 2))","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:08:33.345400Z","iopub.execute_input":"2023-01-01T12:08:33.346080Z","iopub.status.idle":"2023-01-01T12:17:21.317776Z","shell.execute_reply.started":"2023-01-01T12:08:33.346026Z","shell.execute_reply":"2023-01-01T12:17:21.316216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 20 words - bigram from test data\nget_frequency_count_plot(df = test_df, title = \"Top 20 words - bigram - Test Data\", n = 20, ngram_range = (2, 2))","metadata":{"execution":{"iopub.status.busy":"2023-01-01T12:17:21.320558Z","iopub.execute_input":"2023-01-01T12:17:21.321048Z","iopub.status.idle":"2023-01-01T12:22:11.904245Z","shell.execute_reply.started":"2023-01-01T12:17:21.321001Z","shell.execute_reply":"2023-01-01T12:22:11.902852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import gc to free memory\nimport gc\n\n# Delete variables\ndel stop_words, stopwords\n\n# Clear the memory\ngc.collect()","metadata":{"id":"x1OXd7U0g5nQ","execution":{"iopub.status.busy":"2023-01-01T12:22:40.237730Z","iopub.execute_input":"2023-01-01T12:22:40.238197Z","iopub.status.idle":"2023-01-01T12:22:40.550108Z","shell.execute_reply.started":"2023-01-01T12:22:40.238161Z","shell.execute_reply":"2023-01-01T12:22:40.548627Z"},"trusted":true},"execution_count":null,"outputs":[]}]}