{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-28T11:30:05.582039Z","iopub.execute_input":"2022-08-28T11:30:05.583595Z","iopub.status.idle":"2022-08-28T11:30:05.598755Z","shell.execute_reply.started":"2022-08-28T11:30:05.583533Z","shell.execute_reply":"2022-08-28T11:30:05.59758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv\")\ndf_train = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv\")\ndf_sample = pd.read_csv(\"/kaggle/input/goodreads-books-reviews-290312/goodreads_sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:05.601384Z","iopub.execute_input":"2022-08-28T11:30:05.602294Z","iopub.status.idle":"2022-08-28T11:30:46.073193Z","shell.execute_reply.started":"2022-08-28T11:30:05.602244Z","shell.execute_reply":"2022-08-28T11:30:46.072166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.sample(n=30000)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:46.074657Z","iopub.execute_input":"2022-08-28T11:30:46.075016Z","iopub.status.idle":"2022-08-28T11:30:46.299996Z","shell.execute_reply.started":"2022-08-28T11:30:46.074983Z","shell.execute_reply":"2022-08-28T11:30:46.29856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA","metadata":{}},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:46.376327Z","iopub.execute_input":"2022-08-28T11:30:46.376809Z","iopub.status.idle":"2022-08-28T11:30:46.403359Z","shell.execute_reply.started":"2022-08-28T11:30:46.37677Z","shell.execute_reply":"2022-08-28T11:30:46.402074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:46.404688Z","iopub.execute_input":"2022-08-28T11:30:46.40506Z","iopub.status.idle":"2022-08-28T11:30:46.422413Z","shell.execute_reply.started":"2022-08-28T11:30:46.405026Z","shell.execute_reply":"2022-08-28T11:30:46.421086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:46.424796Z","iopub.execute_input":"2022-08-28T11:30:46.425381Z","iopub.status.idle":"2022-08-28T11:30:46.440814Z","shell.execute_reply.started":"2022-08-28T11:30:46.425326Z","shell.execute_reply":"2022-08-28T11:30:46.439392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train DF: \", df_train.shape)\nprint(\"Test DF: \", df_test.shape)\nprint(\"Sample DF: \", df_sample.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:46.442559Z","iopub.execute_input":"2022-08-28T11:30:46.443015Z","iopub.status.idle":"2022-08-28T11:30:46.450388Z","shell.execute_reply.started":"2022-08-28T11:30:46.442975Z","shell.execute_reply":"2022-08-28T11:30:46.448774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train[\"n_comments\"] != 0].value_counts().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:46.452641Z","iopub.execute_input":"2022-08-28T11:30:46.453143Z","iopub.status.idle":"2022-08-28T11:30:46.552881Z","shell.execute_reply.started":"2022-08-28T11:30:46.453093Z","shell.execute_reply":"2022-08-28T11:30:46.551573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:46.558598Z","iopub.execute_input":"2022-08-28T11:30:46.55927Z","iopub.status.idle":"2022-08-28T11:30:47.256066Z","shell.execute_reply.started":"2022-08-28T11:30:46.559201Z","shell.execute_reply":"2022-08-28T11:30:47.254842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.rating.plot.density(color='blue')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:47.257564Z","iopub.execute_input":"2022-08-28T11:30:47.257961Z","iopub.status.idle":"2022-08-28T11:30:48.069358Z","shell.execute_reply.started":"2022-08-28T11:30:47.257919Z","shell.execute_reply":"2022-08-28T11:30:48.067114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['rating'].value_counts().plot.pie(explode=[0,0.1, 0.1, 0.1, 0.1, 0.1],autopct='%1.1f%%',shadow=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:48.070973Z","iopub.execute_input":"2022-08-28T11:30:48.071778Z","iopub.status.idle":"2022-08-28T11:30:48.262804Z","shell.execute_reply.started":"2022-08-28T11:30:48.071736Z","shell.execute_reply":"2022-08-28T11:30:48.260919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = df_train.rating.value_counts()\nsns.barplot(x.index, x)\nplt.gca().set_ylabel(\"Rating\")\nplt.gca().set_xlabel(\"Train Data Analysis\")","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:48.265766Z","iopub.execute_input":"2022-08-28T11:30:48.266988Z","iopub.status.idle":"2022-08-28T11:30:48.511181Z","shell.execute_reply.started":"2022-08-28T11:30:48.266915Z","shell.execute_reply":"2022-08-28T11:30:48.510032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## https://www.sfu.ca/~mjbrydon/tutorials/BAinPy/08_correlation.html\nax = sns.scatterplot(x=\"rating\", y=\"n_votes\", data=df_train);\nax.set_title(\"Concrete Rating vs. Votes\")\nax.set_xlabel(\"Rating\");","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:48.512867Z","iopub.execute_input":"2022-08-28T11:30:48.513266Z","iopub.status.idle":"2022-08-28T11:30:48.807373Z","shell.execute_reply.started":"2022-08-28T11:30:48.513233Z","shell.execute_reply":"2022-08-28T11:30:48.806066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lmplot(x=\"rating\", y=\"n_votes\", data=df_train);","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:48.809699Z","iopub.execute_input":"2022-08-28T11:30:48.810264Z","iopub.status.idle":"2022-08-28T11:30:52.673889Z","shell.execute_reply.started":"2022-08-28T11:30:48.810193Z","shell.execute_reply":"2022-08-28T11:30:52.672393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['lograting'] = np.log(df_train['rating'])\nsns.kdeplot(x=df_train['lograting'], shade=True, linewidth=2)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:52.675796Z","iopub.execute_input":"2022-08-28T11:30:52.676777Z","iopub.status.idle":"2022-08-28T11:30:53.073446Z","shell.execute_reply.started":"2022-08-28T11:30:52.676737Z","shell.execute_reply":"2022-08-28T11:30:53.072528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x=df_train['n_votes'])","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:53.075049Z","iopub.execute_input":"2022-08-28T11:30:53.075899Z","iopub.status.idle":"2022-08-28T11:30:53.273141Z","shell.execute_reply.started":"2022-08-28T11:30:53.075862Z","shell.execute_reply":"2022-08-28T11:30:53.271785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(y=df_train['rating'],  color='lightgreen', showmeans=True);","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:53.275291Z","iopub.execute_input":"2022-08-28T11:30:53.276057Z","iopub.status.idle":"2022-08-28T11:30:53.448803Z","shell.execute_reply.started":"2022-08-28T11:30:53.276007Z","shell.execute_reply":"2022-08-28T11:30:53.446661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(y=df_train['n_votes'],  color='lightgreen', showmeans=True);","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:53.450693Z","iopub.execute_input":"2022-08-28T11:30:53.451068Z","iopub.status.idle":"2022-08-28T11:30:53.654259Z","shell.execute_reply.started":"2022-08-28T11:30:53.451036Z","shell.execute_reply":"2022-08-28T11:30:53.653012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(x=df_train['rating'], bins=10, kde=True);","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:53.656547Z","iopub.execute_input":"2022-08-28T11:30:53.656959Z","iopub.status.idle":"2022-08-28T11:30:54.077701Z","shell.execute_reply.started":"2022-08-28T11:30:53.656923Z","shell.execute_reply":"2022-08-28T11:30:54.076498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import stats\nstats.pearsonr(df_train['rating'], df_train['n_votes'])","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:54.080049Z","iopub.execute_input":"2022-08-28T11:30:54.081079Z","iopub.status.idle":"2022-08-28T11:30:54.099404Z","shell.execute_reply.started":"2022-08-28T11:30:54.081034Z","shell.execute_reply":"2022-08-28T11:30:54.096931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_voting = df_train[\"n_votes\"]\ndf_rating = df_train[\"rating\"]\ncormat = df_rating.corr(df_voting)\nprint(cormat)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:54.102333Z","iopub.execute_input":"2022-08-28T11:30:54.104956Z","iopub.status.idle":"2022-08-28T11:30:54.12537Z","shell.execute_reply.started":"2022-08-28T11:30:54.104869Z","shell.execute_reply":"2022-08-28T11:30:54.123247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_voting = df_train[\"n_comments\"]\ndf_rating = df_train[\"rating\"]\ncormat = df_rating.corr(df_voting)\nprint(cormat)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:54.130021Z","iopub.execute_input":"2022-08-28T11:30:54.132963Z","iopub.status.idle":"2022-08-28T11:30:54.147234Z","shell.execute_reply.started":"2022-08-28T11:30:54.132873Z","shell.execute_reply":"2022-08-28T11:30:54.145654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating a contingency table","metadata":{}},{"cell_type":"code","source":"contab_freq = pd.crosstab(\n    df_train['rating'],\n    df_train['n_votes'],\n    margins = True,\n    #normalize='index'\n   )\ncontab_freq","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:54.149707Z","iopub.execute_input":"2022-08-28T11:30:54.150932Z","iopub.status.idle":"2022-08-28T11:30:54.340047Z","shell.execute_reply.started":"2022-08-28T11:30:54.150878Z","shell.execute_reply":"2022-08-28T11:30:54.338704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Chi-squared test of independence**","metadata":{}},{"cell_type":"code","source":"contab_obs = pd.crosstab(\n    df_train['rating'],\n    df_train['n_votes'],\n    margins = False)\nchi = stats.chi2_contingency(contab_obs)\nchi","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:54.34646Z","iopub.execute_input":"2022-08-28T11:30:54.347423Z","iopub.status.idle":"2022-08-28T11:30:54.399617Z","shell.execute_reply.started":"2022-08-28T11:30:54.347383Z","shell.execute_reply":"2022-08-28T11:30:54.398363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Feature Extraction and Feature Engineering**","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb\nfrom tqdm import tqdm\nfrom sklearn.svm import SVC\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.naive_bayes import MultinomialNB\nfrom nltk import word_tokenize\nfrom nltk.corpus import stopwords\nstop_words = stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:54.403581Z","iopub.execute_input":"2022-08-28T11:30:54.403982Z","iopub.status.idle":"2022-08-28T11:30:55.564068Z","shell.execute_reply.started":"2022-08-28T11:30:54.40394Z","shell.execute_reply":"2022-08-28T11:30:55.562972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preprocessing","metadata":{}},{"cell_type":"code","source":"import re\ndef CleanTxt(text):\n    text=re.sub(r'@[A-Za-z0-9]+', '', text)\n    text=re.sub(r'#', ' ', text) # hashtag silmek için\n    text=re.sub(r'RT[\\s]+', '', text) #rt\n    text=re.sub(r'https?:\\/\\/\\S+', '', text)  #web siteleri için\n    text=re.sub(r\"\\b[a-zA-Z]\\b\", \"\", text)\n#     text=re.sub(\"\\d\",\"\", text)\n    text=re.sub(\"[^A-Za-z0-9\\.]\",\" \", text)\n    return text.lower()    ","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:55.565493Z","iopub.execute_input":"2022-08-28T11:30:55.565903Z","iopub.status.idle":"2022-08-28T11:30:55.57473Z","shell.execute_reply.started":"2022-08-28T11:30:55.56586Z","shell.execute_reply":"2022-08-28T11:30:55.573302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"clean_review\"]=df_train[\"review_text\"].apply(CleanTxt)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:55.576793Z","iopub.execute_input":"2022-08-28T11:30:55.577638Z","iopub.status.idle":"2022-08-28T11:30:58.756169Z","shell.execute_reply.started":"2022-08-28T11:30:55.577586Z","shell.execute_reply":"2022-08-28T11:30:58.754986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:58.757876Z","iopub.execute_input":"2022-08-28T11:30:58.759536Z","iopub.status.idle":"2022-08-28T11:30:58.786146Z","shell.execute_reply.started":"2022-08-28T11:30:58.759481Z","shell.execute_reply":"2022-08-28T11:30:58.784923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_test[\"clean_review\"]=df_test[\"review_text\"].apply(CleanTxt)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:58.787555Z","iopub.execute_input":"2022-08-28T11:30:58.787887Z","iopub.status.idle":"2022-08-28T11:30:58.795073Z","shell.execute_reply.started":"2022-08-28T11:30:58.787858Z","shell.execute_reply":"2022-08-28T11:30:58.794099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = df_train[\"clean_review\"] \n# y = df_train[\"rating\"] \n\n# ##==================\n\n# x_test = df_test[\"clean_review\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:58.796909Z","iopub.execute_input":"2022-08-28T11:30:58.797661Z","iopub.status.idle":"2022-08-28T11:30:58.804572Z","shell.execute_reply.started":"2022-08-28T11:30:58.797626Z","shell.execute_reply":"2022-08-28T11:30:58.803505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# yvalid = df_sample[\"rating\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:58.806277Z","iopub.execute_input":"2022-08-28T11:30:58.806921Z","iopub.status.idle":"2022-08-28T11:30:58.819692Z","shell.execute_reply.started":"2022-08-28T11:30:58.806883Z","shell.execute_reply":"2022-08-28T11:30:58.818747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(df_train[\"clean_review\"].values, df_train[\"rating\"], test_size=0.20, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:58.821183Z","iopub.execute_input":"2022-08-28T11:30:58.821845Z","iopub.status.idle":"2022-08-28T11:30:58.837911Z","shell.execute_reply.started":"2022-08-28T11:30:58.821806Z","shell.execute_reply":"2022-08-28T11:30:58.83696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfv = TfidfVectorizer(min_df = 3, max_features=None, strip_accents='unicode',\n                        analyzer='word', token_pattern=r'\\w{1,}',\n                        ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,\n                        stop_words = 'english')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:58.839579Z","iopub.execute_input":"2022-08-28T11:30:58.840193Z","iopub.status.idle":"2022-08-28T11:30:58.847519Z","shell.execute_reply.started":"2022-08-28T11:30:58.840156Z","shell.execute_reply":"2022-08-28T11:30:58.846502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"X_train: \", X_train.shape, \" - X_test:\", X_test.shape, \" - y_train:\", y_train.shape, \" - y_test:\", y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:58.849087Z","iopub.execute_input":"2022-08-28T11:30:58.849533Z","iopub.status.idle":"2022-08-28T11:30:58.861317Z","shell.execute_reply.started":"2022-08-28T11:30:58.849489Z","shell.execute_reply":"2022-08-28T11:30:58.860273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fitting TF-IDF to both training and test sets (semi-supervised learning)\ntfv.fit(list(X_train) +  list(X_test))\nxtrain_tfv =  tfv.transform(X_train) \nxvalid_tfv = tfv.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:30:58.862612Z","iopub.execute_input":"2022-08-28T11:30:58.863705Z","iopub.status.idle":"2022-08-28T11:31:25.095192Z","shell.execute_reply.started":"2022-08-28T11:30:58.863665Z","shell.execute_reply":"2022-08-28T11:31:25.093929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Training","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, confusion_matrix, classification_report","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:31:25.096735Z","iopub.execute_input":"2022-08-28T11:31:25.09712Z","iopub.status.idle":"2022-08-28T11:31:25.102839Z","shell.execute_reply.started":"2022-08-28T11:31:25.097087Z","shell.execute_reply":"2022-08-28T11:31:25.101483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fitting a simple Logistic Regression on TFIDF\nclf = LogisticRegression(C=1.0)\nclf.fit(xtrain_tfv, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:31:25.104293Z","iopub.execute_input":"2022-08-28T11:31:25.105325Z","iopub.status.idle":"2022-08-28T11:32:13.211699Z","shell.execute_reply.started":"2022-08-28T11:31:25.105287Z","shell.execute_reply":"2022-08-28T11:32:13.210419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_test = clf.predict(xvalid_tfv)\naccuracy_score(y_test, y_pred_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:32:13.213559Z","iopub.execute_input":"2022-08-28T11:32:13.214852Z","iopub.status.idle":"2022-08-28T11:32:13.242865Z","shell.execute_reply.started":"2022-08-28T11:32:13.214803Z","shell.execute_reply":"2022-08-28T11:32:13.242041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_dict={}\nwith open('../input/glovedata/glove.6B.100d.txt','r') as f:\n    for line in f:\n        values=line.split()\n        word=values[0]\n        vectors=np.asarray(values[1:], 'float32')\n        embedding_dict[word]=vectors\nf.close()\nprint('Found %s word vectors.' % len(embedding_dict))","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:27:40.25909Z","iopub.execute_input":"2022-08-28T12:27:40.259943Z","iopub.status.idle":"2022-08-28T12:27:49.127507Z","shell.execute_reply.started":"2022-08-28T12:27:40.2599Z","shell.execute_reply":"2022-08-28T12:27:49.1261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import spatial","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:29:02.845908Z","iopub.execute_input":"2022-08-28T12:29:02.846366Z","iopub.status.idle":"2022-08-28T12:29:02.852507Z","shell.execute_reply.started":"2022-08-28T12:29:02.846328Z","shell.execute_reply":"2022-08-28T12:29:02.851039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_closest_embeddings(embedding):\n    return sorted(embedding_dict.keys(), key=lambda word: spatial.distance.euclidean(embedding_dict[word], embedding))","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:28:41.18072Z","iopub.execute_input":"2022-08-28T12:28:41.181566Z","iopub.status.idle":"2022-08-28T12:28:41.186996Z","shell.execute_reply.started":"2022-08-28T12:28:41.181524Z","shell.execute_reply":"2022-08-28T12:28:41.185923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"find_closest_embeddings(embedding_dict[\"king\"])[:5]","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:30:01.642914Z","iopub.execute_input":"2022-08-28T12:30:01.644019Z","iopub.status.idle":"2022-08-28T12:30:05.070868Z","shell.execute_reply.started":"2022-08-28T12:30:01.643976Z","shell.execute_reply":"2022-08-28T12:30:05.069479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nnltk.download('punkt')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:32:23.33599Z","iopub.execute_input":"2022-08-28T11:32:23.33643Z","iopub.status.idle":"2022-08-28T11:32:23.609808Z","shell.execute_reply.started":"2022-08-28T11:32:23.336392Z","shell.execute_reply":"2022-08-28T11:32:23.608461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this function creates a normalized vector for the whole sentence\ndef sent2vec(s):\n    words = s.lower()\n    words = word_tokenize(words)\n    words = [w for w in words if not w in stop_words]\n    words = [w for w in words if w.isalpha()]\n    M = []\n    for w in words:\n        try:\n            M.append(embedding_dict[w])\n        except:\n            continue\n    M = np.array(M)\n    v = M.sum(axis=0)\n    if type(v) != np.ndarray:\n        return np.zeros(300)\n    return v / np.sqrt((v ** 2).sum())","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:32:23.611573Z","iopub.execute_input":"2022-08-28T11:32:23.612114Z","iopub.status.idle":"2022-08-28T11:32:23.621698Z","shell.execute_reply.started":"2022-08-28T11:32:23.612062Z","shell.execute_reply":"2022-08-28T11:32:23.62047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  for x in tqdm(X_train):\n#         print(x)\n#         break","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:42:26.173624Z","iopub.execute_input":"2022-08-28T12:42:26.174167Z","iopub.status.idle":"2022-08-28T12:42:26.186281Z","shell.execute_reply.started":"2022-08-28T12:42:26.174129Z","shell.execute_reply":"2022-08-28T12:42:26.184916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create sentence vectors using the above function for training and validation set\nxtrain_glove = [sent2vec(x) for x in tqdm(X_train)]\nxvalid_glove = [sent2vec(x) for x in tqdm(X_test)]","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:35:55.978591Z","iopub.execute_input":"2022-08-28T11:35:55.979047Z","iopub.status.idle":"2022-08-28T11:37:06.81853Z","shell.execute_reply.started":"2022-08-28T11:35:55.97901Z","shell.execute_reply":"2022-08-28T11:37:06.817173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xvalid_glove_1 = np.array(xvalid_glove)\nxtrain_glove_1 = np.array(xtrain_glove)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:37:57.426619Z","iopub.execute_input":"2022-08-28T11:37:57.427108Z","iopub.status.idle":"2022-08-28T11:37:57.440745Z","shell.execute_reply.started":"2022-08-28T11:37:57.427068Z","shell.execute_reply":"2022-08-28T11:37:57.43966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain_glove.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:34:23.03473Z","iopub.execute_input":"2022-08-28T11:34:23.03517Z","iopub.status.idle":"2022-08-28T11:34:23.043679Z","shell.execute_reply.started":"2022-08-28T11:34:23.035131Z","shell.execute_reply":"2022-08-28T11:34:23.042072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xtrain_glove_1[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:10:36.396202Z","iopub.execute_input":"2022-08-28T12:10:36.396646Z","iopub.status.idle":"2022-08-28T12:10:36.405641Z","shell.execute_reply.started":"2022-08-28T12:10:36.396612Z","shell.execute_reply":"2022-08-28T12:10:36.404429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:34:27.734518Z","iopub.execute_input":"2022-08-28T11:34:27.734944Z","iopub.status.idle":"2022-08-28T11:34:27.743717Z","shell.execute_reply.started":"2022-08-28T11:34:27.734912Z","shell.execute_reply":"2022-08-28T11:34:27.742453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xvalid_glove.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:34:29.961835Z","iopub.execute_input":"2022-08-28T11:34:29.962283Z","iopub.status.idle":"2022-08-28T11:34:29.969464Z","shell.execute_reply.started":"2022-08-28T11:34:29.962241Z","shell.execute_reply":"2022-08-28T11:34:29.968606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:33:33.210168Z","iopub.status.idle":"2022-08-28T11:33:33.210811Z","shell.execute_reply.started":"2022-08-28T11:33:33.210505Z","shell.execute_reply":"2022-08-28T11:33:33.210534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fitting a simple Logistic Regression on TFIDF\nclf = LogisticRegression(C=1.0)\nclf.fit(xtrain_glove_1, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:11:06.314824Z","iopub.execute_input":"2022-08-28T12:11:06.315657Z","iopub.status.idle":"2022-08-28T12:11:06.343338Z","shell.execute_reply.started":"2022-08-28T12:11:06.315612Z","shell.execute_reply":"2022-08-28T12:11:06.341765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = xgb.XGBClassifier(nthread=6, silent=False)\nclf.fit(xtrain_glove.reshape((1,-1)), y_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:33:33.215968Z","iopub.status.idle":"2022-08-28T11:33:33.216544Z","shell.execute_reply.started":"2022-08-28T11:33:33.216307Z","shell.execute_reply":"2022-08-28T11:33:33.216332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_test = clf.predict(xvalid_glove)\naccuracy_score(y_test, y_pred_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T11:33:33.217927Z","iopub.status.idle":"2022-08-28T11:33:33.21839Z","shell.execute_reply.started":"2022-08-28T11:33:33.218148Z","shell.execute_reply":"2022-08-28T11:33:33.218169Z"},"trusted":true},"execution_count":null,"outputs":[]}]}