{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport nltk\nfrom nltk.tokenize import RegexpTokenizer\nfrom nltk.stem import PorterStemmer\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score, f1_score\nfrom scipy.sparse import hstack","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-28T18:03:45.294410Z","iopub.execute_input":"2022-10-28T18:03:45.295276Z","iopub.status.idle":"2022-10-28T18:03:47.189185Z","shell.execute_reply.started":"2022-10-28T18:03:45.295159Z","shell.execute_reply":"2022-10-28T18:03:47.187167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_path = \"../input/goodreads-books-reviews-290312/goodreads_sample_submission.csv\"\ntest_path = \"../input/goodreads-books-reviews-290312/goodreads_test.csv\"\ntrain_path = \"../input/goodreads-books-reviews-290312/goodreads_train.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:03:47.191769Z","iopub.execute_input":"2022-10-28T18:03:47.192976Z","iopub.status.idle":"2022-10-28T18:03:47.198406Z","shell.execute_reply.started":"2022-10-28T18:03:47.192927Z","shell.execute_reply":"2022-10-28T18:03:47.197180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(train_path)\ntest_df = pd.read_csv(test_path)\nsub_df = pd.read_csv(sub_path)","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:03:47.200342Z","iopub.execute_input":"2022-10-28T18:03:47.201050Z","iopub.status.idle":"2022-10-28T18:04:29.067353Z","shell.execute_reply.started":"2022-10-28T18:03:47.201005Z","shell.execute_reply":"2022-10-28T18:04:29.066126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:05:49.315942Z","iopub.execute_input":"2022-10-27T14:05:49.316355Z","iopub.status.idle":"2022-10-27T14:05:49.321614Z","shell.execute_reply.started":"2022-10-27T14:05:49.316318Z","shell.execute_reply":"2022-10-27T14:05:49.320331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:05:49.323135Z","iopub.execute_input":"2022-10-27T14:05:49.323569Z","iopub.status.idle":"2022-10-27T14:05:49.335793Z","shell.execute_reply.started":"2022-10-27T14:05:49.323533Z","shell.execute_reply":"2022-10-27T14:05:49.334319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.drop([\"user_id\", \"book_id\", \"review_id\", \"date_added\", \"date_updated\", \"read_at\", \"started_at\"], inplace=True, axis=1)\ntest_df.drop([\"user_id\", \"book_id\", \"review_id\", \"date_added\", \"date_updated\", \"read_at\", \"started_at\"], inplace=True, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:04:29.068834Z","iopub.execute_input":"2022-10-28T18:04:29.069252Z","iopub.status.idle":"2022-10-28T18:04:29.281581Z","shell.execute_reply.started":"2022-10-28T18:04:29.069204Z","shell.execute_reply":"2022-10-28T18:04:29.280678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # delete all rows with column 'rating' has value 0\n# index = train_df[ (train_df['rating'] == 0)].index\n# train_df.drop(index , inplace=True)\n# train_df.reset_index(inplace=True, drop=True)\n# train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-28T15:21:26.170939Z","iopub.execute_input":"2022-10-28T15:21:26.171842Z","iopub.status.idle":"2022-10-28T15:21:26.266778Z","shell.execute_reply.started":"2022-10-28T15:21:26.171797Z","shell.execute_reply":"2022-10-28T15:21:26.265532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(index)","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:05:49.561908Z","iopub.execute_input":"2022-10-27T14:05:49.562286Z","iopub.status.idle":"2022-10-27T14:05:49.575382Z","shell.execute_reply.started":"2022-10-27T14:05:49.562253Z","shell.execute_reply":"2022-10-27T14:05:49.574076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:05:49.590745Z","iopub.execute_input":"2022-10-27T14:05:49.591130Z","iopub.status.idle":"2022-10-27T14:05:49.600190Z","shell.execute_reply.started":"2022-10-27T14:05:49.591096Z","shell.execute_reply":"2022-10-27T14:05:49.599284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[:,'review_text'] = train_df.review_text.apply(lambda x : str.lower(x))\ntest_df.loc[:,'review_text'] = test_df.review_text.apply(lambda x : str.lower(x))","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:04:29.283765Z","iopub.execute_input":"2022-10-28T18:04:29.284184Z","iopub.status.idle":"2022-10-28T18:04:31.978754Z","shell.execute_reply.started":"2022-10-28T18:04:29.284153Z","shell.execute_reply":"2022-10-28T18:04:31.977581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_text = train_df['review_text']\ntest_text = test_df['review_text']\nall_text = pd.concat([train_text, test_text])","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:04:31.980389Z","iopub.execute_input":"2022-10-28T18:04:31.981262Z","iopub.status.idle":"2022-10-28T18:04:32.017121Z","shell.execute_reply.started":"2022-10-28T18:04:31.981228Z","shell.execute_reply":"2022-10-28T18:04:32.015942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords_list = stopwords.words('english')\nstopwords_list = stopwords_list[:116]\n# stopwords_list = stopwords_list.extend(['d', 'll', 're', 's', 've'])","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:04:32.018780Z","iopub.execute_input":"2022-10-28T18:04:32.019140Z","iopub.status.idle":"2022-10-28T18:04:32.029816Z","shell.execute_reply.started":"2022-10-28T18:04:32.019108Z","shell.execute_reply":"2022-10-28T18:04:32.028964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords_list.extend(['d', 'll', 're', 's', 've', 'http', 'www', 'com', 'ani', 'becaus', 'befor', 'doe', 'dure', 'ha', 'hi', 'onc', 'ourselv', 'themselv', 'thi', 'wa', 'whi', 'yourselv'])","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:52:17.166583Z","iopub.execute_input":"2022-10-28T18:52:17.166963Z","iopub.status.idle":"2022-10-28T18:52:17.173241Z","shell.execute_reply.started":"2022-10-28T18:52:17.166934Z","shell.execute_reply":"2022-10-28T18:52:17.172400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from nltk.corpus import stopwords\n# stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:10:01.895575Z","iopub.execute_input":"2022-10-28T18:10:01.896000Z","iopub.status.idle":"2022-10-28T18:10:01.899774Z","shell.execute_reply.started":"2022-10-28T18:10:01.895966Z","shell.execute_reply":"2022-10-28T18:10:01.898959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stemmer = PorterStemmer()\ntokenizer = RegexpTokenizer(r'\\w+') \ndef tokenize(text):\n    tokens = tokenizer.tokenize(text)\n    tokens = [token for token in tokens if token not in stopwords_list]\n    stems = [stemmer.stem(item) for item in tokens]\n    return stems","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:51:51.931479Z","iopub.execute_input":"2022-10-28T18:51:51.931891Z","iopub.status.idle":"2022-10-28T18:51:51.937915Z","shell.execute_reply.started":"2022-10-28T18:51:51.931858Z","shell.execute_reply":"2022-10-28T18:51:51.936525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nword_vectorizer = TfidfVectorizer(\n    sublinear_tf=True,\n    strip_accents='unicode',\n    analyzer='word',\n    tokenizer=tokenize,\n    stop_words=stopwords_list,\n    ngram_range=(1, 1),\n    max_features=10000)\nword_vectorizer.fit(all_text)\ntrain_word_features = word_vectorizer.transform(train_text)\ntest_word_features = word_vectorizer.transform(test_text) ","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:52:20.931041Z","iopub.execute_input":"2022-10-28T18:52:20.931457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_vectorizer.vocabulary_","metadata":{"execution":{"iopub.status.busy":"2022-10-28T18:17:13.791747Z","iopub.execute_input":"2022-10-28T18:17:13.792120Z","iopub.status.idle":"2022-10-28T18:17:13.840366Z","shell.execute_reply.started":"2022-10-28T18:17:13.792075Z","shell.execute_reply":"2022-10-28T18:17:13.839048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_word_features","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:12:45.276778Z","iopub.execute_input":"2022-10-27T14:12:45.277196Z","iopub.status.idle":"2022-10-27T14:12:45.282576Z","shell.execute_reply.started":"2022-10-27T14:12:45.277159Z","shell.execute_reply":"2022-10-27T14:12:45.281353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.array(train_df['n_votes']).shape","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:12:45.284202Z","iopub.execute_input":"2022-10-27T14:12:45.284665Z","iopub.status.idle":"2022-10-27T14:12:45.296343Z","shell.execute_reply.started":"2022-10-27T14:12:45.284617Z","shell.execute_reply":"2022-10-27T14:12:45.295189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_features = hstack( [train_word_features, np.array(train_df['n_votes']).reshape((-1, 1)), np.array(train_df['n_comments']).reshape((-1, 1))])\n# test_features = hstack([test_word_features, np.array(test_df['n_votes']).reshape((-1, 1)), np.array(test_df['n_comments']).reshape((-1, 1))])","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:12:45.297694Z","iopub.execute_input":"2022-10-27T14:12:45.298851Z","iopub.status.idle":"2022-10-27T14:12:45.311409Z","shell.execute_reply.started":"2022-10-27T14:12:45.298809Z","shell.execute_reply":"2022-10-27T14:12:45.310234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features = hstack([train_word_features])\ntest_features = hstack([test_word_features])","metadata":{"execution":{"iopub.status.busy":"2022-10-28T15:31:06.335309Z","iopub.execute_input":"2022-10-28T15:31:06.335722Z","iopub.status.idle":"2022-10-28T15:31:06.802340Z","shell.execute_reply.started":"2022-10-28T15:31:06.335690Z","shell.execute_reply":"2022-10-28T15:31:06.801313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_features","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:12:45.896374Z","iopub.execute_input":"2022-10-27T14:12:45.896710Z","iopub.status.idle":"2022-10-27T14:12:45.902009Z","shell.execute_reply.started":"2022-10-27T14:12:45.896680Z","shell.execute_reply":"2022-10-27T14:12:45.900784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target = train_df['rating']\nclassifier = LogisticRegression(solver='sag')\nclassifier.fit(train_features, train_target)","metadata":{"execution":{"iopub.status.busy":"2022-10-28T15:31:09.359179Z","iopub.execute_input":"2022-10-28T15:31:09.359963Z","iopub.status.idle":"2022-10-28T15:32:25.270288Z","shell.execute_reply.started":"2022-10-28T15:31:09.359921Z","shell.execute_reply":"2022-10-28T15:32:25.269496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = classifier.predict(test_features)","metadata":{"execution":{"iopub.status.busy":"2022-10-28T15:32:25.272047Z","iopub.execute_input":"2022-10-28T15:32:25.272702Z","iopub.status.idle":"2022-10-28T15:32:25.497677Z","shell.execute_reply.started":"2022-10-28T15:32:25.272665Z","shell.execute_reply":"2022-10-28T15:32:25.496520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred","metadata":{"execution":{"iopub.status.busy":"2022-10-28T15:32:25.498972Z","iopub.execute_input":"2022-10-28T15:32:25.499327Z","iopub.status.idle":"2022-10-28T15:32:25.506503Z","shell.execute_reply.started":"2022-10-28T15:32:25.499295Z","shell.execute_reply":"2022-10-28T15:32:25.505343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df['rating'] = test_pred","metadata":{"execution":{"iopub.status.busy":"2022-10-28T15:32:25.509154Z","iopub.execute_input":"2022-10-28T15:32:25.509959Z","iopub.status.idle":"2022-10-28T15:32:25.518511Z","shell.execute_reply.started":"2022-10-28T15:32:25.509911Z","shell.execute_reply":"2022-10-28T15:32:25.517156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:45:51.263041Z","iopub.status.idle":"2022-10-27T14:45:51.263460Z","shell.execute_reply.started":"2022-10-27T14:45:51.263260Z","shell.execute_reply":"2022-10-27T14:45:51.263280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.rating.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-28T15:32:25.520007Z","iopub.execute_input":"2022-10-28T15:32:25.520453Z","iopub.status.idle":"2022-10-28T15:32:25.535503Z","shell.execute_reply.started":"2022-10-28T15:32:25.520416Z","shell.execute_reply":"2022-10-28T15:32:25.534385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.rating.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:26:46.096009Z","iopub.execute_input":"2022-10-27T14:26:46.096508Z","iopub.status.idle":"2022-10-27T14:26:46.110639Z","shell.execute_reply.started":"2022-10-27T14:26:46.096468Z","shell.execute_reply":"2022-10-27T14:26:46.109681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.rating.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-27T14:26:47.081626Z","iopub.execute_input":"2022-10-27T14:26:47.082476Z","iopub.status.idle":"2022-10-27T14:26:47.101636Z","shell.execute_reply.started":"2022-10-27T14:26:47.082416Z","shell.execute_reply":"2022-10-27T14:26:47.100295Z"},"trusted":true},"execution_count":null,"outputs":[]}]}