{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":33579,"databundleVersionId":3073370,"sourceType":"competition"}],"dockerImageVersionId":30357,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, TfidfTransformer\nfrom sklearn.pipeline import Pipeline, FeatureUnion\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import FunctionTransformer\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold, train_test_split\n\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem.porter import PorterStemmer\nimport re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-12-04T18:20:42.806762Z","iopub.execute_input":"2024-12-04T18:20:42.807051Z","iopub.status.idle":"2024-12-04T18:20:44.521612Z","shell.execute_reply.started":"2024-12-04T18:20:42.806991Z","shell.execute_reply":"2024-12-04T18:20:44.520615Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\npd.set_option('max_colwidth', 500)","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:57:29.406597Z","iopub.execute_input":"2024-12-04T18:57:29.407468Z","iopub.status.idle":"2024-12-04T18:57:29.412119Z","shell.execute_reply.started":"2024-12-04T18:57:29.407415Z","shell.execute_reply":"2024-12-04T18:57:29.411278Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"custom_style = {'axes.labelcolor': 'white',\n                'xtick.color': 'black',\n                'ytick.color': 'black',\n                'text.color': 'black'}\nsns.set_style(\"darkgrid\", rc=custom_style)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:57:34.201388Z","iopub.execute_input":"2024-12-04T18:57:34.202064Z","iopub.status.idle":"2024-12-04T18:57:34.206476Z","shell.execute_reply.started":"2024-12-04T18:57:34.202034Z","shell.execute_reply":"2024-12-04T18:57:34.205618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def barchart(data, x, y):\n   sns.barplot(x = x,\n               y = y,\n               data = data)\n   plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:57:35.810464Z","iopub.execute_input":"2024-12-04T18:57:35.811140Z","iopub.status.idle":"2024-12-04T18:57:35.815325Z","shell.execute_reply.started":"2024-12-04T18:57:35.811107Z","shell.execute_reply":"2024-12-04T18:57:35.814341Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def piechart(data, values, labels):\n    colors = sns.color_palette('pastel')[0:5]\n    explode = [0.1, 0, 0, 0, 0, 0]\n    plt.pie(data[values], labels=data[labels], colors=colors, explode=explode, autopct='%.0f%%')\n    plt.show()\n          ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:57:37.612209Z","iopub.execute_input":"2024-12-04T18:57:37.612932Z","iopub.status.idle":"2024-12-04T18:57:37.617707Z","shell.execute_reply.started":"2024-12-04T18:57:37.612899Z","shell.execute_reply":"2024-12-04T18:57:37.616804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_text(text):\n    '''Make text lowercase, remove text in square brackets,remove links,remove punctuation\n    and remove words containing numbers.'''\n    text = str(text).lower()\n    text = re.sub('\\[.*?\\]', '', text)\n    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n    text = re.sub('<.*?>+', '', text)\n    text = re.sub('\\n', '', text)\n    text = re.sub('\\w*\\d\\w*', '', text)\n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:57:51.245939Z","iopub.execute_input":"2024-12-04T18:57:51.246923Z","iopub.status.idle":"2024-12-04T18:57:51.252360Z","shell.execute_reply.started":"2024-12-04T18:57:51.246889Z","shell.execute_reply":"2024-12-04T18:57:51.251424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stop_words = stopwords.words('english')\nmore_stopwords = ['u', 'im', 'c']\nstop_words = stop_words + more_stopwords","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:57:54.456600Z","iopub.execute_input":"2024-12-04T18:57:54.456937Z","iopub.status.idle":"2024-12-04T18:57:54.465984Z","shell.execute_reply.started":"2024-12-04T18:57:54.456901Z","shell.execute_reply":"2024-12-04T18:57:54.465079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stemmer = PorterStemmer()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:57:56.416181Z","iopub.execute_input":"2024-12-04T18:57:56.416849Z","iopub.status.idle":"2024-12-04T18:57:56.420992Z","shell.execute_reply.started":"2024-12-04T18:57:56.416816Z","shell.execute_reply":"2024-12-04T18:57:56.420004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_review(text):\n    # Clean puntuation, urls, and so on\n    text = clean_text(text)\n    # Remove stopwords\n    text = ' '.join(word for word in text.split(' ') if word not in stop_words)\n    # Stemm all the words in the sentence\n    text = ' '.join(stemmer.stem(word) for word in text.split(' '))\n    \n    return text","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T18:57:58.137459Z","iopub.execute_input":"2024-12-04T18:57:58.137857Z","iopub.status.idle":"2024-12-04T18:57:58.142743Z","shell.execute_reply.started":"2024-12-04T18:57:58.137815Z","shell.execute_reply":"2024-12-04T18:57:58.141823Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Classification in training dataset","metadata":{}},{"cell_type":"code","source":"train_df=pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:58:00.527765Z","iopub.execute_input":"2024-12-04T18:58:00.528089Z","iopub.status.idle":"2024-12-04T18:58:23.498087Z","shell.execute_reply.started":"2024-12-04T18:58:00.528060Z","shell.execute_reply":"2024-12-04T18:58:23.497171Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:58:29.570801Z","iopub.execute_input":"2024-12-04T18:58:29.571712Z","iopub.status.idle":"2024-12-04T18:58:29.578954Z","shell.execute_reply.started":"2024-12-04T18:58:29.571669Z","shell.execute_reply":"2024-12-04T18:58:29.578057Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:58:32.751895Z","iopub.execute_input":"2024-12-04T18:58:32.752670Z","iopub.status.idle":"2024-12-04T18:58:33.095665Z","shell.execute_reply.started":"2024-12-04T18:58:32.752635Z","shell.execute_reply":"2024-12-04T18:58:33.094806Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sns.heatmap(train_df.isnull())","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:58:36.432669Z","iopub.execute_input":"2024-12-04T18:58:36.433216Z","iopub.status.idle":"2024-12-04T18:58:47.827119Z","shell.execute_reply.started":"2024-12-04T18:58:36.433171Z","shell.execute_reply":"2024-12-04T18:58:47.826235Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"r_bid=train_df.groupby('book_id').rating.mean().sort_values(ascending=False).reset_index().head(20)\nr_bid.rename({'rating':'mean_rating'}, axis=1, inplace=True)\nr_bid","metadata":{"execution":{"iopub.status.busy":"2024-12-03T00:08:13.992831Z","iopub.execute_input":"2024-12-03T00:08:13.993230Z","iopub.status.idle":"2024-12-03T00:08:14.075462Z","shell.execute_reply.started":"2024-12-03T00:08:13.993200Z","shell.execute_reply":"2024-12-03T00:08:14.074463Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nr_users=train_df.rating.value_counts(normalize=True).reset_index()\nnr_users.columns='rating rating_weight'.split()\nnr_users","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:05.841208Z","iopub.execute_input":"2024-12-04T18:59:05.842285Z","iopub.status.idle":"2024-12-04T18:59:05.861834Z","shell.execute_reply.started":"2024-12-04T18:59:05.842247Z","shell.execute_reply":"2024-12-04T18:59:05.860854Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"piechart(nr_users, 'rating_weight', 'rating')       ","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:09.572105Z","iopub.execute_input":"2024-12-04T18:59:09.572794Z","iopub.status.idle":"2024-12-04T18:59:09.681820Z","shell.execute_reply.started":"2024-12-04T18:59:09.572761Z","shell.execute_reply":"2024-12-04T18:59:09.680563Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATE_FORMAT = '%a %b %d %H:%M:%S %z %Y'\n\ndef parse_date(date_str):\n    if (date_str != date_str) or (date_str is None):\n        return None\n    if len(date_str.split()) != 6:\n        return None\n    else:\n        try:\n            return datetime.strptime(date_str, DATE_FORMAT)\n        except Exception as e:\n            print(date_str, e)\n        return None","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:13.896660Z","iopub.execute_input":"2024-12-04T18:59:13.897279Z","iopub.status.idle":"2024-12-04T18:59:13.903040Z","shell.execute_reply.started":"2024-12-04T18:59:13.897249Z","shell.execute_reply":"2024-12-04T18:59:13.902049Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datetime import datetime\ntrain_df['started_at_dt'] = train_df.started_at.apply(lambda x : parse_date(x))\ntrain_df['read_at_dt'] = train_df.read_at.apply(lambda x : parse_date(x))\ntrain_df['duration_dt'] = train_df.read_at_dt - train_df.started_at_dt","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:17.471289Z","iopub.execute_input":"2024-12-04T18:59:17.471982Z","iopub.status.idle":"2024-12-04T18:59:38.773525Z","shell.execute_reply.started":"2024-12-04T18:59:17.471946Z","shell.execute_reply":"2024-12-04T18:59:38.772469Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.duration_dt=train_df.duration_dt.astype(str).apply(lambda x: 'nan' if 'nan' in x else(x[:x.find('days')] if 'days' in x else 0)).astype('float64')","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:42.202163Z","iopub.execute_input":"2024-12-04T18:59:42.203333Z","iopub.status.idle":"2024-12-04T18:59:43.181198Z","shell.execute_reply.started":"2024-12-04T18:59:42.203283Z","shell.execute_reply":"2024-12-04T18:59:43.180468Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.duration_dt.fillna(train_df.duration_dt.mean(), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:45.035210Z","iopub.execute_input":"2024-12-04T18:59:45.036046Z","iopub.status.idle":"2024-12-04T18:59:45.049576Z","shell.execute_reply.started":"2024-12-04T18:59:45.036002Z","shell.execute_reply":"2024-12-04T18:59:45.048769Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df2=train_df[train_df.duration_dt.isin(range(110))]\ntrain_df2.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:47.251860Z","iopub.execute_input":"2024-12-04T18:59:47.252497Z","iopub.status.idle":"2024-12-04T18:59:47.760601Z","shell.execute_reply.started":"2024-12-04T18:59:47.252463Z","shell.execute_reply":"2024-12-04T18:59:47.759766Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df2.groupby('rating').duration_dt.mean()","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:50.116844Z","iopub.execute_input":"2024-12-04T18:59:50.117172Z","iopub.status.idle":"2024-12-04T18:59:50.134326Z","shell.execute_reply.started":"2024-12-04T18:59:50.117142Z","shell.execute_reply":"2024-12-04T18:59:50.133371Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rat_mean_days=train_df2.groupby('rating').duration_dt.mean().reset_index()\nrat_mean_days.rename({'duration_dt':'mean_n_days_to_read'}, axis=1, inplace=True)\nrat_mean_days","metadata":{"execution":{"iopub.status.busy":"2024-12-04T18:59:52.622177Z","iopub.execute_input":"2024-12-04T18:59:52.622521Z","iopub.status.idle":"2024-12-04T18:59:52.641729Z","shell.execute_reply.started":"2024-12-04T18:59:52.622491Z","shell.execute_reply":"2024-12-04T18:59:52.640930Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"barchart(rat_mean_days, 'rating', 'mean_n_days_to_read')","metadata":{"execution":{"iopub.status.busy":"2024-12-04T19:00:21.485600Z","iopub.execute_input":"2024-12-04T19:00:21.486294Z","iopub.status.idle":"2024-12-04T19:00:21.700076Z","shell.execute_reply.started":"2024-12-04T19:00:21.486259Z","shell.execute_reply":"2024-12-04T19:00:21.699297Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df2['review_clean']=train_df2['review_text'].apply(preprocess_review)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:00:33.765169Z","iopub.execute_input":"2024-12-04T19:00:33.765822Z","iopub.status.idle":"2024-12-04T19:24:36.414002Z","shell.execute_reply.started":"2024-12-04T19:00:33.765787Z","shell.execute_reply":"2024-12-04T19:24:36.412917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features='n_votes n_comments duration_dt review_clean rating'.split()\nX=train_df2[features]\nY=X.pop('rating')","metadata":{"execution":{"iopub.status.busy":"2024-12-04T19:29:57.862835Z","iopub.execute_input":"2024-12-04T19:29:57.863590Z","iopub.status.idle":"2024-12-04T19:29:58.264892Z","shell.execute_reply.started":"2024-12-04T19:29:57.863552Z","shell.execute_reply":"2024-12-04T19:29:58.263809Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_test, Y_train, Y_test=train_test_split(X,Y, test_size=0.3, random_state=42)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy import sparse\n\nnum_feats_train = X_train['n_votes n_comments duration_dt'.split()].values\nnum_feats_test = X_test['n_votes n_comments duration_dt'.split()].values\n\nvectorizer = TfidfVectorizer()\ntf_x_train = vectorizer.fit_transform(X_train.review_clean)\ntf_x_test = vectorizer.transform(X_test.review_clean)\n\ntf_x_train = sparse.hstack((tf_x_train, num_feats_train))\ntf_x_test = sparse.hstack((tf_x_test, num_feats_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:30:02.363525Z","iopub.execute_input":"2024-12-04T19:30:02.364183Z","iopub.status.idle":"2024-12-04T19:30:49.113897Z","shell.execute_reply.started":"2024-12-04T19:30:02.364147Z","shell.execute_reply":"2024-12-04T19:30:49.113158Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 1. LinearSVC","metadata":{}},{"cell_type":"code","source":"from sklearn.svm import LinearSVC\nclf = LinearSVC(random_state=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:31:28.681830Z","iopub.execute_input":"2024-12-04T19:31:28.682174Z","iopub.status.idle":"2024-12-04T19:31:28.686337Z","shell.execute_reply.started":"2024-12-04T19:31:28.682142Z","shell.execute_reply":"2024-12-04T19:31:28.685453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"clf.fit(tf_x_train, Y_train)\ny_pred = clf.predict(tf_x_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T19:31:32.954143Z","iopub.execute_input":"2024-12-04T19:31:32.954890Z","iopub.status.idle":"2024-12-04T19:58:27.373257Z","shell.execute_reply.started":"2024-12-04T19:31:32.954850Z","shell.execute_reply":"2024-12-04T19:58:27.372289Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(Y_test, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:04:47.709007Z","iopub.execute_input":"2024-12-04T20:04:47.709716Z","iopub.status.idle":"2024-12-04T20:04:48.011238Z","shell.execute_reply.started":"2024-12-04T20:04:47.709679Z","shell.execute_reply":"2024-12-04T20:04:48.010343Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Finding best parameters for model using cross validation (not applied on first run)","metadata":{}},{"cell_type":"code","source":"# param_grid = {'clf__n_estimators': np.linspace(1, 100, 10, dtype=int),\n#               'clf__min_samples_split': [3, 10],\n#               'clf__min_samples_leaf': [3],\n#               'clf__max_features': [7],\n#               'clf__max_depth': [None],\n#               'clf__criterion': ['gini'],\n#               'clf__bootstrap': [False]}","metadata":{"execution":{"iopub.status.busy":"2024-11-28T01:52:59.319327Z","iopub.execute_input":"2024-11-28T01:52:59.319803Z","iopub.status.idle":"2024-11-28T01:52:59.327244Z","shell.execute_reply.started":"2024-11-28T01:52:59.319742Z","shell.execute_reply":"2024-11-28T01:52:59.325858Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# kfold = StratifiedKFold(n_splits=5)\n# scoring = {'Accuracy': 'accuracy', 'F1': 'f1_macro'}\n# refit = 'F1'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# rf_model = GridSearchCV(clf, param_grid=param_grid, cv=kfold, scoring=scoring, \n#                          refit=refit, n_jobs=-1, return_train_score=True, verbose=1)\n# rf_model.fit(X_train, Y_train)\n# rf_best = rf_model.best_estimator_","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### 2. Logistic Regression","metadata":{}},{"cell_type":"code","source":"clf2 = LogisticRegression(max_iter=1000,solver='saga')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T20:04:42.478602Z","iopub.execute_input":"2024-12-02T20:04:42.478922Z","iopub.status.idle":"2024-12-02T20:04:42.483354Z","shell.execute_reply.started":"2024-12-02T20:04:42.478897Z","shell.execute_reply":"2024-12-02T20:04:42.482407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.concat([pd.Series(Y_test).reset_index(drop=True), pd.Series(y_pred).rename('predicted_rating').reset_index(drop=True)], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T20:04:44.703250Z","iopub.execute_input":"2024-12-02T20:04:44.703961Z","iopub.status.idle":"2024-12-02T20:04:44.716435Z","shell.execute_reply.started":"2024-12-02T20:04:44.703932Z","shell.execute_reply":"2024-12-02T20:04:44.715568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"clf2.fit(tf_x_train, Y_train)\ny_pred = clf2.predict(tf_x_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T20:04:54.774761Z","iopub.execute_input":"2024-12-02T20:04:54.775459Z","iopub.status.idle":"2024-12-02T20:36:00.397380Z","shell.execute_reply.started":"2024-12-02T20:04:54.775431Z","shell.execute_reply":"2024-12-02T20:36:00.396400Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(classification_report(Y_test, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T20:40:08.255438Z","iopub.execute_input":"2024-12-02T20:40:08.256148Z","iopub.status.idle":"2024-12-02T20:40:08.525990Z","shell.execute_reply.started":"2024-12-02T20:40:08.256117Z","shell.execute_reply":"2024-12-02T20:40:08.524919Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Classification on test set","metadata":{}},{"cell_type":"code","source":"test_df=pd.read_csv('/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-04T20:04:58.294873Z","iopub.execute_input":"2024-12-04T20:04:58.295205Z","iopub.status.idle":"2024-12-04T20:05:10.916042Z","shell.execute_reply.started":"2024-12-04T20:04:58.295176Z","shell.execute_reply":"2024-12-04T20:05:10.915078Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:05:12.368914Z","iopub.execute_input":"2024-12-04T20:05:12.369629Z","iopub.status.idle":"2024-12-04T20:05:12.556134Z","shell.execute_reply.started":"2024-12-04T20:05:12.369596Z","shell.execute_reply":"2024-12-04T20:05:12.555150Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from datetime import datetime\ntest_df['started_at_dt'] = test_df.started_at.apply(lambda x : parse_date(x))\ntest_df['read_at_dt'] = test_df.read_at.apply(lambda x : parse_date(x))\ntest_df['duration_dt'] = test_df.read_at_dt - test_df.started_at_dt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:05:14.593281Z","iopub.execute_input":"2024-12-04T20:05:14.594125Z","iopub.status.idle":"2024-12-04T20:05:26.524741Z","shell.execute_reply.started":"2024-12-04T20:05:14.594076Z","shell.execute_reply":"2024-12-04T20:05:26.523969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.duration_dt=test_df.duration_dt.astype(str).apply(lambda x: 'nan' if 'nan' in x else(x[:x.find('days')] if 'days' in x else 0)).astype('float64')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:05:30.232355Z","iopub.execute_input":"2024-12-04T20:05:30.232768Z","iopub.status.idle":"2024-12-04T20:05:30.775568Z","shell.execute_reply.started":"2024-12-04T20:05:30.232736Z","shell.execute_reply":"2024-12-04T20:05:30.774538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.duration_dt.fillna(test_df.duration_dt.mean(), inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:05:32.876245Z","iopub.execute_input":"2024-12-04T20:05:32.876596Z","iopub.status.idle":"2024-12-04T20:05:32.886850Z","shell.execute_reply.started":"2024-12-04T20:05:32.876565Z","shell.execute_reply":"2024-12-04T20:05:32.886012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['review_clean']=test_df['review_text'].apply(preprocess_review)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:05:37.678995Z","iopub.execute_input":"2024-12-04T20:05:37.679305Z","iopub.status.idle":"2024-12-04T20:23:13.766014Z","shell.execute_reply.started":"2024-12-04T20:05:37.679279Z","shell.execute_reply":"2024-12-04T20:23:13.764979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:26:27.281972Z","iopub.execute_input":"2024-12-04T20:26:27.282308Z","iopub.status.idle":"2024-12-04T20:26:27.298305Z","shell.execute_reply.started":"2024-12-04T20:26:27.282278Z","shell.execute_reply":"2024-12-04T20:26:27.297253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_review_vect=vectorizer.transform(test_df.review_clean)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:32:13.167355Z","iopub.execute_input":"2024-12-04T20:32:13.167699Z","iopub.status.idle":"2024-12-04T20:32:46.091301Z","shell.execute_reply.started":"2024-12-04T20:32:13.167674Z","shell.execute_reply":"2024-12-04T20:32:46.090394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_num_feats=test_df['n_votes n_comments duration_dt'.split()].values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:33:33.687508Z","iopub.execute_input":"2024-12-04T20:33:33.687849Z","iopub.status.idle":"2024-12-04T20:33:33.929672Z","shell.execute_reply.started":"2024-12-04T20:33:33.687821Z","shell.execute_reply":"2024-12-04T20:33:33.928909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_input=sparse.hstack((test_review_vect, test_num_feats))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:34:43.697928Z","iopub.execute_input":"2024-12-04T20:34:43.698266Z","iopub.status.idle":"2024-12-04T20:34:44.588090Z","shell.execute_reply.started":"2024-12-04T20:34:43.698236Z","shell.execute_reply":"2024-12-04T20:34:44.587197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['rating']=clf.predict(test_input)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T20:36:01.243102Z","iopub.execute_input":"2024-12-04T20:36:01.243699Z","iopub.status.idle":"2024-12-04T20:36:03.905193Z","shell.execute_reply.started":"2024-12-04T20:36:01.243665Z","shell.execute_reply":"2024-12-04T20:36:03.904452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['review_id rating'.split()].to_csv(r'goodreads_submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}