{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-25T21:31:12.922558Z","iopub.execute_input":"2022-08-25T21:31:12.923839Z","iopub.status.idle":"2022-08-25T21:31:12.956016Z","shell.execute_reply.started":"2022-08-25T21:31:12.923713Z","shell.execute_reply":"2022-08-25T21:31:12.954637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ndf_train = pd.read_csv(\"../input/goodreads-books-reviews-290312/goodreads_train.csv\")\ndf_test = pd.read_csv(\"../input/goodreads-books-reviews-290312/goodreads_test.csv\")\ndf_sub = pd.read_csv(\"../input/goodreads-books-reviews-290312/goodreads_sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:12.958916Z","iopub.execute_input":"2022-08-25T21:31:12.960164Z","iopub.status.idle":"2022-08-25T21:31:55.256871Z","shell.execute_reply.started":"2022-08-25T21:31:12.960109Z","shell.execute_reply":"2022-08-25T21:31:55.255633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.shape, df_test.shape)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:55.259653Z","iopub.execute_input":"2022-08-25T21:31:55.260038Z","iopub.status.idle":"2022-08-25T21:31:55.289836Z","shell.execute_reply.started":"2022-08-25T21:31:55.260004Z","shell.execute_reply":"2022-08-25T21:31:55.288499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.isna().sum())\nprint(\"-----------\")\nprint(df_test.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:55.291412Z","iopub.execute_input":"2022-08-25T21:31:55.291726Z","iopub.status.idle":"2022-08-25T21:31:55.860852Z","shell.execute_reply.started":"2022-08-25T21:31:55.291698Z","shell.execute_reply":"2022-08-25T21:31:55.859595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train['user_id'].unique()), len(df_train['book_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:55.863606Z","iopub.execute_input":"2022-08-25T21:31:55.864066Z","iopub.status.idle":"2022-08-25T21:31:56.005825Z","shell.execute_reply.started":"2022-08-25T21:31:55.864029Z","shell.execute_reply":"2022-08-25T21:31:56.004712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['rating'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.008606Z","iopub.execute_input":"2022-08-25T21:31:56.009088Z","iopub.status.idle":"2022-08-25T21:31:56.029440Z","shell.execute_reply.started":"2022-08-25T21:31:56.009038Z","shell.execute_reply":"2022-08-25T21:31:56.028129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vote_counts= df_train['n_votes'].value_counts().reset_index(drop=False).sort_values(by=\"index\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.031196Z","iopub.execute_input":"2022-08-25T21:31:56.031654Z","iopub.status.idle":"2022-08-25T21:31:56.055152Z","shell.execute_reply.started":"2022-08-25T21:31:56.031600Z","shell.execute_reply":"2022-08-25T21:31:56.054201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(df_train['n_votes'].to_numpy().clip(max=20), bins=100)\nplt.show() \n","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.057898Z","iopub.execute_input":"2022-08-25T21:31:56.059285Z","iopub.status.idle":"2022-08-25T21:31:56.460648Z","shell.execute_reply.started":"2022-08-25T21:31:56.059229Z","shell.execute_reply":"2022-08-25T21:31:56.459299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_user_list = df_train['book_id'].value_counts().reset_index()\ntest_user_list = df_test['book_id'].value_counts().reset_index()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.462403Z","iopub.execute_input":"2022-08-25T21:31:56.462875Z","iopub.status.idle":"2022-08-25T21:31:56.517541Z","shell.execute_reply.started":"2022-08-25T21:31:56.462827Z","shell.execute_reply":"2022-08-25T21:31:56.516651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_user_list[train_user_list['book_id']==1].shape","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.518965Z","iopub.execute_input":"2022-08-25T21:31:56.519833Z","iopub.status.idle":"2022-08-25T21:31:56.527955Z","shell.execute_reply.started":"2022-08-25T21:31:56.519798Z","shell.execute_reply":"2022-08-25T21:31:56.526916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_test_users = list(set(train_user_list['index']) - set(test_user_list['index']))","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.531997Z","iopub.execute_input":"2022-08-25T21:31:56.532796Z","iopub.status.idle":"2022-08-25T21:31:56.552773Z","shell.execute_reply.started":"2022-08-25T21:31:56.532751Z","shell.execute_reply":"2022-08-25T21:31:56.551695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_user_ids = list(train_user_list['index'])\ntest_user_ids = list(test_user_list['index'])\n","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.554448Z","iopub.execute_input":"2022-08-25T21:31:56.555622Z","iopub.status.idle":"2022-08-25T21:31:56.569088Z","shell.execute_reply.started":"2022-08-25T21:31:56.555583Z","shell.execute_reply":"2022-08-25T21:31:56.567906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(list(set(train_user_ids) & set(test_user_ids)))","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.570714Z","iopub.execute_input":"2022-08-25T21:31:56.571921Z","iopub.status.idle":"2022-08-25T21:31:56.597665Z","shell.execute_reply.started":"2022-08-25T21:31:56.571878Z","shell.execute_reply":"2022-08-25T21:31:56.596405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books_train = df_train['book_id'].value_counts(normalize=False).reset_index()\nbooks_test = df_test['book_id'].value_counts(normalize=False).reset_index()\nbooks = pd.merge(\n    books_train,\n    books_test,\n    how='outer',\n    on=['index']\n)\nbooks = books.fillna(0.0)\nbooks= books.sort_values('book_id_x', ascending=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.599189Z","iopub.execute_input":"2022-08-25T21:31:56.599949Z","iopub.status.idle":"2022-08-25T21:31:56.663757Z","shell.execute_reply.started":"2022-08-25T21:31:56.599904Z","shell.execute_reply":"2022-08-25T21:31:56.662587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"books","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.665443Z","iopub.execute_input":"2022-08-25T21:31:56.666077Z","iopub.status.idle":"2022-08-25T21:31:56.681288Z","shell.execute_reply.started":"2022-08-25T21:31:56.666041Z","shell.execute_reply":"2022-08-25T21:31:56.680020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decider = df_train.groupby(by=['book_id']).agg({\n    'rating': ['mean', 'std'],\n    'n_votes': ['sum'], \n    'n_comments': ['sum'],\n    'user_id': ['count']\n}).reset_index()\n\ndecider.columns = ['_'.join(col) for col in decider.columns.values]\ndecider","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.683044Z","iopub.execute_input":"2022-08-25T21:31:56.683788Z","iopub.status.idle":"2022-08-25T21:31:56.886882Z","shell.execute_reply.started":"2022-08-25T21:31:56.683728Z","shell.execute_reply":"2022-08-25T21:31:56.885641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decider[decider['rating_std']>1.0].sort_values('rating_mean')","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.888351Z","iopub.execute_input":"2022-08-25T21:31:56.888749Z","iopub.status.idle":"2022-08-25T21:31:56.911796Z","shell.execute_reply.started":"2022-08-25T21:31:56.888716Z","shell.execute_reply":"2022-08-25T21:31:56.910508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(decider['user_id_count'], decider['rating_mean'])\nplt.xlabel('user_id_count')\nplt.ylabel('rating_mean')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:56.913266Z","iopub.execute_input":"2022-08-25T21:31:56.913832Z","iopub.status.idle":"2022-08-25T21:31:57.171144Z","shell.execute_reply.started":"2022-08-25T21:31:56.913798Z","shell.execute_reply":"2022-08-25T21:31:57.170264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(decider['user_id_count'], decider['rating_std'])\nplt.xlabel('user_id_count')\nplt.ylabel('rating_std')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:57.172587Z","iopub.execute_input":"2022-08-25T21:31:57.172949Z","iopub.status.idle":"2022-08-25T21:31:57.437293Z","shell.execute_reply.started":"2022-08-25T21:31:57.172914Z","shell.execute_reply":"2022-08-25T21:31:57.436314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport math\ndecider_ind = decider.set_index('book_id_')\ndec_prob = decider_ind.to_dict('index')\nresult = []\nfor index, row in tqdm(df_test.iterrows(), total=df_test.shape[0]):\n    if row['book_id'] in dec_prob:\n        rating_mean = dec_prob[row['book_id']]['rating_mean']\n        rating_std = dec_prob[row['book_id']]['rating_std']\n        result.append(int(round(rating_mean, 0)))\n        \n    else:\n        result.append(4)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:31:57.438543Z","iopub.execute_input":"2022-08-25T21:31:57.439884Z","iopub.status.idle":"2022-08-25T21:32:29.944273Z","shell.execute_reply.started":"2022-08-25T21:31:57.439842Z","shell.execute_reply":"2022-08-25T21:32:29.943075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['rating'] = result\ndf_test_sub = df_test[['review_id', 'rating']]\ndf_test_sub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-25T21:32:29.946442Z","iopub.execute_input":"2022-08-25T21:32:29.946889Z","iopub.status.idle":"2022-08-25T21:32:30.829754Z","shell.execute_reply.started":"2022-08-25T21:32:29.946852Z","shell.execute_reply":"2022-08-25T21:32:30.828119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}