{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-15T09:23:02.301673Z","iopub.execute_input":"2022-04-15T09:23:02.301988Z","iopub.status.idle":"2022-04-15T09:23:02.308987Z","shell.execute_reply.started":"2022-04-15T09:23:02.301952Z","shell.execute_reply":"2022-04-15T09:23:02.308266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Note: parsing the data and datetimes is sloooow. ","metadata":{}},{"cell_type":"code","source":"%%time\ntest = pd.read_csv(\"../input/goodreads-books-reviews-290312/goodreads_test.csv\",parse_dates=['date_added',\n       'date_updated', 'read_at', 'started_at'],infer_datetime_format=True)\nprint(test.columns)\ntest","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:25:16.441097Z","iopub.execute_input":"2022-04-15T08:25:16.441415Z","iopub.status.idle":"2022-04-15T08:31:33.768801Z","shell.execute_reply.started":"2022-04-15T08:25:16.441376Z","shell.execute_reply":"2022-04-15T08:31:33.768215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = pd.read_csv(\"../input/goodreads-books-reviews-290312/goodreads_train.csv\",parse_dates=['date_added',\n       'date_updated', 'read_at', 'started_at'],infer_datetime_format=True)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:31:33.770032Z","iopub.execute_input":"2022-04-15T08:31:33.770495Z","iopub.status.idle":"2022-04-15T08:50:37.763349Z","shell.execute_reply.started":"2022-04-15T08:31:33.770463Z","shell.execute_reply":"2022-04-15T08:50:37.76241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Shrink data\n* We don't need review IDs in train data\n* We can encode the user IDs as integers to save space\n    * we could reencode the book id as int16, and the test review IDs + a lookup (but that adds overhead and saves less) ","metadata":{}},{"cell_type":"code","source":"train.drop(columns=[\"review_id\"],inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:50:37.764872Z","iopub.execute_input":"2022-04-15T08:50:37.765139Z","iopub.status.idle":"2022-04-15T08:50:37.92579Z","shell.execute_reply.started":"2022-04-15T08:50:37.765109Z","shell.execute_reply":"2022-04-15T08:50:37.924877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"users = list(set(test[\"user_id\"] + train[\"user_id\"]))\nprint(len(users),\"# unique users\")","metadata":{"execution":{"iopub.status.busy":"2022-04-15T08:50:37.927124Z","iopub.execute_input":"2022-04-15T08:50:37.92733Z","iopub.status.idle":"2022-04-15T08:50:38.478605Z","shell.execute_reply.started":"2022-04-15T08:50:37.927305Z","shell.execute_reply":"2022-04-15T08:50:38.47783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.concat([test[\"user_id\"] , train[\"user_id\"]])","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:24:37.992468Z","iopub.execute_input":"2022-04-15T09:24:37.993315Z","iopub.status.idle":"2022-04-15T09:24:38.01097Z","shell.execute_reply.started":"2022-04-15T09:24:37.993268Z","shell.execute_reply":"2022-04-15T09:24:38.01004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# enc = OrdinalEncoder()\nenc = LabelEncoder()\nenc.fit(pd.concat([test[\"user_id\"] , train[\"user_id\"]]).values.reshape(-1, 1))","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:25:42.794663Z","iopub.execute_input":"2022-04-15T09:25:42.795503Z","iopub.status.idle":"2022-04-15T09:25:42.905192Z","shell.execute_reply.started":"2022-04-15T09:25:42.795458Z","shell.execute_reply":"2022-04-15T09:25:42.904136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train[\"user_id\"].nunique())\nprint(train[\"user_id\"].tail())","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:26:12.517287Z","iopub.execute_input":"2022-04-15T09:26:12.518144Z","iopub.status.idle":"2022-04-15T09:26:12.62414Z","shell.execute_reply.started":"2022-04-15T09:26:12.518097Z","shell.execute_reply":"2022-04-15T09:26:12.623418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"user_id\"] = enc.transform(train[\"user_id\"])\ntest[\"user_id\"] = enc.transform(test[\"user_id\"])\n","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:26:43.700909Z","iopub.execute_input":"2022-04-15T09:26:43.701249Z","iopub.status.idle":"2022-04-15T09:26:44.321823Z","shell.execute_reply.started":"2022-04-15T09:26:43.701214Z","shell.execute_reply":"2022-04-15T09:26:44.320957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train[\"user_id\"].nunique())\nprint(train[\"user_id\"].tail())","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:27:00.493267Z","iopub.execute_input":"2022-04-15T09:27:00.493684Z","iopub.status.idle":"2022-04-15T09:27:00.824649Z","shell.execute_reply.started":"2022-04-15T09:27:00.493653Z","shell.execute_reply":"2022-04-15T09:27:00.823786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check train/test overlaps\n* Could also be temporal/time split? ","metadata":{}},{"cell_type":"code","source":"print(len(train[\"user_id\"]))\nprint(len(test[\"user_id\"]))","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:04:18.904223Z","iopub.execute_input":"2022-04-15T09:04:18.905233Z","iopub.status.idle":"2022-04-15T09:04:18.911629Z","shell.execute_reply.started":"2022-04-15T09:04:18.905081Z","shell.execute_reply":"2022-04-15T09:04:18.910725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train[\"user_id\"].isin(test[\"user_id\"])), \"users overlap\")\nprint(len(~train[\"user_id\"].isin(test[\"user_id\"])), \"users overlap\")\nprint(len(test[\"user_id\"].isin(train[\"user_id\"])), \"users overlap\")","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:04:31.511022Z","iopub.execute_input":"2022-04-15T09:04:31.511391Z","iopub.status.idle":"2022-04-15T09:04:31.628426Z","shell.execute_reply.started":"2022-04-15T09:04:31.511357Z","shell.execute_reply":"2022-04-15T09:04:31.627467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train[\"book_id\"].isin(test[\"book_id\"])), \"book overlap\")","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:03:03.595781Z","iopub.execute_input":"2022-04-15T09:03:03.596107Z","iopub.status.idle":"2022-04-15T09:03:03.620574Z","shell.execute_reply.started":"2022-04-15T09:03:03.596065Z","shell.execute_reply":"2022-04-15T09:03:03.619514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### counts EDA\n\n* there are varied cases of something appearing only once\n* If doing global avg, would need smoothing","metadata":{}},{"cell_type":"code","source":"train[\"user_id\"].value_counts().hist();","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:30:28.767578Z","iopub.execute_input":"2022-04-15T09:30:28.76784Z","iopub.status.idle":"2022-04-15T09:30:28.993788Z","shell.execute_reply.started":"2022-04-15T09:30:28.767811Z","shell.execute_reply":"2022-04-15T09:30:28.992982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"book_id\"].value_counts().hist();","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:30:31.781288Z","iopub.execute_input":"2022-04-15T09:30:31.781581Z","iopub.status.idle":"2022-04-15T09:30:32.021085Z","shell.execute_reply.started":"2022-04-15T09:30:31.781546Z","shell.execute_reply":"2022-04-15T09:30:32.020445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### export compressed rep\n\n* can add avg rating per book, user","metadata":{}},{"cell_type":"code","source":"# train[\"tr\"] = True\n# test[\"tr\"] = False\n\n# df = pd.concat([train[['user_id', 'book_id', 'rating']],test])[['user_id', 'book_id', 'rating']]\n# df[\"book_count\"] = df.groupby(\"book_id\")['user_id'].transform(\"count\")\n# df[\"user_count\"] = df.groupby(\"user_id\")['book_id'].transform(\"count\")\n# df","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:33:30.535565Z","iopub.execute_input":"2022-04-15T09:33:30.5359Z","iopub.status.idle":"2022-04-15T09:33:31.389923Z","shell.execute_reply.started":"2022-04-15T09:33:30.535864Z","shell.execute_reply":"2022-04-15T09:33:31.388995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_user = df.loc[df[\"user_count\"]>10]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### export","metadata":{}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:35:37.740625Z","iopub.execute_input":"2022-04-15T09:35:37.741157Z","iopub.status.idle":"2022-04-15T09:35:37.7615Z","shell.execute_reply.started":"2022-04-15T09:35:37.741109Z","shell.execute_reply":"2022-04-15T09:35:37.760681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:35:45.81026Z","iopub.execute_input":"2022-04-15T09:35:45.810567Z","iopub.status.idle":"2022-04-15T09:35:45.831988Z","shell.execute_reply.started":"2022-04-15T09:35:45.81053Z","shell.execute_reply":"2022-04-15T09:35:45.831202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.to_parquet(\"goodreads_test.parquet\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_parquet(\"goodreads_train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-04-15T09:28:20.019381Z","iopub.execute_input":"2022-04-15T09:28:20.019708Z","iopub.status.idle":"2022-04-15T09:28:20.027457Z","shell.execute_reply.started":"2022-04-15T09:28:20.019671Z","shell.execute_reply":"2022-04-15T09:28:20.026399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}