{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-24T22:11:06.82736Z","iopub.execute_input":"2022-05-24T22:11:06.829076Z","iopub.status.idle":"2022-05-24T22:11:06.842644Z","shell.execute_reply.started":"2022-05-24T22:11:06.829003Z","shell.execute_reply":"2022-05-24T22:11:06.840858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv_filename = \"/kaggle/input/goodreads-books-reviews-290312/goodreads_test.csv\"\ntrain_csv_filename = \"/kaggle/input/goodreads-books-reviews-290312/goodreads_train.csv\"\nsample_submission_csv_filename = \"/kaggle/input/goodreads-books-reviews-290312/goodreads_sample_submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-24T22:11:18.488906Z","iopub.execute_input":"2022-05-24T22:11:18.489428Z","iopub.status.idle":"2022-05-24T22:11:18.493756Z","shell.execute_reply.started":"2022-05-24T22:11:18.489387Z","shell.execute_reply":"2022-05-24T22:11:18.492814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysis of CSV","metadata":{}},{"cell_type":"code","source":"# show first lines of test data\n!head $test_csv_filename","metadata":{"execution":{"iopub.status.busy":"2022-05-24T15:57:27.639389Z","iopub.execute_input":"2022-05-24T15:57:27.639936Z","iopub.status.idle":"2022-05-24T15:57:28.45132Z","shell.execute_reply.started":"2022-05-24T15:57:27.6399Z","shell.execute_reply":"2022-05-24T15:57:28.450536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show first lines of train data\n!head $train_csv_filename","metadata":{"execution":{"iopub.status.busy":"2022-05-24T15:57:28.453128Z","iopub.execute_input":"2022-05-24T15:57:28.453401Z","iopub.status.idle":"2022-05-24T15:57:29.248053Z","shell.execute_reply.started":"2022-05-24T15:57:28.453366Z","shell.execute_reply":"2022-05-24T15:57:29.246861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show first lines of train data\n!head $train_csv_filename","metadata":{"execution":{"iopub.status.busy":"2022-05-24T15:57:29.249532Z","iopub.execute_input":"2022-05-24T15:57:29.249842Z","iopub.status.idle":"2022-05-24T15:57:30.039665Z","shell.execute_reply.started":"2022-05-24T15:57:29.249804Z","shell.execute_reply":"2022-05-24T15:57:30.038249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show first lines of sample submission\n!head $sample_submission_csv_filename","metadata":{"execution":{"iopub.status.busy":"2022-05-24T15:57:30.042965Z","iopub.execute_input":"2022-05-24T15:57:30.043274Z","iopub.status.idle":"2022-05-24T15:57:30.858774Z","shell.execute_reply.started":"2022-05-24T15:57:30.043226Z","shell.execute_reply":"2022-05-24T15:57:30.857923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load CSV and write Parquet\n\n**Warning** loading time is looooong because of date parsing:\n- test is 6min\n- train is 20min","metadata":{}},{"cell_type":"code","source":"# Output filenames\ntest_parquet_filename = \"goodreads_test.parquet\"\ntrain_parquet_filename = \"goodreads_train.parquet\"\n\n# Parsing parameters\ndate_columns = [\"date_added\", \"date_updated\", \"read_at\", \"started_at\"]\ncustom_dtypes = {\n    \"rating\": np.uint8,\n    \"book_id\": np.uint32,\n    \"n_votes\": np.uint16,\n    \"n_comments\": np.uint16\n}\n\ndef parse_goodreads_data(csv_filename):\n    return pd.read_csv(\n        csv_filename,\n        parse_dates=date_columns,\n        infer_datetime_format=True,\n        dtype=custom_dtypes\n    )","metadata":{"execution":{"iopub.status.busy":"2022-05-24T22:11:30.584749Z","iopub.execute_input":"2022-05-24T22:11:30.585533Z","iopub.status.idle":"2022-05-24T22:11:30.592673Z","shell.execute_reply.started":"2022-05-24T22:11:30.585485Z","shell.execute_reply":"2022-05-24T22:11:30.591302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test data","metadata":{}},{"cell_type":"code","source":"%%time\ndf_test = parse_goodreads_data(test_csv_filename)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T15:57:30.868787Z","iopub.execute_input":"2022-05-24T15:57:30.869019Z","iopub.status.idle":"2022-05-24T16:02:28.868917Z","shell.execute_reply.started":"2022-05-24T15:57:30.868992Z","shell.execute_reply":"2022-05-24T16:02:28.867859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.to_parquet(test_parquet_filename)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T16:02:28.870442Z","iopub.execute_input":"2022-05-24T16:02:28.870923Z","iopub.status.idle":"2022-05-24T16:02:34.317Z","shell.execute_reply.started":"2022-05-24T16:02:28.87088Z","shell.execute_reply":"2022-05-24T16:02:34.316218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_test","metadata":{"execution":{"iopub.status.busy":"2022-05-24T16:02:34.31826Z","iopub.execute_input":"2022-05-24T16:02:34.318496Z","iopub.status.idle":"2022-05-24T16:02:34.406551Z","shell.execute_reply.started":"2022-05-24T16:02:34.318468Z","shell.execute_reply":"2022-05-24T16:02:34.405679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train data","metadata":{}},{"cell_type":"code","source":"%%time\ndf_train = parse_goodreads_data(train_csv_filename)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T16:02:34.407858Z","iopub.execute_input":"2022-05-24T16:02:34.408231Z","iopub.status.idle":"2022-05-24T16:18:12.606364Z","shell.execute_reply.started":"2022-05-24T16:02:34.408202Z","shell.execute_reply":"2022-05-24T16:18:12.604386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.to_parquet(train_parquet_filename)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T16:18:12.612043Z","iopub.execute_input":"2022-05-24T16:18:12.612561Z","iopub.status.idle":"2022-05-24T16:18:24.366913Z","shell.execute_reply.started":"2022-05-24T16:18:12.612479Z","shell.execute_reply":"2022-05-24T16:18:24.365952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_train","metadata":{"execution":{"iopub.status.busy":"2022-05-24T16:18:24.368429Z","iopub.execute_input":"2022-05-24T16:18:24.36915Z","iopub.status.idle":"2022-05-24T16:18:24.633556Z","shell.execute_reply.started":"2022-05-24T16:18:24.36911Z","shell.execute_reply":"2022-05-24T16:18:24.632467Z"},"trusted":true},"execution_count":null,"outputs":[]}]}