{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Basic dataset\n### Full","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndf = pd.read_parquet(\"../input/goodreads-books-reviews-parquet/goodreads_train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T23:59:56.368742Z","iopub.execute_input":"2022-05-25T23:59:56.369530Z","iopub.status.idle":"2022-05-26T00:00:07.544498Z","shell.execute_reply.started":"2022-05-25T23:59:56.369409Z","shell.execute_reply":"2022-05-26T00:00:07.543576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ndf_train, df_valid_test = train_test_split(df, test_size=0.10, stratify=df[\"rating\"])\ndf_valid, df_test = train_test_split(df_valid_test, test_size=0.50, stratify=df_valid_test[\"rating\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-26T00:00:14.603642Z","iopub.execute_input":"2022-05-26T00:00:14.603991Z","iopub.status.idle":"2022-05-26T00:00:16.245009Z","shell.execute_reply.started":"2022-05-26T00:00:14.603949Z","shell.execute_reply":"2022-05-26T00:00:16.243981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import Dataset, DatasetDict\n\nds = DatasetDict({\n    \"train\": Dataset.from_pandas(df_train, preserve_index=False, split=\"train\"),\n    \"valid\": Dataset.from_pandas(df_valid, preserve_index=False, split=\"valid\"),\n    \"test\": Dataset.from_pandas(df_test, preserve_index=False, split=\"test\")\n})","metadata":{"execution":{"iopub.status.busy":"2022-05-26T00:00:21.088461Z","iopub.execute_input":"2022-05-26T00:00:21.088778Z","iopub.status.idle":"2022-05-26T00:00:32.481875Z","shell.execute_reply.started":"2022-05-26T00:00:21.088742Z","shell.execute_reply":"2022-05-26T00:00:32.480724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds.save_to_disk(\"goodreads\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T21:56:21.978282Z","iopub.execute_input":"2022-05-25T21:56:21.978966Z","iopub.status.idle":"2022-05-25T21:56:23.348344Z","shell.execute_reply.started":"2022-05-25T21:56:21.978939Z","shell.execute_reply":"2022-05-25T21:56:23.347261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Small","metadata":{}},{"cell_type":"code","source":"small_size = 1000\nseed = 123\n\nsmall_ds = DatasetDict({\n    \"train\": ds[\"train\"].shuffle(seed=seed).select(range(small_size)),\n    \"valid\": ds[\"valid\"].shuffle(seed=seed).select(range(small_size)),\n    \"test\": ds[\"test\"].shuffle(seed=seed).select(range(small_size))\n})\n\nsmall_ds.save_to_disk(\"goodreads-small\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T21:56:23.350454Z","iopub.execute_input":"2022-05-25T21:56:23.350706Z","iopub.status.idle":"2022-05-25T21:56:24.100322Z","shell.execute_reply.started":"2022-05-25T21:56:23.350679Z","shell.execute_reply":"2022-05-25T21:56:24.099832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bert tokenized dataset\n### Full","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n\n\ndef tokenize_function(examples):\n    return tokenizer(examples[\"review_text\"], padding=\"max_length\", truncation=True)\n\n\ntokenized_ds = ds.map(tokenize_function, batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-26T00:01:31.412140Z","iopub.execute_input":"2022-05-26T00:01:31.412489Z","iopub.status.idle":"2022-05-26T00:10:47.540740Z","shell.execute_reply.started":"2022-05-26T00:01:31.412457Z","shell.execute_reply":"2022-05-26T00:10:47.539744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenized_ds.save_to_disk(\"goodreads-bert\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T22:05:06.389788Z","iopub.execute_input":"2022-05-25T22:05:06.390216Z","iopub.status.idle":"2022-05-25T22:05:15.587064Z","shell.execute_reply.started":"2022-05-25T22:05:06.390171Z","shell.execute_reply":"2022-05-25T22:05:15.586193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Small","metadata":{}},{"cell_type":"code","source":"small_tokenized_ds = DatasetDict({\n    \"train\": tokenized_ds[\"train\"].shuffle(seed=seed).select(range(small_size)),\n    \"valid\": tokenized_ds[\"valid\"].shuffle(seed=seed).select(range(small_size)),\n    \"test\": tokenized_ds[\"test\"].shuffle(seed=seed).select(range(small_size))\n})\n\nsmall_tokenized_ds.save_to_disk(\"goodreads-bert-small\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T22:05:15.588188Z","iopub.execute_input":"2022-05-25T22:05:15.588788Z","iopub.status.idle":"2022-05-25T22:05:19.538636Z","shell.execute_reply.started":"2022-05-25T22:05:15.588759Z","shell.execute_reply":"2022-05-25T22:05:19.537913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Medium","metadata":{}},{"cell_type":"code","source":"medium_size = 20000\n\nmedium_tokenized_ds = DatasetDict({\n    \"train\": tokenized_ds[\"train\"].shuffle(seed=seed).select(range(medium_size)),\n    \"valid\": tokenized_ds[\"valid\"].shuffle(seed=seed).select(range(medium_size)),\n    \"test\": tokenized_ds[\"test\"].shuffle(seed=seed).select(range(medium_size))\n})\n\nmedium_tokenized_ds.save_to_disk(\"goodreads-bert-medium\")","metadata":{"execution":{"iopub.status.busy":"2022-05-26T00:11:35.098828Z","iopub.execute_input":"2022-05-26T00:11:35.099908Z","iopub.status.idle":"2022-05-26T00:12:34.150072Z","shell.execute_reply.started":"2022-05-26T00:11:35.099866Z","shell.execute_reply":"2022-05-26T00:12:34.149310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}