{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Dataset . Detect Sleep States\n\n\n### Versions\n- `2023-09-07` First version: 10 time-series as s `csv` file.\n- `2023-09-10` Added `event` property to time-series using the `events` file. Renamed folder from `train-series` to `train_series`.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format='retina'\n\nimport datetime\nimport os\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.lines import Line2D\nimport seaborn as sns\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom kaggle_secrets import UserSecretsClient","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:13:22.058939Z","iopub.execute_input":"2023-09-10T06:13:22.060147Z","iopub.status.idle":"2023-09-10T06:13:23.290774Z","shell.execute_reply.started":"2023-09-10T06:13:22.060104Z","shell.execute_reply":"2023-09-10T06:13:23.289663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create the directory if it does not exist already to write the Kaggle token to.\n!mkdir -p ~/.kaggle/\n\n# Write the Kaggle token to the file.\nuser_secrets = UserSecretsClient()\nwith open('/root/.kaggle/kaggle.json', 'w') as f:\n    f.write(user_secrets.get_secret(\"KAGGLE\"))\n\n# Make the file read-only.\n!chmod 600 ~/.kaggle/kaggle.json","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:13:23.293040Z","iopub.execute_input":"2023-09-10T06:13:23.294226Z","iopub.status.idle":"2023-09-10T06:13:25.705203Z","shell.execute_reply.started":"2023-09-10T06:13:23.294189Z","shell.execute_reply":"2023-09-10T06:13:25.703608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets","metadata":{}},{"cell_type":"code","source":"# Read the timeseries dataset.\ntrain_series_df = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n\nprint(train_series_df.shape)\ntrain_series_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:13:25.707121Z","iopub.execute_input":"2023-09-10T06:13:25.707478Z","iopub.status.idle":"2023-09-10T06:14:21.752366Z","shell.execute_reply.started":"2023-09-10T06:13:25.707449Z","shell.execute_reply":"2023-09-10T06:14:21.751386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the timeseries dataset.\ntrain_events_df = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\ntrain_events_df.sort_values(by='timestamp', inplace=True)\n\nprint(train_events_df.shape)\ntrain_events_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:14:21.753357Z","iopub.execute_input":"2023-09-10T06:14:21.754591Z","iopub.status.idle":"2023-09-10T06:14:21.808890Z","shell.execute_reply.started":"2023-09-10T06:14:21.754474Z","shell.execute_reply":"2023-09-10T06:14:21.807903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf dataset","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:14:21.811422Z","iopub.execute_input":"2023-09-10T06:14:21.811951Z","iopub.status.idle":"2023-09-10T06:14:23.466658Z","shell.execute_reply.started":"2023-09-10T06:14:21.811921Z","shell.execute_reply":"2023-09-10T06:14:23.464836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2023-09-07","metadata":{}},{"cell_type":"code","source":"# Using date for versioning.\nDATASET_VERSION = datetime.date(2023, 9, 7)\n\n# Create a directory for the clean dataset.\noutput_directory = Path(f'dataset/detect-sleep-states/{DATASET_VERSION}/train_series/')\n\n# Remove the directory if it already exists and re-create it.\n!rm -rf {output_directory}\noutput_directory.mkdir(parents=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:14:23.468175Z","iopub.execute_input":"2023-09-10T06:14:23.468588Z","iopub.status.idle":"2023-09-10T06:14:24.746402Z","shell.execute_reply.started":"2023-09-10T06:14:23.468556Z","shell.execute_reply":"2023-09-10T06:14:24.744668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = list(train_series_df['series_id'].unique())\nfor series_id in tqdm(series_ids[:10]):\n    subset_series_df = train_series_df[train_series_df['series_id'] == series_id].copy().reset_index(drop=True)\n    subset_series_df.to_csv(output_directory / f'{series_id}.csv', index=False)\n\n# The number of lines will be one more than the number of files.\n!ls -lh {output_directory} | wc -l","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:14:24.748750Z","iopub.execute_input":"2023-09-10T06:14:24.749854Z","iopub.status.idle":"2023-09-10T06:16:42.965116Z","shell.execute_reply.started":"2023-09-10T06:14:24.749785Z","shell.execute_reply":"2023-09-10T06:16:42.963093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copy the other datasets as is given they are not as big.\n!cp \"/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv\" \"/kaggle/working/dataset/detect-sleep-states/{DATASET_VERSION}/\"\n!cp \"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\" \"/kaggle/working/dataset/detect-sleep-states/{DATASET_VERSION}/\"\n\n# Convert the test series from parquet to csv.\npd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\").to_csv(f\"/kaggle/working/dataset/detect-sleep-states/{DATASET_VERSION}/test_series.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:16:42.967615Z","iopub.execute_input":"2023-09-10T06:16:42.968046Z","iopub.status.idle":"2023-09-10T06:16:45.355007Z","shell.execute_reply.started":"2023-09-10T06:16:42.968014Z","shell.execute_reply":"2023-09-10T06:16:45.353582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2023-09-10","metadata":{}},{"cell_type":"code","source":"# Using date for versioning.\nDATASET_VERSION = datetime.date(2023, 9, 10)\n\n# Create a directory for the clean dataset.\noutput_directory = Path(f'dataset/detect-sleep-states/{DATASET_VERSION}/train_series/')\n\n# Remove the directory if it already exists and re-create it.\n!rm -rf {output_directory}\noutput_directory.mkdir(parents=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:16:45.357781Z","iopub.execute_input":"2023-09-10T06:16:45.358499Z","iopub.status.idle":"2023-09-10T06:16:46.543756Z","shell.execute_reply.started":"2023-09-10T06:16:45.358447Z","shell.execute_reply":"2023-09-10T06:16:46.541781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = list(train_series_df['series_id'].unique())\nfor series_id in tqdm(series_ids[:10]):\n    subset_series_df = train_series_df[train_series_df['series_id'] == series_id].copy().reset_index(drop=True)\n\n    # Merge events of the series to get the event property.\n    merged_df = pd.merge(subset_series_df, train_events_df[['series_id', 'step', 'event']], on=['series_id', 'step'], how='left')\n    merged_df['event'] = merged_df['event'].ffill()\n\n    merged_df.to_csv(output_directory / f'{series_id}.csv', index=False)\n\n# The number of lines will be one more than the number of files.\n!ls -lh {output_directory} | wc -l","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:16:46.546438Z","iopub.execute_input":"2023-09-10T06:16:46.547282Z","iopub.status.idle":"2023-09-10T06:19:04.353488Z","shell.execute_reply.started":"2023-09-10T06:16:46.547240Z","shell.execute_reply":"2023-09-10T06:19:04.351971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(merged_df.shape)\nmerged_df.tail(3)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:19:04.355069Z","iopub.execute_input":"2023-09-10T06:19:04.355461Z","iopub.status.idle":"2023-09-10T06:19:04.372126Z","shell.execute_reply.started":"2023-09-10T06:19:04.355426Z","shell.execute_reply":"2023-09-10T06:19:04.370898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Copy the other datasets as is given they are not as big.\n!cp \"/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv\" \"/kaggle/working/dataset/detect-sleep-states/{DATASET_VERSION}/\"\n!cp \"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\" \"/kaggle/working/dataset/detect-sleep-states/{DATASET_VERSION}/\"\n\n# Convert the test series from parquet to csv.\npd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\").to_csv(f\"/kaggle/working/dataset/detect-sleep-states/{DATASET_VERSION}/test_series.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:19:04.374302Z","iopub.execute_input":"2023-09-10T06:19:04.374761Z","iopub.status.idle":"2023-09-10T06:19:06.714750Z","shell.execute_reply.started":"2023-09-10T06:19:04.374717Z","shell.execute_reply":"2023-09-10T06:19:06.713252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Upload to Kaggle","metadata":{}},{"cell_type":"code","source":"# NOTE: Double check the dataset directory given everything in the directory will be uploaded to Kaggle.\n!tree -h {output_directory.parent.parent}","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:19:06.717366Z","iopub.execute_input":"2023-09-10T06:19:06.718197Z","iopub.status.idle":"2023-09-10T06:19:07.867617Z","shell.execute_reply.started":"2023-09-10T06:19:06.718151Z","shell.execute_reply":"2023-09-10T06:19:07.865934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!echo '{\"title\": \"Detect sleep states\", \"id\": \"bkowshik/detect-sleep-states\", \"licenses\": [{\"name\": \"copyright-authors\"}]}' > dataset/dataset-metadata.json","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:19:07.873446Z","iopub.execute_input":"2023-09-10T06:19:07.873864Z","iopub.status.idle":"2023-09-10T06:19:09.014165Z","shell.execute_reply.started":"2023-09-10T06:19:07.873832Z","shell.execute_reply":"2023-09-10T06:19:09.012013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NOTE: Uncomment one of the lines below if data needs to uploaded to Kaggle.\n\n# Creating the dataset for the very first time.\n# !kaggle datasets create -p dataset -r skip --dir-mode tar\n\n# Creating subsequent versions of the datasets.\n# !kaggle datasets version -p dataset -r skip --dir-mode tar -m 'Updated dataset'","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:43:29.494447Z","iopub.execute_input":"2023-09-10T06:43:29.496048Z","iopub.status.idle":"2023-09-10T06:43:57.009184Z","shell.execute_reply.started":"2023-09-10T06:43:29.495995Z","shell.execute_reply":"2023-09-10T06:43:57.007529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}