{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nimport random\nimport torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\nfrom itertools import groupby\nimport gc\n\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\n\nimport xgboost as xgb\n\ncolor_pal = sns.color_palette()\nplt.style.use('fivethirtyeight')\n\nnp.random.seed(0)\npd.set_option('display.precision', 2)  \npd.set_option('display.float_format', '{:.2f}'.format) ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-20T10:30:57.216166Z","iopub.execute_input":"2023-09-20T10:30:57.216609Z","iopub.status.idle":"2023-09-20T10:30:57.226049Z","shell.execute_reply.started":"2023-09-20T10:30:57.216574Z","shell.execute_reply":"2023-09-20T10:30:57.224893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Data","metadata":{}},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:30:57.235671Z","iopub.execute_input":"2023-09-20T10:30:57.236879Z","iopub.status.idle":"2023-09-20T10:30:57.265076Z","shell.execute_reply.started":"2023-09-20T10:30:57.236834Z","shell.execute_reply":"2023-09-20T10:30:57.263441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_has_nan = train_events.groupby('series_id')['step'].apply(lambda x: x.isnull().any())\nseries_clean = series_has_nan[~series_has_nan].index.tolist()\nseries_clean[:4]","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:30:57.267053Z","iopub.execute_input":"2023-09-20T10:30:57.267541Z","iopub.status.idle":"2023-09-20T10:30:57.325739Z","shell.execute_reply.started":"2023-09-20T10:30:57.267507Z","shell.execute_reply":"2023-09-20T10:30:57.324469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.multiprocessing as mp\nimport polars as pl\nimport functools\n\n@functools.lru_cache(maxsize=None)\ndef get_one_series(series_id):\n    print(f'fetch {series_id} \\n')\n    one_series = pd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet\", filters=[('series_id','=',series_id)])\n    one_series.set_index('timestamp', inplace=True)\n    # Save about 40% memory\n    one_series['series_id'] = one_series['series_id'].astype('category')\n    one_series['awake'] = one_series['awake'].astype('int16')\n    one_series.index = pd.PeriodIndex(one_series.index, freq=\"5s\")\n    \n    return one_series\n\n# Fetching series using multiple Threads\ndef get_multi_series(series_ids, n_workers=1):\n    n_workers = min(n_workers, len(series_ids)) # Sometimes series_ids length are smaller than n_workers\n    if len(series_ids) == 1:\n        return get_one_series(series_ids[0])\n    print(f'start {n_workers} threads')\n    with mp.Pool(n_workers) as ex:\n        train_all = ex.map(get_one_series, series_ids)\n    return pd.concat(train_all)\n\ndef split_series(one_series):\n    total = len(one_series)\n    splitIdx = int(total * 0.8)\n    return one_series[:splitIdx], one_series[splitIdx:]","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:30:57.327584Z","iopub.execute_input":"2023-09-20T10:30:57.328002Z","iopub.status.idle":"2023-09-20T10:30:57.340119Z","shell.execute_reply.started":"2023-09-20T10:30:57.327973Z","shell.execute_reply":"2023-09-20T10:30:57.339282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time one_series = get_one_series(series_clean[0])\nprint(f'memory usage:{one_series.memory_usage().sum() / 1024**2:.2f} MB')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:30:57.341321Z","iopub.execute_input":"2023-09-20T10:30:57.342144Z","iopub.status.idle":"2023-09-20T10:31:07.601494Z","shell.execute_reply.started":"2023-09-20T10:30:57.342111Z","shell.execute_reply":"2023-09-20T10:31:07.600265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_series.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:31:07.604795Z","iopub.execute_input":"2023-09-20T10:31:07.605548Z","iopub.status.idle":"2023-09-20T10:31:07.618751Z","shell.execute_reply.started":"2023-09-20T10:31:07.605506Z","shell.execute_reply":"2023-09-20T10:31:07.617444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Plot","metadata":{}},{"cell_type":"code","source":"train,val = split_series(one_series)\nlen(train), len(val)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:31:07.619933Z","iopub.execute_input":"2023-09-20T10:31:07.620403Z","iopub.status.idle":"2023-09-20T10:31:07.628950Z","shell.execute_reply.started":"2023-09-20T10:31:07.620374Z","shell.execute_reply":"2023-09-20T10:31:07.627824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:31:07.630718Z","iopub.execute_input":"2023-09-20T10:31:07.631779Z","iopub.status.idle":"2023-09-20T10:31:07.648840Z","shell.execute_reply.started":"2023-09-20T10:31:07.631745Z","shell.execute_reply":"2023-09-20T10:31:07.647924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Open-high-low-close(OHLC) resampling\nIn finance, a popular way to aggregate a time series is to compute four values for each bucket: the first(open), last(close), maximnu(high), and minimal(low) values. By using the ohlc aggregate function, you will obtain a DataFrame having columns containing these four aggregates, which are effeciently computed in a single function call:","metadata":{}},{"cell_type":"code","source":"from pandas.tseries.offsets import Day\n\ndef plot_series(series, freq='1min', days=2):\n    \"\"\"\n    freq = '1min' : Sampling frequency, Optional 5s, 1min, 5min, 1H\n    \"\"\"\n    lastTime = series.index[0] + Day(days) # Only display data for a specified number of days\n        \n    fig, (ax0, ax1) = plt.subplots(2,1, figsize=(25, 10), sharex=True)\n\n    series['anglez'][:lastTime].resample(freq).ohlc().plot(ax=ax0)\n    ax0.set_title('anglez data')\n    \n    series['enmo'][:lastTime].resample(freq).ohlc().plot(ax=ax1)\n    ax1.set_title('enmo data')\n    plt.show()\n\nplot_series(train,freq='1min',days=3)\n# plot_series(val,freq='1min',days=1)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:31:07.650225Z","iopub.execute_input":"2023-09-20T10:31:07.650597Z","iopub.status.idle":"2023-09-20T10:31:08.882897Z","shell.execute_reply.started":"2023-09-20T10:31:07.650564Z","shell.execute_reply":"2023-09-20T10:31:08.881787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Multi Series OHLC","metadata":{}},{"cell_type":"code","source":"%time multi_series = get_multi_series(series_clean[:4], n_workers=4)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:31:08.884085Z","iopub.execute_input":"2023-09-20T10:31:08.884608Z","iopub.status.idle":"2023-09-20T10:31:22.306608Z","shell.execute_reply.started":"2023-09-20T10:31:08.884576Z","shell.execute_reply":"2023-09-20T10:31:22.305049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multi_series.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:31:22.308552Z","iopub.execute_input":"2023-09-20T10:31:22.309757Z","iopub.status.idle":"2023-09-20T10:31:22.325902Z","shell.execute_reply.started":"2023-09-20T10:31:22.309717Z","shell.execute_reply":"2023-09-20T10:31:22.323737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To do the same resampling for each value of series, we introduce the pandas.Grouper object:","metadata":{}},{"cell_type":"code","source":"ts2 = multi_series.loc[:, ['series_id','anglez', 'enmo']].copy()\ntime_key = pd.Grouper(freq='1min')\nts2 = ts2.groupby([\"series_id\",time_key]).ohlc()\nts2","metadata":{"execution":{"iopub.status.busy":"2023-09-20T10:31:22.330856Z","iopub.execute_input":"2023-09-20T10:31:22.331364Z","iopub.status.idle":"2023-09-20T10:31:22.940135Z","shell.execute_reply.started":"2023-09-20T10:31:22.331300Z","shell.execute_reply":"2023-09-20T10:31:22.938700Z"},"trusted":true},"execution_count":null,"outputs":[]}]}