{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\n\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-22T01:17:45.592493Z","iopub.execute_input":"2023-10-22T01:17:45.593464Z","iopub.status.idle":"2023-10-22T01:17:45.598288Z","shell.execute_reply.started":"2023-10-22T01:17:45.593428Z","shell.execute_reply":"2023-10-22T01:17:45.597238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Motivation\n\nDiscussions such as [this one](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/448683#2491393) highlight the difficulty of working with datetimes that have timezone information.\n\n## Timezones and Daylights Savings\n\nWorking with datetimes can be confusing and annoying due to things like timezones and other peculiarities such as [daylights savings time](https://en.wikipedia.org/wiki/Daylight_saving_time). \n\nWorking directly with python packages and strings can be difficult, but thankfully polars provides a means to overcome this.\n\n#### Update\n\n* Split by `series_id` and saved into respective parquet files to avoid memory errors as suggested [here](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/447593)","metadata":{}},{"cell_type":"code","source":"# Memory reduction for train_series\n#The following function has been taken from the notebook\n# https://www.kaggle.com/code/renatoreggiani/zzzs-feat-eng-ideas-60-memory-reduction?scriptVersionId=143987308\n\nfrom pandas.api.types import is_datetime64_ns_dtype\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndef reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n\n    df['series_id'] = df['series_id'].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    \n    return df","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-10-22T01:17:46.057651Z","iopub.execute_input":"2023-10-22T01:17:46.058038Z","iopub.status.idle":"2023-10-22T01:17:46.075808Z","shell.execute_reply.started":"2023-10-22T01:17:46.058009Z","shell.execute_reply":"2023-10-22T01:17:46.074688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The easiest way to overcome this issue is to add `time_zone='UTC'` as below:","metadata":{}},{"cell_type":"code","source":"dt_transforms = [\n    pl.col('timestamp').str.to_datetime(time_zone='UTC').alias('timestamp'),\n    (pl.col('timestamp').str.to_datetime().dt.date()).alias('date'), \n    pl.col('timestamp').str.to_datetime().dt.time().alias('time')\n]","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:17:46.672640Z","iopub.execute_input":"2023-10-22T01:17:46.673068Z","iopub.status.idle":"2023-10-22T01:17:46.694358Z","shell.execute_reply.started":"2023-10-22T01:17:46.673034Z","shell.execute_reply":"2023-10-22T01:17:46.693351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrn_series = (\n    pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n    .with_columns((dt_transforms))\n    .collect()\n    .to_pandas()\n)\n\ntrn_events = (\n    pl.scan_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n    .with_columns((dt_transforms))\n    .collect()\n    .to_pandas()\n)\n\n# download test_series data\ntst_series = (\n    pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n    .with_columns((dt_transforms))\n    .collect()\n    .to_pandas()\n)\n\nreduce_mem_usage(trn_series)\nreduce_mem_usage(tst_series)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:17:46.913294Z","iopub.execute_input":"2023-10-22T01:17:46.913718Z","iopub.status.idle":"2023-10-22T01:22:11.026274Z","shell.execute_reply.started":"2023-10-22T01:17:46.913684Z","shell.execute_reply":"2023-10-22T01:22:11.025224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trn_series","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:22:11.027921Z","iopub.execute_input":"2023-10-22T01:22:11.028222Z","iopub.status.idle":"2023-10-22T01:22:11.051917Z","shell.execute_reply.started":"2023-10-22T01:22:11.028195Z","shell.execute_reply":"2023-10-22T01:22:11.050614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = trn_series.query('`series_id`==\"3c336d6ba566\"')","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:22:11.053446Z","iopub.execute_input":"2023-10-22T01:22:11.053847Z","iopub.status.idle":"2023-10-22T01:22:12.739972Z","shell.execute_reply.started":"2023-10-22T01:22:11.053817Z","shell.execute_reply":"2023-10-22T01:22:12.738870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s[42117:42123:]","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:22:12.744086Z","iopub.execute_input":"2023-10-22T01:22:12.744605Z","iopub.status.idle":"2023-10-22T01:22:12.764943Z","shell.execute_reply.started":"2023-10-22T01:22:12.744554Z","shell.execute_reply":"2023-10-22T01:22:12.763687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that converting to UTC now normalizes for daylights savings. Therefore downstream modeling will not run into an artificial 1 hour skip that will affect training and modeling performance.","metadata":{}},{"cell_type":"code","source":"trn_series.to_parquet(\"trn_series.parquet\")\ntst_series.to_parquet(\"tst_series.parquet\")\ntrn_events.to_parquet(\"trn_events.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:22:12.766755Z","iopub.execute_input":"2023-10-22T01:22:12.767120Z","iopub.status.idle":"2023-10-22T01:22:43.970880Z","shell.execute_reply.started":"2023-10-22T01:22:12.767089Z","shell.execute_reply":"2023-10-22T01:22:43.969910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trn_events","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:22:43.972288Z","iopub.execute_input":"2023-10-22T01:22:43.972631Z","iopub.status.idle":"2023-10-22T01:22:43.995270Z","shell.execute_reply.started":"2023-10-22T01:22:43.972600Z","shell.execute_reply":"2023-10-22T01:22:43.993915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Also save into separate chunks per series\nfor series_id in tqdm(trn_events.series_id.unique()):\n    _s_df = trn_series.query(f\"`series_id`=='{series_id}'\")\n    _s_df.to_parquet(f\"series_{series_id}.parquet\")\n    _e_df = trn_events.query(f\"`series_id`=='{series_id}'\")\n    _e_df.to_parquet(f\"events_{series_id}.parquet\")\n    \n    del _s_df\n    del _e_df","metadata":{"execution":{"iopub.status.busy":"2023-10-22T01:25:03.002524Z","iopub.execute_input":"2023-10-22T01:25:03.002957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}