{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is checking if daylight saving time is correctly handled in `train_series.parquet`.\n\nI think the switch between EST (-5) and EDT (-4) are done at wrong time (as of 2023-10-02), one week later than it should be.\n\nShouldn't the change occur on 2018-11-04 not on 2018-11-11, for example?\n\nDaylight saving time bug discussion thread:\nhttps://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/440697\n\nThe bug is probably unimportant, only the string representation is wrong and physical time (e.g. in UTC) seems to be correct.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport argparse\n\ndi = '/kaggle/input/child-mind-institute-detect-sleep-states'","metadata":{"execution":{"iopub.status.busy":"2023-10-02T01:11:54.497298Z","iopub.execute_input":"2023-10-02T01:11:54.497994Z","iopub.status.idle":"2023-10-02T01:11:54.905970Z","shell.execute_reply.started":"2023-10-02T01:11:54.497962Z","shell.execute_reply":"2023-10-02T01:11:54.904765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_parquet(di + '/train_series.parquet')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-02T01:11:54.908045Z","iopub.execute_input":"2023-10-02T01:11:54.908614Z","iopub.status.idle":"2023-10-02T01:12:48.493921Z","shell.execute_reply.started":"2023-10-02T01:11:54.908575Z","shell.execute_reply":"2023-10-02T01:12:48.492760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('date:    ', df.iloc[0]['timestamp'][:10])\nprint('time:    ', df.iloc[0]['timestamp'][11:19])\nprint('timezone:', df.iloc[0]['timestamp'][-5:])","metadata":{"execution":{"iopub.status.busy":"2023-10-02T01:12:48.495789Z","iopub.execute_input":"2023-10-02T01:12:48.496454Z","iopub.status.idle":"2023-10-02T01:12:48.503239Z","shell.execute_reply.started":"2023-10-02T01:12:48.496415Z","shell.execute_reply":"2023-10-02T01:12:48.502119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print days with multiple timezones\nfor series_id, df1 in df.groupby('series_id'):   \n    # Split datetime to date, time, timezone\n    df1['date'] = df1['timestamp'].apply(lambda s: s.split('T')[0])  # str\n    df1['local_time'] = df1['timestamp'].apply(lambda s: s[11:19])\n    df1['timezone'] = df1['timestamp'].apply(lambda s: s[-5:])\n\n    n_timezones = len(set(df1['timezone']))\n    if n_timezones != 1:\n        assert n_timezones == 2\n        \n        for date_str, d in df1.groupby('date'):\n            first = d['timestamp'].iloc[0]\n            last = d['timestamp'].iloc[-1]\n            \n            if first[-5:] !=  last[-5:]:\n                print(series_id, date_str, first, last)","metadata":{"execution":{"iopub.status.busy":"2023-10-02T01:14:38.793678Z","iopub.execute_input":"2023-10-02T01:14:38.794053Z","iopub.status.idle":"2023-10-02T01:17:14.117846Z","shell.execute_reply.started":"2023-10-02T01:14:38.794003Z","shell.execute_reply":"2023-10-02T01:17:14.116621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As far I searched, the switch happends on following days in US.\n\n```text\n2017\tMarch 12\tNovember 5\n2018\tMarch 11\tNovember 4\n2019\tMarch 10\tNovember 3\n```\n\nThe above dates are 1 week later.\n\n","metadata":{}},{"cell_type":"markdown","source":"# Check\n\nThe following checks if the timestamps are consistent with pandas 'US/Eastern' timezone; currently they are not consistent.","metadata":{}},{"cell_type":"code","source":"def check_timezone(df, *, verbose=False):\n    for series_id, df1 in df.groupby('series_id'):\n        # Original datetime in str\n        timestamp_str = df1['timestamp'].values\n        # e.g., 2018-08-14T15:30:00-0400\n        # timezone is -0400 or -0500 depending on daylight saving\n\n        # Convert to pd.Timestamp\n        timestamp_utc = pd.to_datetime(df1['timestamp'], utc=True)\n        timestamp_et = timestamp_utc.dt.tz_convert('US/Eastern')\n\n        # str in ET\n        x = timestamp_et.dt.strftime('%Y-%m-%dT%H:%M:%S%z')\n\n        check = (timestamp_str == x).all()\n        if verbose:\n            print(series_id, check)","metadata":{"execution":{"iopub.status.busy":"2023-10-02T01:23:40.541629Z","iopub.execute_input":"2023-10-02T01:23:40.542088Z","iopub.status.idle":"2023-10-02T01:23:40.551564Z","shell.execute_reply.started":"2023-10-02T01:23:40.542053Z","shell.execute_reply":"2023-10-02T01:23:40.550207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check_timezone(df, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-02T01:23:43.614613Z","iopub.execute_input":"2023-10-02T01:23:43.615050Z"},"trusted":true},"execution_count":null,"outputs":[]}]}