{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\n# Import the basics\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Data Location\nINPUT_PATH = r\"/kaggle/input/child-mind-institute-detect-sleep-states\"\nSAMPLE_FILE_PATH = INPUT_PATH + \"/sample_submission.csv\"\nTEST_SERIES_FILE_PATH = INPUT_PATH + \"/test.parquet\"\nTRAIN_EVENTS_FILE_PATH = INPUT_PATH + \"/train_events.csv\"\nTRAIN_SERIES_FILE_PATH = INPUT_PATH + \"/train_series.parquet\"\n\n#  Custom Date parser\ndateparse = lambda x: pd.to_datetime(x, format=\"ISO8601\",utc=True, yearfirst=True, errors=\"coerce\",cache=True)\n\n# Read in the train_events.csv and parse dates \ntrain_events = pd.read_csv(\n    TRAIN_EVENTS_FILE_PATH,\\\n    parse_dates=['timestamp'],\\\n    \n    # Send all dates to be re-parsed as \"ISO8601\". \n    date_parser=dateparse).assign(\n        \n        # Zulu Time, also Known as GMT(Greenwich Mean Time) is the time recorder at the Prime Meridian per observastion. \n        # UTC, GMT, Zulu Time, Z-Time, etc. does not observe daylight savings time. \n        zulu_time = lambda x: x.timestamp.dt.strftime('%Y/%m/%d %H:%M:%S'),\\\n    \n        # The Local time is obsered from New York state. Thankfully their is only one time zone in New York.\n        local_time = lambda x: x.timestamp.dt.tz_convert('us/eastern'),\\\n#     us/eastern\n#     America/New_York\n        # Timestamp was lost and replaced with Zulu time.\n        ).drop(columns=['timestamp'], axis=1)\n\n\n#  Record the changes in the Daylight Savings\n# \"EDT\" : Event was observed during Eastern Daylight Time (UTC-4)\n# \"EST\" : Event was observed during Eastern Standard Time (UTC-5)\ntrain_events['tz_abbrvtn'] = [\"EDT\" if tza=='-04:00'else(\"EST\" if tza==\"-05:00\" else tza)\\\n                      for tza in train_events.local_time.apply(lambda x: str(x)[-6:]).tolist()]\\\n\n# Tidy up format and remove UTC details.\ntrain_events['local_time'] = train_events['local_time'].dt.strftime('%Y/%m/%d %H:%M:%S')\n\n# Observe the results \nn_rows, n_cols = train_events.shape\nprint(f\"\\nRows: {n_rows}\\nColumns: {n_cols}\\n\")\ntrain_events.sample(13)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-12T19:38:08.042292Z","iopub.execute_input":"2023-09-12T19:38:08.042657Z","iopub.status.idle":"2023-09-12T19:38:10.395896Z","shell.execute_reply.started":"2023-09-12T19:38:08.042624Z","shell.execute_reply":"2023-09-12T19:38:10.394905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## __How many series_id can be obseved with a shift in the UTC from Eastern Daylight Time to Eastern Standard time or vice versa?__","metadata":{}},{"cell_type":"code","source":"# A list of all unique Ids\nSERIES_IDS = train_events.series_id.unique().tolist()\n\nSERIES_IDS_WITH_TIME_CHANGES = []\n\n# Loop all unique series and check for changes in the Daylight Savings\nfor series_id in SERIES_IDS:\n    series_id_tzs = train_events.query(\"series_id==@series_id\").tz_abbrvtn.values\n    if \"EDT\" in series_id_tzs and \"EST\" in series_id_tzs:\n        SERIES_IDS_WITH_TIME_CHANGES.append(series_id)\n\n# Rep\nresults = f\"\"\"\nThere are {len(SERIES_IDS_WITH_TIME_CHANGES)} series_id observed with a shift from Eastern Daylight Time (UTC-4) to Eastern Standard time (UTC-5) or vice versa.\n\"\"\"        \nprint(results)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-12T19:37:18.591701Z","iopub.status.idle":"2023-09-12T19:37:18.592511Z","shell.execute_reply.started":"2023-09-12T19:37:18.592303Z","shell.execute_reply":"2023-09-12T19:37:18.592325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## __Observe the first series_id with an observerd shift in Daylight savings. Then verify if the shift date is accurate or not__","metadata":{}},{"cell_type":"code","source":"temp = train_events.query(\"series_id == @SERIES_IDS_WITH_TIME_CHANGES[0]\")\ntemp.sort_values(by=['night','zulu_time'],inplace=True)\nprint(f\"\\nSeries ID: {SERIES_IDS_WITH_TIME_CHANGES[0]}\\n\")\ntemp","metadata":{"execution":{"iopub.status.busy":"2023-09-12T12:54:43.905177Z","iopub.execute_input":"2023-09-12T12:54:43.905541Z","iopub.status.idle":"2023-09-12T12:54:43.946768Z","shell.execute_reply.started":"2023-09-12T12:54:43.905513Z","shell.execute_reply":"2023-09-12T12:54:43.945800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### The first Series ID (10469f6765bf) with a shift in the UTC, starts with it's first observation on 2018/11/03 at 00:52:00 EDT. At this point, Zulu time should be 4 hours ahead of the local time. On 2018/11/04 at 21:56:00 EST, the series reports a shift from EDT (UTC-4) to EST(UTC-5).  We can see the Zulu time had updatesd as expspected and is now 5 hours ahead of the local time (NY). The Shift date to verify is 2018/11/04 21:56:00.\n\n#### If you visit [timeanddate](https://www.timeanddate.com/time/zone/usa/new-york) website, you'll find historical dates to the UTC shift dates for New York state. When you visit, change the __Time zone changes for:__ in drop down menu to say \"2010 - 2019\". Now, if you observe the 2018 year in the \"Time Changes in New York Over the Years\" table, the shift in the UTC can be at November 4th, 2017 at 2 AM.  \n\n<br> \n## ","metadata":{}},{"cell_type":"code","source":"# ERay issue Date issue.\n\ntrain_events.query(\"series_id == '8a306e0890c0' and night.isin([3,4,5,10,11,12,])\")\n\n# Daylight Savings occured localy on Nov 5, 2017, pass\n# Zulu time is 5 hours ahead on Nov 6, 2017, pass\n# Zulu time is 4 hours ahead on Nov 4, 2017, pass","metadata":{"execution":{"iopub.status.busy":"2023-09-12T12:51:16.737599Z","iopub.execute_input":"2023-09-12T12:51:16.737966Z","iopub.status.idle":"2023-09-12T12:51:16.761239Z","shell.execute_reply.started":"2023-09-12T12:51:16.737937Z","shell.execute_reply":"2023-09-12T12:51:16.759882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}