{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-07T07:12:49.982088Z","iopub.execute_input":"2023-09-07T07:12:49.982936Z","iopub.status.idle":"2023-09-07T07:12:50.445820Z","shell.execute_reply.started":"2023-09-07T07:12:49.982892Z","shell.execute_reply":"2023-09-07T07:12:50.444692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1. Import and Explore the Data\n2. Preprocess data to remove noise.\n3. Train machine learning model to identify sleep onset and wake times.\n4. Validate model with ground-truth data.\n5. Score model based on average precision metric.\n6. Submit predictions in specified format.","metadata":{}},{"cell_type":"markdown","source":"### Explore the Data\n","metadata":{}},{"cell_type":"code","source":"## Data too large\n# import pandas as pd\n\n# df_train_series = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n\n# print(df_train_series.columns)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:12:50.448346Z","iopub.execute_input":"2023-09-07T07:12:50.448941Z","iopub.status.idle":"2023-09-07T07:12:50.454352Z","shell.execute_reply.started":"2023-09-07T07:12:50.448900Z","shell.execute_reply":"2023-09-07T07:12:50.453218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Data too large\n\n# df_train_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n# print(df_train_series.columns)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:12:50.456234Z","iopub.execute_input":"2023-09-07T07:12:50.457022Z","iopub.status.idle":"2023-09-07T07:12:50.466554Z","shell.execute_reply.started":"2023-09-07T07:12:50.456980Z","shell.execute_reply":"2023-09-07T07:12:50.465621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data too large to read at once\n## INITIAL EXPLORATION OF DATA\n\n### Notes:\n1. large data size\n2. not many columns\n3. Don't know what any of the columns mean","metadata":{}},{"cell_type":"code","source":"import dask.dataframe as dd\n\ndf_train_series = dd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n\nprint(df_train_series.columns)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:12:50.468891Z","iopub.execute_input":"2023-09-07T07:12:50.469520Z","iopub.status.idle":"2023-09-07T07:12:52.173706Z","shell.execute_reply.started":"2023-09-07T07:12:50.469488Z","shell.execute_reply":"2023-09-07T07:12:52.172494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train_series.head())\nprint(df_train_series.tail())\nprint(df_train_series.describe().compute())","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:12:52.175434Z","iopub.execute_input":"2023-09-07T07:12:52.176388Z","iopub.status.idle":"2023-09-07T07:14:25.521975Z","shell.execute_reply.started":"2023-09-07T07:12:52.176344Z","shell.execute_reply":"2023-09-07T07:14:25.520895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring Time Data","metadata":{}},{"cell_type":"markdown","source":"#### Notes: \n1. Don't know for certain when data starts and ends due to size.\n2. Need to explore Dask\n3. 5 second intervals","metadata":{}},{"cell_type":"code","source":"time_stamp = df_train_series['timestamp']\n\nprint(time_stamp.tail())\nprint(time_stamp.head())","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:14:25.523443Z","iopub.execute_input":"2023-09-07T07:14:25.523957Z","iopub.status.idle":"2023-09-07T07:14:27.107619Z","shell.execute_reply.started":"2023-09-07T07:14:25.523928Z","shell.execute_reply":"2023-09-07T07:14:27.106435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Convert timestamp to datetime object\ntime_stamp = time_stamp.astype('M8[us]')\n\n# Extract year, month, and day\ntime_stamp['year'] = time_stamp.dt.year.compute()\ntime_stamp['month'] = time_stamp.dt.month.compute()\ntime_stamp['day'] = time_stamp.dt.day.compute()\n\n# Group by year, month, and day, then count\ngrouped_time_stamp = time_stamp.groupby(['year', 'month', 'day']).size().compute()\n\n# Plotting\nplt.figure(figsize=(12, 6))\ngrouped_time_stamp.plot(kind='line')\nplt.xlabel('Date')\nplt.ylabel('Count')\nplt.title('Data Points Per Day')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:18:52.237923Z","iopub.execute_input":"2023-09-07T07:18:52.238377Z","iopub.status.idle":"2023-09-07T07:21:31.662118Z","shell.execute_reply.started":"2023-09-07T07:18:52.238339Z","shell.execute_reply":"2023-09-07T07:21:31.660537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring Anglez Data","metadata":{}},{"cell_type":"code","source":"anglez_data = df_train_series['anglez']\n\nprint(anglez_data.tail())","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:14:27.109502Z","iopub.execute_input":"2023-09-07T07:14:27.109924Z","iopub.status.idle":"2023-09-07T07:14:27.136707Z","shell.execute_reply.started":"2023-09-07T07:14:27.109885Z","shell.execute_reply":"2023-09-07T07:14:27.135235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(anglez_data.describe().compute())","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:14:27.138626Z","iopub.execute_input":"2023-09-07T07:14:27.138976Z","iopub.status.idle":"2023-09-07T07:14:30.922407Z","shell.execute_reply.started":"2023-09-07T07:14:27.138943Z","shell.execute_reply":"2023-09-07T07:14:30.921221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ENMO Data","metadata":{}},{"cell_type":"markdown","source":"1. Min is 0 max is 1.14 - but don't know for sure bc only some of the data\n2. std of 1 so they are probably all pretty close together whatever they are","metadata":{}},{"cell_type":"code","source":"enmo_data = df_train_series['enmo']\n\nprint(enmo_data.describe().compute())","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:14:30.923999Z","iopub.execute_input":"2023-09-07T07:14:30.924772Z","iopub.status.idle":"2023-09-07T07:14:35.268256Z","shell.execute_reply.started":"2023-09-07T07:14:30.924727Z","shell.execute_reply":"2023-09-07T07:14:35.266995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(enmo_data.head())\nprint(enmo_data.tail())","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:14:35.271900Z","iopub.execute_input":"2023-09-07T07:14:35.272350Z","iopub.status.idle":"2023-09-07T07:14:35.371907Z","shell.execute_reply.started":"2023-09-07T07:14:35.272310Z","shell.execute_reply":"2023-09-07T07:14:35.369826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Series ID And Step Data:\n1. I am thinking this data is not significant but could be used for orginization? ","metadata":{}},{"cell_type":"code","source":"series_id = df_train_series['series_id']\nstep = df_train_series['step']\n\nprint(series_id.tail(), series_id.describe().compute())\nprint(step.tail(), step.describe().compute())\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:15:55.087662Z","iopub.execute_input":"2023-09-07T07:15:55.088075Z","iopub.status.idle":"2023-09-07T07:16:37.512321Z","shell.execute_reply.started":"2023-09-07T07:15:55.088043Z","shell.execute_reply":"2023-09-07T07:16:37.511077Z"},"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}