{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-13T12:41:40.301782Z","iopub.execute_input":"2023-10-13T12:41:40.302326Z","iopub.status.idle":"2023-10-13T12:41:40.698466Z","shell.execute_reply.started":"2023-10-13T12:41:40.302293Z","shell.execute_reply":"2023-10-13T12:41:40.697361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport random\n\nimport dask #Library for parallel computing\nimport dask.array as da\nimport dask.dataframe as dd","metadata":{"execution":{"iopub.status.busy":"2023-10-13T12:41:50.019458Z","iopub.execute_input":"2023-10-13T12:41:50.020122Z","iopub.status.idle":"2023-10-13T12:41:51.619838Z","shell.execute_reply.started":"2023-10-13T12:41:50.020080Z","shell.execute_reply":"2023-10-13T12:41:51.618723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n#Magic command - Time execution of a Python statement or expression. It prints the CPU and wall clock times.\n\nimport polars as pl\ntrain_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n                .with_columns(\n                    (\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")),\n\n                    )\n                )\n                .collect()\n                .to_pandas()\n               )","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:11:50.481182Z","iopub.execute_input":"2023-10-12T19:11:50.481732Z","iopub.status.idle":"2023-10-12T19:12:44.341014Z","shell.execute_reply.started":"2023-10-12T19:11:50.481701Z","shell.execute_reply":"2023-10-12T19:12:44.339956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Why polars?\nPandas is the go-to option for data practitioners using Python, but, it comes with several important limitations when it comes to dealing with large datasets - pandas is threaded, meaning that it cannot leverage parallel computing techniques that are now common in data science.The core mission of polars is to offer a lightning-fast dataframe library that addresses all the problems of pandas.","metadata":{}},{"cell_type":"markdown","source":"### Memory reduction","metadata":{}},{"cell_type":"code","source":"from pandas.api.types import is_datetime64_ns_dtype\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndef reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n\n    df['series_id'] = df['series_id'].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:12:55.073820Z","iopub.execute_input":"2023-10-12T19:12:55.074251Z","iopub.status.idle":"2023-10-12T19:12:55.086391Z","shell.execute_reply.started":"2023-10-12T19:12:55.074218Z","shell.execute_reply":"2023-10-12T19:12:55.085062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = reduce_mem_usage(train_series)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:13:05.371167Z","iopub.execute_input":"2023-10-12T19:13:05.371537Z","iopub.status.idle":"2023-10-12T19:13:17.706428Z","shell.execute_reply.started":"2023-10-12T19:13:05.371509Z","shell.execute_reply":"2023-10-12T19:13:17.705179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:13:21.631044Z","iopub.execute_input":"2023-10-12T19:13:21.631604Z","iopub.status.idle":"2023-10-12T19:13:21.649667Z","shell.execute_reply.started":"2023-10-12T19:13:21.631576Z","shell.execute_reply":"2023-10-12T19:13:21.648713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Read train_events csv file\ntrain_events = (pl.scan_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n                .with_columns(\n                    (\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.year().alias(\"year\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.month().alias(\"month\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.day().alias(\"day\")),\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\").dt.hour().alias(\"hour\")),\n                    )\n                )\n                .collect()\n                .to_pandas()\n               )\ntrain_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:13:28.346593Z","iopub.execute_input":"2023-10-12T19:13:28.347002Z","iopub.status.idle":"2023-10-12T19:13:28.409959Z","shell.execute_reply.started":"2023-10-12T19:13:28.346967Z","shell.execute_reply":"2023-10-12T19:13:28.408764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Convert timestamp to year, month, day, and hour\n#train_events['date'] =  pd.to_datetime(train_events['timestamp'], errors = 'coerce', utc = True) ## pandas recognizes your format\n\n#train_events['year'] = train_events['date'].dt.year\n#train_events['month'] = train_events['date'].dt.month\n#train_events['day'] = train_events['date'].dt.day\n#train_events['hour'] = train_events['date'].dt.hour\n#train_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-10T20:41:54.429133Z","iopub.execute_input":"2023-10-10T20:41:54.429680Z","iopub.status.idle":"2023-10-10T20:41:54.444673Z","shell.execute_reply.started":"2023-10-10T20:41:54.429649Z","shell.execute_reply":"2023-10-10T20:41:54.443204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # ***Preprocess the data***","metadata":{}},{"cell_type":"code","source":"#Sort the train_series dataset by the timestamp\ntrain_series = train_series.sort_values('timestamp').reset_index(drop=True)\ntrain_series.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:13:40.022264Z","iopub.execute_input":"2023-10-12T19:13:40.022751Z","iopub.status.idle":"2023-10-12T19:13:58.982670Z","shell.execute_reply.started":"2023-10-12T19:13:40.022717Z","shell.execute_reply":"2023-10-12T19:13:58.981482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Sort the train_events dataset by the timestamp\ntrain_events = train_events.sort_values('timestamp').reset_index(drop=True)\ntrain_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:14:34.789857Z","iopub.execute_input":"2023-10-12T19:14:34.790226Z","iopub.status.idle":"2023-10-12T19:14:34.811271Z","shell.execute_reply.started":"2023-10-12T19:14:34.790199Z","shell.execute_reply":"2023-10-12T19:14:34.810055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(\"Train Series Shape:\", train_series.shape)\n#print(\"Train Events Shape:\", train_events.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-10T20:43:46.899946Z","iopub.execute_input":"2023-10-10T20:43:46.901360Z","iopub.status.idle":"2023-10-10T20:43:46.907347Z","shell.execute_reply.started":"2023-10-10T20:43:46.901319Z","shell.execute_reply":"2023-10-10T20:43:46.906370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:14:44.406374Z","iopub.execute_input":"2023-10-12T19:14:44.406765Z","iopub.status.idle":"2023-10-12T19:14:44.430001Z","shell.execute_reply.started":"2023-10-12T19:14:44.406735Z","shell.execute_reply":"2023-10-12T19:14:44.428794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"### Sleep hours distribution\n\n* Extract the hour and day components from the difference between the rows of the timestamp columns\n* Make the difference between the rows of the timestamp columns\n* Create a pandas dataframe named train_sleep\n\nSince there are missing values in the train_events dataframe that can make the difference big, eliminate rows with a \"day\" value greater than 0 (it is good practice)  - of course assuming they have never slept for more than 24 hours.\nAfter the difference, the events are equal to -1 and +1, corresponding to hours of sleep and hours of wakefulness, respectively.\nFor our purpose, only the events = -1 will be kept.","metadata":{}},{"cell_type":"code","source":"### Difference between the rows of the timestamp column\n#Get Time Difference between Timestamps\ntrain_events_hours = train_events.timestamp.diff().dt.components['hours']   # Extract the hour component from the difference\ntrain_events_days  = train_events.timestamp.diff().dt.components['days']    # Extract the day component from the difference\ntrain_events_event = train_events.event.map({'onset':1, 'wakeup':0}).diff() # Encode the events and make the difference","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:15:05.306711Z","iopub.execute_input":"2023-10-12T19:15:05.307128Z","iopub.status.idle":"2023-10-12T19:15:05.508246Z","shell.execute_reply.started":"2023-10-12T19:15:05.307093Z","shell.execute_reply":"2023-10-12T19:15:05.507152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Create the pandas dataframe\ntrain_sleep = pd.DataFrame([train_events.month, train_events.day, train_events_days, train_events_hours, train_events_event], \n                           index=['month', 'day', 'days_sleep', 'hours_sleep', 'event']).T.dropna()\ntrain_sleep = train_sleep[(train_sleep.days_sleep == 0) & (train_sleep.event == -1)].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:15:12.402576Z","iopub.execute_input":"2023-10-12T19:15:12.402980Z","iopub.status.idle":"2023-10-12T19:15:12.503487Z","shell.execute_reply.started":"2023-10-12T19:15:12.402947Z","shell.execute_reply":"2023-10-12T19:15:12.502352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sleep.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:15:32.868511Z","iopub.execute_input":"2023-10-12T19:15:32.868887Z","iopub.status.idle":"2023-10-12T19:15:32.881116Z","shell.execute_reply.started":"2023-10-12T19:15:32.868857Z","shell.execute_reply":"2023-10-12T19:15:32.880279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sleep.describe()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:15:54.973840Z","iopub.execute_input":"2023-10-12T19:15:54.974270Z","iopub.status.idle":"2023-10-12T19:15:54.999653Z","shell.execute_reply.started":"2023-10-12T19:15:54.974241Z","shell.execute_reply":"2023-10-12T19:15:54.998722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Interpolation of the timestamp\nTime series interpolation is the process of estimating missing data points in a time series by using the surrounding data points. This is useful here since we are working with data that is missing values. Interpolation can help to smooth out the data and make it easier to analyze and visualize.I","metadata":{}},{"cell_type":"code","source":"train_events.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:16:00.681033Z","iopub.execute_input":"2023-10-12T19:16:00.681441Z","iopub.status.idle":"2023-10-12T19:16:00.704318Z","shell.execute_reply.started":"2023-10-12T19:16:00.681412Z","shell.execute_reply":"2023-10-12T19:16:00.703350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events['timestamp'] = train_events.timestamp.interpolate()\ntrain_events.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:16:08.246025Z","iopub.execute_input":"2023-10-12T19:16:08.246417Z","iopub.status.idle":"2023-10-12T19:16:08.269781Z","shell.execute_reply.started":"2023-10-12T19:16:08.246377Z","shell.execute_reply":"2023-10-12T19:16:08.269007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Merge the train_event and train_series datasets\n\nMerge the train_series dataset AND the 'timestamp' and'event' features/columns in the train_events dataset.","metadata":{}},{"cell_type":"code","source":"train_events_to_merge = train_events[['timestamp', 'event']]\ntrain_events_to_merge.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:16:15.114867Z","iopub.execute_input":"2023-10-12T19:16:15.115290Z","iopub.status.idle":"2023-10-12T19:16:15.125878Z","shell.execute_reply.started":"2023-10-12T19:16:15.115259Z","shell.execute_reply":"2023-10-12T19:16:15.125141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check the number of NaN values in the two columns of the newly created train_events_to_merge pandas DataFrame.\ntrain_events_to_merge.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:16:20.428886Z","iopub.execute_input":"2023-10-12T19:16:20.429287Z","iopub.status.idle":"2023-10-12T19:16:20.438631Z","shell.execute_reply.started":"2023-10-12T19:16:20.429257Z","shell.execute_reply":"2023-10-12T19:16:20.437515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events_to_merge.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:16:24.946056Z","iopub.execute_input":"2023-10-12T19:16:24.946438Z","iopub.status.idle":"2023-10-12T19:16:24.958438Z","shell.execute_reply.started":"2023-10-12T19:16:24.946409Z","shell.execute_reply":"2023-10-12T19:16:24.957243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_merged = train_series.merge(train_events_to_merge, how='left', on='timestamp')\ntrain_series_merged.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:16:33.226162Z","iopub.execute_input":"2023-10-12T19:16:33.226533Z","iopub.status.idle":"2023-10-12T19:16:56.955617Z","shell.execute_reply.started":"2023-10-12T19:16:33.226506Z","shell.execute_reply":"2023-10-12T19:16:56.954373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fill the missing values in the event column\n\nUse the 'ffill' method to replace NaN values with the value from the previous row.","metadata":{}},{"cell_type":"code","source":"train_series_merged['event'] = train_series_merged.event.fillna(method='ffill')","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:17:05.666432Z","iopub.execute_input":"2023-10-12T19:17:05.667435Z","iopub.status.idle":"2023-10-12T19:17:14.848235Z","shell.execute_reply.started":"2023-10-12T19:17:05.667395Z","shell.execute_reply":"2023-10-12T19:17:14.847119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this manner, the initial rows with missing values remain unchanged since there are no preceding values to replace them. When inspecting the train_events dataset, we observe that the initial event is 'onset', signifying the start of sleep. Consequently, the event immediately preceding 'onset' is 'wakeup.' To address this, we will proceed to populate the remaining rows with missing values with 'wakeup.'","metadata":{}},{"cell_type":"code","source":"train_series_merged['event'] = train_series_merged.event.fillna('wakeup')\ntrain_series_merged.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:17:19.806107Z","iopub.execute_input":"2023-10-12T19:17:19.806468Z","iopub.status.idle":"2023-10-12T19:17:32.045366Z","shell.execute_reply.started":"2023-10-12T19:17:19.806441Z","shell.execute_reply":"2023-10-12T19:17:32.044237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check for missing values in the event column\ntrain_series_merged.event.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:17:43.369283Z","iopub.execute_input":"2023-10-12T19:17:43.369663Z","iopub.status.idle":"2023-10-12T19:17:48.931944Z","shell.execute_reply.started":"2023-10-12T19:17:43.369635Z","shell.execute_reply":"2023-10-12T19:17:48.930691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Exploration","metadata":{}},{"cell_type":"code","source":"#Check train_events dataset\ntrain_events.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:17:53.809249Z","iopub.execute_input":"2023-10-12T19:17:53.809608Z","iopub.status.idle":"2023-10-12T19:17:53.822887Z","shell.execute_reply.started":"2023-10-12T19:17:53.809581Z","shell.execute_reply":"2023-10-12T19:17:53.822063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Explore the train_events dataset\ntrain_events.describe()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:17:59.708593Z","iopub.execute_input":"2023-10-12T19:17:59.708997Z","iopub.status.idle":"2023-10-12T19:17:59.744146Z","shell.execute_reply.started":"2023-10-12T19:17:59.708963Z","shell.execute_reply":"2023-10-12T19:17:59.743160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get unique values of Series object\nprint('Number of unique identifiers for each series of accelerometer data:', len(train_events.series_id.unique()))","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:18:08.048386Z","iopub.execute_input":"2023-10-12T19:18:08.048799Z","iopub.status.idle":"2023-10-12T19:18:08.054836Z","shell.execute_reply.started":"2023-10-12T19:18:08.048769Z","shell.execute_reply":"2023-10-12T19:18:08.053702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hours distribution","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\nplt.title('Hour distribution with onset and wakeup')\nsns.histplot(x=train_events.dropna().hour, hue=train_events.dropna().event, stat='density', bins=24, binrange=(-0.5, 23.5))\nsns.kdeplot(train_events.dropna().hour, bw_adjust=0.45);","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:18:37.588340Z","iopub.execute_input":"2023-10-12T19:18:37.588855Z","iopub.status.idle":"2023-10-12T19:18:38.238000Z","shell.execute_reply.started":"2023-10-12T19:18:37.588806Z","shell.execute_reply":"2023-10-12T19:18:38.236740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Sleep hours distribution\nplt.figure(figsize=(10,4))\nplt.title('Sleep hours distribution')\nsns.histplot(x=train_sleep.hours_sleep, stat='density', bins=19, binrange=(-0.5, 18.5))\nsns.kdeplot(train_sleep.hours_sleep, bw_adjust=2, color='red');","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:18:45.626176Z","iopub.execute_input":"2023-10-12T19:18:45.626854Z","iopub.status.idle":"2023-10-12T19:18:46.062194Z","shell.execute_reply.started":"2023-10-12T19:18:45.626821Z","shell.execute_reply":"2023-10-12T19:18:46.060915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Average sleep hours per day\nplt.figure(figsize=(15,3))\nplt.title('Average sleep hours per day')\nsns.barplot(x=train_sleep.groupby('day').hours_sleep.mean().index, y=train_sleep.groupby('day').hours_sleep.mean().values);","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:18:53.574320Z","iopub.execute_input":"2023-10-12T19:18:53.574681Z","iopub.status.idle":"2023-10-12T19:18:54.169283Z","shell.execute_reply.started":"2023-10-12T19:18:53.574654Z","shell.execute_reply":"2023-10-12T19:18:54.167926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Average sleep hours per month\nplt.figure(figsize=(10,3))\nplt.title('Average sleep hours per month')\nsns.barplot(x=train_sleep.groupby('month').hours_sleep.mean().index, y=train_sleep.groupby('month').hours_sleep.mean().values);","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:19:13.341614Z","iopub.execute_input":"2023-10-12T19:19:13.342035Z","iopub.status.idle":"2023-10-12T19:19:13.645896Z","shell.execute_reply.started":"2023-10-12T19:19:13.342000Z","shell.execute_reply":"2023-10-12T19:19:13.644733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The average number of sleep hour is about 12 hours. It could have been hypothesized that in the summer months and on weekends, the hours of sleep were greater.Is this hypothesis visible in the data? Not at all! Thus it is rejected.","metadata":{}},{"cell_type":"markdown","source":"## Explore the train_series_merged dataset","metadata":{}},{"cell_type":"code","source":"train_series.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:19:23.801505Z","iopub.execute_input":"2023-10-12T19:19:23.801988Z","iopub.status.idle":"2023-10-12T19:19:23.816468Z","shell.execute_reply.started":"2023-10-12T19:19:23.801948Z","shell.execute_reply":"2023-10-12T19:19:23.815165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check for any missing values\ntrain_series.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:19:30.901885Z","iopub.execute_input":"2023-10-12T19:19:30.902618Z","iopub.status.idle":"2023-10-12T19:19:31.802781Z","shell.execute_reply.started":"2023-10-12T19:19:30.902581Z","shell.execute_reply":"2023-10-12T19:19:31.801637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Get unique values of Series object\nprint(f'In the dataset there are {len(train_series.series_id.unique())} accelerometer series')","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:19:40.414702Z","iopub.execute_input":"2023-10-12T19:19:40.415105Z","iopub.status.idle":"2023-10-12T19:19:41.083784Z","shell.execute_reply.started":"2023-10-12T19:19:40.415073Z","shell.execute_reply":"2023-10-12T19:19:41.082647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14,3))\nsns.barplot(x=train_series.groupby('series_id').anglez.mean(),\n            y=train_series.groupby('series_id').anglez.mean())\ng = plt.xticks(ticks=[])","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:20:05.533681Z","iopub.execute_input":"2023-10-12T19:20:05.534084Z","iopub.status.idle":"2023-10-12T19:20:09.284884Z","shell.execute_reply.started":"2023-10-12T19:20:05.534053Z","shell.execute_reply":"2023-10-12T19:20:09.283692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Take five random series_id and explore them","metadata":{}},{"cell_type":"code","source":"series_id_selected = random.choices(train_series_merged.series_id.unique(), k = 5)\nseries_id_selected","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:30:14.402448Z","iopub.execute_input":"2023-10-12T19:30:14.402854Z","iopub.status.idle":"2023-10-12T19:30:15.082610Z","shell.execute_reply.started":"2023-10-12T19:30:14.402823Z","shell.execute_reply":"2023-10-12T19:30:15.081405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract a subset of the dataset with the selected series id\ntrain_series_subset = train_series_merged[train_series_merged.series_id.isin(series_id_selected)].reset_index(drop=True)\ntrain_series_subset.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:31:43.414369Z","iopub.execute_input":"2023-10-12T19:31:43.414788Z","iopub.status.idle":"2023-10-12T19:31:44.865968Z","shell.execute_reply.started":"2023-10-12T19:31:43.414758Z","shell.execute_reply":"2023-10-12T19:31:44.864791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Convert from datetime to timestamp\ntrain_series_subset['timestamp_new'] = train_series_subset.timestamp.astype(int).div(10**9)\ntrain_series_subset.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:33:58.476092Z","iopub.execute_input":"2023-10-12T19:33:58.476629Z","iopub.status.idle":"2023-10-12T19:33:58.513016Z","shell.execute_reply.started":"2023-10-12T19:33:58.476585Z","shell.execute_reply":"2023-10-12T19:33:58.511604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nplt.figure(figsize=(12,3))\nsns.scatterplot(data=train_series_subset[train_series_subset.series_id == series_id_selected[0]],\n             x='timestamp', y='anglez', hue='event');","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:35:50.455522Z","iopub.execute_input":"2023-10-12T19:35:50.455894Z","iopub.status.idle":"2023-10-12T19:36:22.132710Z","shell.execute_reply.started":"2023-10-12T19:35:50.455867Z","shell.execute_reply":"2023-10-12T19:36:22.131625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nplt.figure(figsize=(12,3))\nsns.scatterplot(x=train_series_subset[train_series_subset.series_id == series_id_selected[0]].timestamp,\n                y=train_series_subset[train_series_subset.series_id == series_id_selected[0]].anglez*train_series_subset[train_series_subset.series_id == series_id_selected[0]].enmo,\n                hue=train_series_subset[train_series_subset.series_id == series_id_selected[0]].event);","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:38:00.355897Z","iopub.execute_input":"2023-10-12T19:38:00.356319Z","iopub.status.idle":"2023-10-12T19:38:18.739230Z","shell.execute_reply.started":"2023-10-12T19:38:00.356288Z","shell.execute_reply":"2023-10-12T19:38:18.738070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor ids in series_id_selected:\n    plt.figure(figsize=(12,3))\n    plt.title(f'z-angle distribution with events for {ids} id')\n    sns.histplot(data=train_series_subset[train_series_subset.series_id == ids],\n                 x=train_series_subset[train_series_subset.series_id == ids].anglez.abs(), hue='event', fill=True, bins=100, alpha=0.3)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:40:28.144177Z","iopub.execute_input":"2023-10-12T19:40:28.144613Z","iopub.status.idle":"2023-10-12T19:40:33.990508Z","shell.execute_reply.started":"2023-10-12T19:40:28.144579Z","shell.execute_reply":"2023-10-12T19:40:33.989270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor ids in series_id_selected:\n    plt.figure(figsize=(12,3))\n    plt.title(f'z-angle distribution with events for {ids} id')\n    sns.histplot(data=train_series_subset[train_series_subset.series_id == ids],\n                 x='enmo', hue='event', fill=True, bins=100, alpha=0.3)\n    plt.yscale('log')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:43:29.119170Z","iopub.execute_input":"2023-10-12T19:43:29.119541Z","iopub.status.idle":"2023-10-12T19:43:37.297656Z","shell.execute_reply.started":"2023-10-12T19:43:29.119511Z","shell.execute_reply":"2023-10-12T19:43:37.296542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observations from the z-angle distribution:\n\n* the wakeup probability decrases as the absolute value of the angle increases.\n* the onset probability is almost constant for all the angle values, except for a few peaks\n\nAlso, from the ENMO distribution both the event count decreases as the enmo encreases","metadata":{}},{"cell_type":"code","source":"%%time\nplt.figure(figsize=(12,3))\nplt.title('z-angle distribution with events')\nsns.histplot(data=train_series_subset[train_series_subset.series_id == series_id_selected[0]],\n             x=np.sin(train_series_subset[train_series_subset.series_id == series_id_selected[0]].anglez)*\\\n             (train_series_subset[train_series_subset.series_id == series_id_selected[0]].enmo), \n             hue='event', fill=True, bins=100, alpha=0.3)\nplt.yscale('log')","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:48:31.719191Z","iopub.execute_input":"2023-10-12T19:48:31.720041Z","iopub.status.idle":"2023-10-12T19:48:33.674033Z","shell.execute_reply.started":"2023-10-12T19:48:31.720001Z","shell.execute_reply":"2023-10-12T19:48:33.672797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocess the test dataset","metadata":{}},{"cell_type":"code","source":"%%time\ntest_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n                .with_columns(\n                    (\n                        (pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%Z\")),\n\n                    )\n                )\n                .collect()\n                .to_pandas()\n               )","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:51:21.756393Z","iopub.execute_input":"2023-10-12T19:51:21.756789Z","iopub.status.idle":"2023-10-12T19:51:21.788558Z","shell.execute_reply.started":"2023-10-12T19:51:21.756761Z","shell.execute_reply":"2023-10-12T19:51:21.787474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series = reduce_mem_usage(test_series)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:51:55.098939Z","iopub.execute_input":"2023-10-12T19:51:55.099352Z","iopub.status.idle":"2023-10-12T19:51:55.109935Z","shell.execute_reply.started":"2023-10-12T19:51:55.099320Z","shell.execute_reply":"2023-10-12T19:51:55.109025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:52:35.219556Z","iopub.execute_input":"2023-10-12T19:52:35.220038Z","iopub.status.idle":"2023-10-12T19:52:35.234831Z","shell.execute_reply.started":"2023-10-12T19:52:35.220000Z","shell.execute_reply":"2023-10-12T19:52:35.233589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save the datasets","metadata":{}},{"cell_type":"code","source":"train_series_merged.to_parquet('/kaggle/working/train_series_preprocessed.parquet')\ntest_series.to_parquet('/kaggle/working/test_series_preprocessed.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-10-12T19:53:44.819418Z","iopub.execute_input":"2023-10-12T19:53:44.820634Z","iopub.status.idle":"2023-10-12T19:54:16.329461Z","shell.execute_reply.started":"2023-10-12T19:53:44.820581Z","shell.execute_reply":"2023-10-12T19:54:16.328089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Note that Kaggle has a memory limit of 8GB per kernel. So since the dataset is large, to avoid long times, preprocessing of the dataset was done in its own notebook , and then uploaded in second notebook and start the training part there. See Detect Sleep States: Decision Tree Model for the trained model.\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}