{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"# packages\n\n# standard\nimport numpy as np\nimport pandas as pd\nimport time\n\n# plots\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-08T06:01:18.119147Z","iopub.execute_input":"2023-09-08T06:01:18.120140Z","iopub.status.idle":"2023-09-08T06:01:20.363762Z","shell.execute_reply.started":"2023-09-08T06:01:18.120103Z","shell.execute_reply":"2023-09-08T06:01:20.362637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"# load csv data\ndf_train_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv') \n\n# Converting the timestamp to datetime format\ndf_train_events['date'] = df_train_events['timestamp'].str.split('T', expand=True)[0]\ndf_train_events['time'] = df_train_events['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\ndf_train_events['timestamp'] = pd.to_datetime(df_train_events['date']+' '+df_train_events['time'])\ndf_train_events['hour'] = df_train_events['timestamp'].dt.hour\ndf_train_events.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:01:22.658549Z","iopub.execute_input":"2023-09-08T06:01:22.659048Z","iopub.status.idle":"2023-09-08T06:01:22.829424Z","shell.execute_reply.started":"2023-09-08T06:01:22.659013Z","shell.execute_reply":"2023-09-08T06:01:22.828242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load parquet data\ndf_train_pq = pd.read_parquet('../input/child-mind-institute-detect-sleep-states/train_series.parquet')\ndf_test_pq = pd.read_parquet('../input/child-mind-institute-detect-sleep-states/test_series.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:01:28.293770Z","iopub.execute_input":"2023-09-08T06:01:28.294142Z","iopub.status.idle":"2023-09-08T06:02:22.614585Z","shell.execute_reply.started":"2023-09-08T06:01:28.294112Z","shell.execute_reply":"2023-09-08T06:02:22.613681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Understanding and EDA for train data Events","metadata":{}},{"cell_type":"code","source":"df_train_events.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:25.163759Z","iopub.execute_input":"2023-09-08T06:02:25.164130Z","iopub.status.idle":"2023-09-08T06:02:25.185321Z","shell.execute_reply.started":"2023-09-08T06:02:25.164099Z","shell.execute_reply":"2023-09-08T06:02:25.184104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_events.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:25.936131Z","iopub.execute_input":"2023-09-08T06:02:25.936571Z","iopub.status.idle":"2023-09-08T06:02:25.963589Z","shell.execute_reply.started":"2023-09-08T06:02:25.936538Z","shell.execute_reply":"2023-09-08T06:02:25.962171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# base statistics\ndf_train_events.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:28.254266Z","iopub.execute_input":"2023-09-08T06:02:28.254669Z","iopub.status.idle":"2023-09-08T06:02:28.304120Z","shell.execute_reply.started":"2023-09-08T06:02:28.254632Z","shell.execute_reply":"2023-09-08T06:02:28.303023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def summary(df):\n    summary_df = pd.DataFrame(df.dtypes, columns=['dtypes'])\n    summary_df['missing#'] = df.isna().sum().values*100\n    summary_df['missing%'] = (df.isna().sum().values*100)/len(df)\n    summary_df['uniques'] = df.nunique().values\n    summary_df['count'] = df.count().values\n    #sum['skew'] = df.skew().values\n    desc = pd.DataFrame(df.describe().T)\n    summary_df['min'] = desc['min']\n    summary_df['max'] = desc['max']\n    summary_df['mean'] = desc['mean']\n    return summary_df\n\nsummary(df_train_events).style.background_gradient(cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:29.586676Z","iopub.execute_input":"2023-09-08T06:02:29.587412Z","iopub.status.idle":"2023-09-08T06:02:29.736460Z","shell.execute_reply.started":"2023-09-08T06:02:29.587365Z","shell.execute_reply":"2023-09-08T06:02:29.735270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Let's visualize the frequency of `onset` and `wakeup` events along the hour axis in a histogram plot.*** <br>\n\nIn the competition, `Onset` and `Wakeup` events signify the times people go to sleep and wake up, respectively.<br>\n\nUnderstanding the temporal distribution of these events can offer critical insights into sleep patterns and behavior.<br>\n\nTo that end, we can employ histograms to visualize the frequency of Onset and Wakeup events along the hour axis.","metadata":{}},{"cell_type":"code","source":"# Set figure size\nplt.figure(figsize=(10, 5))\n\n# Assuming your DataFrame is called train_events\ndf_onset = df_train_events[df_train_events['event'] == 'onset']\ndf_wakeup = df_train_events[df_train_events['event'] == 'wakeup']\n\n# Plotting the distribution plot for onset events\nsns.histplot(df_onset['hour'].dropna(), kde=False, bins=24, label='Onset', color = 'green') \n\n# Plotting the distribution plot for wakeup events\nsns.histplot(df_wakeup['hour'].dropna(), kde=False, bins=24, label='Wakeup', color = 'red')\n\n# Adding labels and title\nplt.xlabel('Hour of Event')\nplt.ylabel('Frequency')\nplt.title('Distribution of Hour of Onset and Wakeup')\nplt.legend()\n\n# Show the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:32.070746Z","iopub.execute_input":"2023-09-08T06:02:32.071614Z","iopub.status.idle":"2023-09-08T06:02:32.627719Z","shell.execute_reply.started":"2023-09-08T06:02:32.071570Z","shell.execute_reply":"2023-09-08T06:02:32.626756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nIn the resulting histogram, the x-axis denotes the 24-hour timestamp, while the y-axis represents the frequency of Onset and Wakeup events.\n\nFrom the data, it is evident that the majority of individuals tend to wake up between 6-8 am and go to sleep around 8-10 pm.\n\nThis temporal pattern aligns well with general expectations and offers a useful baseline for further analyses.","metadata":{}},{"cell_type":"code","source":"# Distribution of event\nsns.countplot(x ='event', data = df_train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:34.943209Z","iopub.execute_input":"2023-09-08T06:02:34.944037Z","iopub.status.idle":"2023-09-08T06:02:35.127617Z","shell.execute_reply.started":"2023-09-08T06:02:34.943990Z","shell.execute_reply":"2023-09-08T06:02:35.126407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of steps\n# plt.style.use('Solarize_Light2')\nvariable = df_train_events[\"step\"].values\nax = sns.displot(variable)\nfig = plt.gcf()\nfig.set_size_inches(12.5, 4.5)\nplt.title(f'Distribution of steps')\nplt.xlabel(\"Step\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:36.909760Z","iopub.execute_input":"2023-09-08T06:02:36.910513Z","iopub.status.idle":"2023-09-08T06:02:37.445764Z","shell.execute_reply.started":"2023-09-08T06:02:36.910464Z","shell.execute_reply":"2023-09-08T06:02:37.444465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data understanding and EDA for train data parquet","metadata":{}},{"cell_type":"code","source":"df_train_pq.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:39.557841Z","iopub.execute_input":"2023-09-08T06:02:39.558266Z","iopub.status.idle":"2023-09-08T06:02:39.571638Z","shell.execute_reply.started":"2023-09-08T06:02:39.558228Z","shell.execute_reply":"2023-09-08T06:02:39.570500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#base info\ndf_train_pq.info(show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:02:43.134517Z","iopub.execute_input":"2023-09-08T06:02:43.135236Z","iopub.status.idle":"2023-09-08T06:02:55.517630Z","shell.execute_reply.started":"2023-09-08T06:02:43.135167Z","shell.execute_reply":"2023-09-08T06:02:55.516547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# base statistics\ndf_train_pq.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:03:08.879845Z","iopub.execute_input":"2023-09-08T06:03:08.880317Z","iopub.status.idle":"2023-09-08T06:04:48.638103Z","shell.execute_reply.started":"2023-09-08T06:03:08.880276Z","shell.execute_reply":"2023-09-08T06:04:48.636986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(df_train_pq).style.background_gradient(cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:04:48.639993Z","iopub.execute_input":"2023-09-08T06:04:48.640321Z","iopub.status.idle":"2023-09-08T06:06:42.980779Z","shell.execute_reply.started":"2023-09-08T06:04:48.640292Z","shell.execute_reply":"2023-09-08T06:06:42.979129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot angle feature\ndf_train_pq.anglez.plot(kind='hist')\nplt.title('anglez')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:06:42.982325Z","iopub.execute_input":"2023-09-08T06:06:42.982697Z","iopub.status.idle":"2023-09-08T06:06:55.291846Z","shell.execute_reply.started":"2023-09-08T06:06:42.982664Z","shell.execute_reply":"2023-09-08T06:06:55.290568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot enmo feature\ndf_train_pq.enmo.plot(kind='hist', bins=100)\nplt.title('enmo')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:06:55.294255Z","iopub.execute_input":"2023-09-08T06:06:55.294621Z","iopub.status.idle":"2023-09-08T06:07:06.012313Z","shell.execute_reply.started":"2023-09-08T06:06:55.294589Z","shell.execute_reply":"2023-09-08T06:07:06.010945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Thanks to Chun Fu for the idea of below graphs. https://www.kaggle.com/code/patrick0302/viz-of-sleeping-time-series\n\ndef plot_series_with_events(series_id, train_series, train_events):\n    # Filter the DataFrame based on the series_id\n    sample_serie = train_series[train_series['series_id'] == series_id]\n    \n    # Convert timestamp to Pandas datetime object\n    sample_serie['date'] = sample_serie['timestamp'].str.split('T', expand=True)[0]\n    sample_serie['time'] = sample_serie['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    sample_serie['timestamp'] = pd.to_datetime(sample_serie['date'] + ' ' + sample_serie['time'])\n    \n    # Filter event data based on the series_id\n    sample_events = train_events[train_events['series_id'] == series_id]\n    sample_onset = sample_events.loc[sample_events['event'] == 'onset', 'timestamp'].dropna()\n    sample_wakeup = sample_events.loc[sample_events['event'] == 'wakeup', 'timestamp'].dropna()\n    \n    # Helper function to plot data and events\n    def plot_data_and_events(data, ylabel):\n        plt.figure(figsize=(20, 3))\n        plt.plot(sample_serie['timestamp'], sample_serie[data], label=data, linewidth=1)\n        \n        for onset in sample_onset:\n            plt.axvline(x=onset, color='r', linestyle='--', label='onset')\n            \n        for wakeup in sample_wakeup:\n            plt.axvline(x=wakeup, color='g', linestyle='--', label='wakeup')\n            \n        handles, labels = plt.gca().get_legend_handles_labels()\n        new_labels, new_handles = [], []\n        for handle, label in zip(handles, labels):\n            if label not in new_labels:\n                new_handles.append(handle)\n                new_labels.append(label)\n        \n        plt.legend(new_handles, new_labels)\n        plt.xlabel('Timestamp')\n        plt.ylabel(ylabel)\n        plt.title(f'{ylabel} over Time with Event Flags - '+series_id)\n        plt.show()\n    \n    # Plot enmo and anglez\n    plot_data_and_events('enmo', 'ENMO Value')\n    plot_data_and_events('anglez', 'anglez Value')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:07:06.013868Z","iopub.execute_input":"2023-09-08T06:07:06.014243Z","iopub.status.idle":"2023-09-08T06:07:06.027377Z","shell.execute_reply.started":"2023-09-08T06:07:06.014204Z","shell.execute_reply":"2023-09-08T06:07:06.025976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series_id in df_train_events['series_id'].unique()[:10]:\n    plot_series_with_events(series_id, df_train_pq, df_train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:07:06.029242Z","iopub.execute_input":"2023-09-08T06:07:06.029672Z","iopub.status.idle":"2023-09-08T06:10:09.062510Z","shell.execute_reply.started":"2023-09-08T06:07:06.029632Z","shell.execute_reply":"2023-09-08T06:10:09.061564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_series_with_events_new(series_id, train_series, train_events):\n    # Filter the DataFrame based on the series_id\n    sample_serie = train_series[train_series['series_id'] == series_id]\n    \n    # Convert timestamp to Pandas datetime object\n    sample_serie['date'] = sample_serie['timestamp'].str.split('T', expand=True)[0]\n    sample_serie['time'] = sample_serie['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    sample_serie['timestamp'] = pd.to_datetime(sample_serie['date'] + ' ' + sample_serie['time'])\n    \n    # Filter event data based on the series_id\n    sample_events = train_events[train_events['series_id'] == series_id]\n    sample_onset = sample_events.loc[sample_events['event'] == 'onset', 'timestamp'].dropna()\n    sample_wakeup = sample_events.loc[sample_events['event'] == 'wakeup', 'timestamp'].dropna()\n    \n    # Helper function to plot data and events\n    def plot_data_and_events_both(ylabel):\n        plt.figure(figsize=(20, 3))\n        plt.plot(sample_serie['timestamp'], sample_serie[\"enmo\"], label=\"enmo\", linewidth=1)\n        plt.plot(sample_serie['timestamp'], sample_serie[\"anglez\"], label=\"anglez\", linewidth=1)\n        for onset in sample_onset:\n            plt.axvline(x=onset, color='r', linestyle='--', label='onset')\n            \n        for wakeup in sample_wakeup:\n            plt.axvline(x=wakeup, color='g', linestyle='--', label='wakeup')\n            \n        handles, labels = plt.gca().get_legend_handles_labels()\n        new_labels, new_handles = [], []\n        for handle, label in zip(handles, labels):\n            if label not in new_labels:\n                new_handles.append(handle)\n                new_labels.append(label)\n        \n        plt.legend(new_handles, new_labels)\n        plt.xlabel('Timestamp')\n        plt.ylabel(ylabel)\n        plt.title(f'{ylabel} over Time with Event Flags - '+series_id)\n        plt.show()\n    \n    # Plot enmo and anglez\n    plot_data_and_events_both('enmo_anglez_value')\n#     plot_data_and_events_both('anglez', 'anglez Value')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:10:09.063718Z","iopub.execute_input":"2023-09-08T06:10:09.064790Z","iopub.status.idle":"2023-09-08T06:10:09.076476Z","shell.execute_reply.started":"2023-09-08T06:10:09.064754Z","shell.execute_reply":"2023-09-08T06:10:09.075540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series_id in df_train_events['series_id'].unique()[:2]:\n    plot_series_with_events_new(series_id, df_train_pq, df_train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:10:09.077854Z","iopub.execute_input":"2023-09-08T06:10:09.078640Z","iopub.status.idle":"2023-09-08T06:10:48.980492Z","shell.execute_reply.started":"2023-09-08T06:10:09.078607Z","shell.execute_reply":"2023-09-08T06:10:48.979472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Not much usefull as the scales are very different","metadata":{}},{"cell_type":"markdown","source":"## Data Understanding and EDA on test data parquet","metadata":{}},{"cell_type":"code","source":"df_test_pq.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:10:48.981924Z","iopub.execute_input":"2023-09-08T06:10:48.982268Z","iopub.status.idle":"2023-09-08T06:10:48.996327Z","shell.execute_reply.started":"2023-09-08T06:10:48.982228Z","shell.execute_reply":"2023-09-08T06:10:48.994619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#base info\ndf_test_pq.info(show_counts=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:10:49.000138Z","iopub.execute_input":"2023-09-08T06:10:49.000604Z","iopub.status.idle":"2023-09-08T06:10:49.019518Z","shell.execute_reply.started":"2023-09-08T06:10:49.000570Z","shell.execute_reply":"2023-09-08T06:10:49.018166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# base statistics\ndf_test_pq.describe(include='all')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:10:49.022248Z","iopub.execute_input":"2023-09-08T06:10:49.023123Z","iopub.status.idle":"2023-09-08T06:10:49.048534Z","shell.execute_reply.started":"2023-09-08T06:10:49.023085Z","shell.execute_reply":"2023-09-08T06:10:49.046811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(df_test_pq).style.background_gradient(cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2023-09-08T06:10:49.049945Z","iopub.execute_input":"2023-09-08T06:10:49.050683Z","iopub.status.idle":"2023-09-08T06:10:49.088424Z","shell.execute_reply.started":"2023-09-08T06:10:49.050617Z","shell.execute_reply":"2023-09-08T06:10:49.087283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}