{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# *Import all Library which gone help to predict and visualize in better way*","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objs as go\nimport plotly\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport cufflinks as cf\nfrom itertools import groupby\nfrom sklearn.ensemble import RandomForestClassifier\ncf.set_config_file(offline=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:20:06.057440Z","iopub.execute_input":"2023-09-10T06:20:06.057825Z","iopub.status.idle":"2023-09-10T06:20:10.292143Z","shell.execute_reply.started":"2023-09-10T06:20:06.057793Z","shell.execute_reply":"2023-09-10T06:20:10.291025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *Let's see Dataset*","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-09-10T04:24:57.649487Z","iopub.execute_input":"2023-09-10T04:24:57.650079Z","iopub.status.idle":"2023-09-10T04:24:57.657551Z","shell.execute_reply.started":"2023-09-10T04:24:57.650030Z","shell.execute_reply":"2023-09-10T04:24:57.656260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\ntest_series = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\ntrain_series.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T04:27:05.777178Z","iopub.execute_input":"2023-09-10T04:27:05.777545Z","iopub.status.idle":"2023-09-10T04:28:10.850935Z","shell.execute_reply.started":"2023-09-10T04:27:05.777518Z","shell.execute_reply":"2023-09-10T04:28:10.849801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n\ntrain_events['date'] = train_events['timestamp'].str.split('T', expand=True)[0]\ntrain_events['time'] = train_events['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\ntrain_events['timestamp'] = pd.to_datetime(train_events['date']+' '+train_events['time'])\ntrain_events['hour'] = train_events['timestamp'].dt.hour\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2023-09-10T04:28:10.853111Z","iopub.execute_input":"2023-09-10T04:28:10.853766Z","iopub.status.idle":"2023-09-10T04:28:11.044198Z","shell.execute_reply.started":"2023-09-10T04:28:10.853725Z","shell.execute_reply":"2023-09-10T04:28:11.042933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *Let's visualize the availability of the train_series (1: available; 0: NaN)*\nThis visualization aids in discerning the periods during which events ('onset' and 'wakeup') have been recorded.\n\nA clustermap is employed for this visualization, where the y-axis represents individual time series clustered based on their data collection similarity.\n\nUpon examining the clustermap, it is evident that the majority of the series tend to have a duration spanning 20 to 30 days.\n\nInterestingly, the periods during which data have been collected are sparsely distributed across the years 2017 to 2019.\n\n","metadata":{}},{"cell_type":"code","source":"df_availability = train_events.pivot_table(columns='date', index='series_id', values='step', aggfunc='count')\n\n# Fill NA values with 0\ndf_availability = (~df_availability.isna()).fillna(0).astype('int')\n\n# Plotting the clustered heatmap\nsns.clustermap(df_availability, col_cluster=False, figsize=(12, 8))","metadata":{"execution":{"iopub.status.busy":"2023-09-10T04:29:38.038519Z","iopub.execute_input":"2023-09-10T04:29:38.039823Z","iopub.status.idle":"2023-09-10T04:29:40.476857Z","shell.execute_reply.started":"2023-09-10T04:29:38.039780Z","shell.execute_reply":"2023-09-10T04:29:40.475932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *Let's visualize the frequency of onset and wakeup events along the hour axis in a histogram plot.*\nIn the competition, Onset and Wakeup events signify the times people go to sleep and wake up, respectively.\n\nUnderstanding the temporal distribution of these events can offer critical insights into sleep patterns and behavior.\n\nTo that end, we can employ histograms to visualize the frequency of Onset and Wakeup events along the hour axis.\n\nIn the resulting histogram, the x-axis denotes the 24-hour timestamp, while the y-axis represents the frequency of Onset and Wakeup events.\n\nFrom the data, it is evident that the majority of individuals tend to wake up between 6-8 am and go to sleep around 8-10 pm.\n\nThis temporal pattern aligns well with general expectations and offers a useful baseline for further analyses.","metadata":{}},{"cell_type":"code","source":"# Set figure size\nplt.figure(figsize=(10, 5))\n\n# Assuming your DataFrame is called train_events\ndf_onset = train_events[train_events['event'] == 'onset']\ndf_wakeup = train_events[train_events['event'] == 'wakeup']\n\n# Plotting the distribution plot for onset events\nsns.histplot(df_onset['hour'].dropna(), kde=False, bins=24, label='Onset', discrete=True)  # You can adjust bins as needed\n\n# Plotting the distribution plot for wakeup events\nsns.histplot(df_wakeup['hour'].dropna(), kde=False, bins=24, label='Wakeup', discrete=True)  # You can adjust bins as needed\n\n# Adding labels and title\nplt.xlabel('Hour of Event')\nplt.ylabel('Frequency')\nplt.title('Distribution of Hour of Onset and Wakeup')\nplt.legend()\n\n# Show the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-10T04:30:28.694402Z","iopub.execute_input":"2023-09-10T04:30:28.694825Z","iopub.status.idle":"2023-09-10T04:30:29.131922Z","shell.execute_reply.started":"2023-09-10T04:30:28.694792Z","shell.execute_reply":"2023-09-10T04:30:29.130842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# *Let's visualize the time series of enmo and anglez with the flag of onset and wakeup for 10 samples in train_series¶*\nTo advance our understanding of sleep behavior, it is essential to delve deeper into the time series of enmo and anglez, particularly in relation to the onset (Go-to-sleep) and wakeup (Wake-up) events.\n\nWe employ time-series plots to flag the events of 'Onset' and 'Wakeup' within the enmo and anglez data streams for series in train_series.\n\nUpon examining the time-series plots, several observations can be made:\n\nenmo levels decrease during sleep periods—defined as the time between Onset and Wakeup events.\nFor anglez, the intensity appears to differ between sleep and wake times, but not very significant.\nInterestingly, not all profiles are consistently labeled, and some appear to be repetitive over multiple days.\n","metadata":{}},{"cell_type":"code","source":"def plot_series_with_events(series_id, train_series, train_events):\n    # Filter the DataFrame based on the series_id\n    sample_serie = train_series[train_series['series_id'] == series_id]\n    \n    # Convert timestamp to Pandas datetime object\n    sample_serie['date'] = sample_serie['timestamp'].str.split('T', expand=True)[0]\n    sample_serie['time'] = sample_serie['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    sample_serie['timestamp'] = pd.to_datetime(sample_serie['date'] + ' ' + sample_serie['time'])\n    \n    # Filter event data based on the series_id\n    sample_events = train_events[train_events['series_id'] == series_id]\n    sample_onset = sample_events.loc[sample_events['event'] == 'onset', 'timestamp'].dropna()\n    sample_wakeup = sample_events.loc[sample_events['event'] == 'wakeup', 'timestamp'].dropna()\n    \n    # Helper function to plot data and events\n    def plot_data_and_events(data, ylabel):\n        plt.figure(figsize=(20, 3))\n        plt.plot(sample_serie['timestamp'], sample_serie[data], label=data, linewidth=1)\n        \n        for onset in sample_onset:\n            plt.axvline(x=onset, color='r', linestyle='--', label='onset')\n            \n        for wakeup in sample_wakeup:\n            plt.axvline(x=wakeup, color='g', linestyle='--', label='wakeup')\n            \n        handles, labels = plt.gca().get_legend_handles_labels()\n        new_labels, new_handles = [], []\n        for handle, label in zip(handles, labels):\n            if label not in new_labels:\n                new_handles.append(handle)\n                new_labels.append(label)\n        \n        plt.legend(new_handles, new_labels)\n        plt.xlabel('Timestamp')\n        plt.ylabel(ylabel)\n        plt.title(f'{ylabel} over Time with Event Flags - '+series_id)\n        plt.show()\n    \n    # Plot enmo and anglez\n    plot_data_and_events('enmo', 'ENMO Value')\n    plot_data_and_events('anglez', 'anglez Value')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-09-10T04:32:04.456319Z","iopub.execute_input":"2023-09-10T04:32:04.456764Z","iopub.status.idle":"2023-09-10T04:32:04.469013Z","shell.execute_reply.started":"2023-09-10T04:32:04.456730Z","shell.execute_reply":"2023-09-10T04:32:04.467736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series_id in train_events['series_id'].unique()[:10]:\n    plot_series_with_events(series_id, train_series, train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-10T04:32:14.966049Z","iopub.execute_input":"2023-09-10T04:32:14.966511Z","iopub.status.idle":"2023-09-10T04:35:22.552863Z","shell.execute_reply.started":"2023-09-10T04:32:14.966470Z","shell.execute_reply":"2023-09-10T04:35:22.551591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet\")\ntest  = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:20:12.916312Z","iopub.execute_input":"2023-09-10T06:20:12.917167Z","iopub.status.idle":"2023-09-10T06:20:22.337362Z","shell.execute_reply.started":"2023-09-10T06:20:12.917115Z","shell.execute_reply":"2023-09-10T06:20:22.336007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for df in (train, test):\n    # parse the timestamp and create an \"hour\" feature\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"],utc=True)\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    # feature cross\n    df[\"anglez_times_enmo\"] = abs(df[\"anglez\"]) * df[\"enmo\"]\n    # \"rolling\" features\n    periods = 50\n    df[\"anglez_diff\"] = df.groupby('series_id')['anglez'].diff(periods=periods).fillna(method=\"bfill\")\n    df[\"enmo_diff\"]   = df.groupby('series_id')['enmo'].diff(periods=periods).fillna(method=\"bfill\")\n    df[\"anglez_rolling\"] = df[\"anglez\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    df[\"enmo_rolling\"]   = df[\"enmo\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    df[\"anglez_diff_rolling\"] = df[\"anglez_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    df[\"enmo_diff_rolling\"]   = df[\"enmo_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    \nfeatures = [\"hour\",\"anglez_times_enmo\",\n           \"anglez\",\"anglez_diff\",\"anglez_rolling\",\"anglez_diff_rolling\",\n           \"enmo\",\"enmo_diff\",\"enmo_rolling\",\"enmo_diff_rolling\"]\n","metadata":{"execution":{"iopub.status.busy":"2023-09-10T06:21:09.487026Z","iopub.execute_input":"2023-09-10T06:21:09.487470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfrom sklearn.ensemble import RandomForestClassifier\nclassifier = RandomForestClassifier(n_estimators=50,\n                                    max_depth=4,\n                                    min_samples_leaf=100)\n\nX_train = train[features]\ny_train = train[\"awake\"]\n\nclassifier.fit(X_train, y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test[features]\ntest[\"score\"] = classifier.predict_proba(X_test)[:,1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"not_awake\"] = 1-test[\"score\"]\n# smoothing of the predictions\nsmoothing_length = 2*250\ntest[\"smooth\"] = test[\"not_awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n# re-binarize\ntest[\"smooth\"] = test[\"smooth\"].round()\n\n# https://stackoverflow.com/questions/73777727/how-to-mark-start-end-of-a-series-of-non-null-and-non-0-values-in-a-column-of-a\ndef get_event(df):\n    lstCV = zip(df.series_id, df.smooth)\n    lstPOI = []\n    for (c, v), g in groupby(lstCV, lambda cv: \n                            (cv[0], cv[1]!=0 and not pd.isnull(cv[1]))):\n        llg = sum(1 for item in g)\n        if v is False: \n            lstPOI.extend([0]*llg)\n        else: \n            lstPOI.extend(['onset']+(llg-2)*[0]+['wakeup'] if llg > 1 else [0])\n    return lstPOI\n\ntest[\"event\"] = get_event(test)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = test.loc[test[\"event\"] != 0][[\"series_id\",\"step\",\"event\",\"score\"]].copy().reset_index(drop=True).reset_index(names=\"row_id\")\nsample_submission.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}