{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Set the maximum number of rows to display\npd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 100)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-02T08:52:01.520994Z","iopub.execute_input":"2023-12-02T08:52:01.521329Z","iopub.status.idle":"2023-12-02T08:52:02.993647Z","shell.execute_reply.started":"2023-12-02T08:52:01.521302Z","shell.execute_reply":"2023-12-02T08:52:02.992936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_full = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\ntrain_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\ntrain_series_full","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:52:09.167354Z","iopub.execute_input":"2023-12-02T08:52:09.167859Z","iopub.status.idle":"2023-12-02T08:52:59.026594Z","shell.execute_reply.started":"2023-12-02T08:52:09.167827Z","shell.execute_reply":"2023-12-02T08:52:59.025889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:53:08.957115Z","iopub.execute_input":"2023-12-02T08:53:08.957534Z","iopub.status.idle":"2023-12-02T08:53:08.976632Z","shell.execute_reply.started":"2023-12-02T08:53:08.957502Z","shell.execute_reply":"2023-12-02T08:53:08.975222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = train_series_full[train_series_full['series_id'] == '038441c925bb']","metadata":{"execution":{"iopub.execute_input":"2023-12-02T08:53:17.289055Z","iopub.status.idle":"2023-12-02T08:53:25.182610Z","shell.execute_reply.started":"2023-12-02T08:53:17.289030Z","shell.execute_reply":"2023-12-02T08:53:25.181573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def numerical_eda(df, numerical_column: str):\n    # Calculate common metrics\n    mean_value = df[numerical_column].mean()\n    median_value = df[numerical_column].median()\n    mode_value = df[numerical_column].mode()[0]  # mode() returns a Series, take the first element\n    min_value = df[numerical_column].min()\n    max_value = df[numerical_column].max()\n    range_value = max_value - min_value\n    iqr_value = df[numerical_column].quantile(0.75) - df[numerical_column].quantile(0.25)\n    variance_value = df[numerical_column].var()\n    std_deviation_value = df[numerical_column].std()\n    skewness_value = df[numerical_column].skew()\n    kurtosis_value = df[numerical_column].kurtosis()\n\n    # Display the results\n    print(f'Mean: {mean_value}')\n    print(f'Median: {median_value}')\n    print(f'Mode: {mode_value}')\n    print(f'Minimum: {min_value}')\n    print(f'Maximum: {max_value}')\n    print(f'Range: {range_value}')\n    print(f'IQR: {iqr_value}')\n    print(f'Variance: {variance_value}')\n    print(f'Standard Deviation: {std_deviation_value}')\n    print(f'Skewness: {skewness_value}')\n    print(f'Kurtosis: {kurtosis_value}')","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:53:27.343895Z","iopub.execute_input":"2023-12-02T08:53:27.344321Z","iopub.status.idle":"2023-12-02T08:53:27.351998Z","shell.execute_reply.started":"2023-12-02T08:53:27.344289Z","shell.execute_reply":"2023-12-02T08:53:27.350698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_eda(train_series, 'anglez')","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:53:35.258133Z","iopub.execute_input":"2023-12-02T08:53:35.259106Z","iopub.status.idle":"2023-12-02T08:53:35.318714Z","shell.execute_reply.started":"2023-12-02T08:53:35.259069Z","shell.execute_reply":"2023-12-02T08:53:35.316197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series['anglez'].hist(bins=20, edgecolor='black')","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:53:50.443064Z","iopub.execute_input":"2023-12-02T08:53:50.443467Z","iopub.status.idle":"2023-12-02T08:53:50.721599Z","shell.execute_reply.started":"2023-12-02T08:53:50.443441Z","shell.execute_reply":"2023-12-02T08:53:50.719933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window_size = 60\n\n# Define linear weights for the WMA with a window size of 60\nweights = np.arange(1, window_size + 1)\n\n# Define a custom function to calculate the weighted average\ndef weighted_average(arr):\n    return np.sum(arr * weights[:len(arr)]) / np.sum(weights[:len(arr)])\n\n# Calculate the WMA using the rolling method and apply the custom function\ntrain_series['anglez_wma'] = train_series['anglez'].rolling(window=window_size, min_periods=1).apply(weighted_average, raw=True)\n\n# Calculate the average using the rolling method with the mean aggregation function\ntrain_series['anglez_ave'] = train_series['anglez'].rolling(window=window_size, min_periods=1).mean()\n\n# Calculate the standard deviation using the rolling method with the std aggregation function\n#train_series['anglez_std'] = train_series['anglez'].rolling(window=window_size, min_periods=1).std()\n#train_series['anglez_std'] = train_series['anglez_std'].fillna(0)\n\n# Calculate the second-order derivative for each non-overlapping window\ntrain_series['anglez_der2'] = train_series['anglez'].diff().rolling(window=window_size, min_periods=1).mean().diff()\ntrain_series['anglez_der2'] = train_series['anglez_der2'].fillna(0)\n\ntrain_series","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:54:06.055521Z","iopub.execute_input":"2023-12-02T08:54:06.056175Z","iopub.status.idle":"2023-12-02T08:54:10.044015Z","shell.execute_reply.started":"2023-12-02T08:54:06.056152Z","shell.execute_reply":"2023-12-02T08:54:10.042784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_eda(train_series, 'enmo')","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:55:53.212123Z","iopub.execute_input":"2023-12-02T08:55:53.212564Z","iopub.status.idle":"2023-12-02T08:55:53.250977Z","shell.execute_reply.started":"2023-12-02T08:55:53.212534Z","shell.execute_reply":"2023-12-02T08:55:53.249883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series['enmo'].hist(bins=500, edgecolor='black')","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:55:57.260793Z","iopub.execute_input":"2023-12-02T08:55:57.261777Z","iopub.status.idle":"2023-12-02T08:55:58.053993Z","shell.execute_reply.started":"2023-12-02T08:55:57.261744Z","shell.execute_reply":"2023-12-02T08:55:58.053062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#epsilon = 1e-10\n#train_series['enmo_log'] = np.log(train_series['enmo'] + epsilon)\noffset = 1\ntrain_series['enmo_log'] = np.log(train_series['enmo'] + offset)\ntrain_series['enmo_log'].hist(bins=500, edgecolor='black')","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:56:02.875277Z","iopub.execute_input":"2023-12-02T08:56:02.875681Z","iopub.status.idle":"2023-12-02T08:56:03.902299Z","shell.execute_reply.started":"2023-12-02T08:56:02.875650Z","shell.execute_reply":"2023-12-02T08:56:03.901254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_eda(train_series, 'enmo_log')","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:56:10.513443Z","iopub.execute_input":"2023-12-02T08:56:10.513774Z","iopub.status.idle":"2023-12-02T08:56:10.548642Z","shell.execute_reply.started":"2023-12-02T08:56:10.513747Z","shell.execute_reply":"2023-12-02T08:56:10.546856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some preprocessing\ntrain_events['date'] = train_events['timestamp'].str.split('T', expand=True)[0]\ntrain_events['time'] = train_events['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\ntrain_events['timestamp'] = pd.to_datetime(train_events['date']+' '+train_events['time'])\ntrain_events['hour'] = train_events['timestamp'].dt.hour","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:56:21.336493Z","iopub.execute_input":"2023-12-02T08:56:21.336846Z","iopub.status.idle":"2023-12-02T08:56:21.419277Z","shell.execute_reply.started":"2023-12-02T08:56:21.336793Z","shell.execute_reply":"2023-12-02T08:56:21.418234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:56:25.168331Z","iopub.execute_input":"2023-12-02T08:56:25.168749Z","iopub.status.idle":"2023-12-02T08:56:25.190715Z","shell.execute_reply.started":"2023-12-02T08:56:25.168719Z","shell.execute_reply":"2023-12-02T08:56:25.188865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_and_process_data(series_id, train_series, train_events):\n    # Filter the DataFrame based on the series_id\n    sample_series = train_series[train_series['series_id'] == series_id]\n\n    # Convert timestamp to Pandas datetime object\n    sample_series['date'] = sample_series['timestamp'].str.split('T', expand=True)[0]\n    sample_series['time'] = sample_series['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    sample_series['timestamp'] = pd.to_datetime(sample_series['date']+' '+sample_series['time'])\n    #sample_series['timestamp'] = pd.to_datetime(sample_series['timestamp'])\n\n    # Filter event data based on the series_id\n    sample_events = train_events[train_events['series_id'] == series_id]\n\n    sample_onset = sample_events.loc[sample_events['event'] == 'onset', 'timestamp'].dropna()\n    sample_wakeup = sample_events.loc[sample_events['event'] == 'wakeup', 'timestamp'].dropna()\n\n    return sample_series, sample_onset, sample_wakeup\n\ndef plot_data_and_events_scatter(sample_series, sample_onset, sample_wakeup, data_column, ylabel, series_id):\n    plt.figure(figsize=(20, 3))\n\n    # Convert timestamps to numerical values for easier comparison\n    numerical_timestamps = sample_series['timestamp'].astype(int) // 10**9  # Convert nanoseconds to seconds\n\n    # Limit x-axis to the first 24 hours\n    plt.xlim(numerical_timestamps.min(), numerical_timestamps.min() + 24 * 60 * 60)\n    #print('numerical_timestamps.min()',numerical_timestamps.min())\n\n    plt.scatter(numerical_timestamps, sample_series[data_column], label=data_column, s=0.1)\n\n    for onset in sample_onset:\n        int_timestamp = int(onset.timestamp())\n        #print('int_timestamp',int_timestamp)\n        plt.axvline(x=int_timestamp, color='r', linestyle='--', label='onset')\n\n    for wakeup in sample_wakeup:\n        int_timestamp = int(wakeup.timestamp())\n        plt.axvline(x=int_timestamp, color='g', linestyle='--', label='wakeup')\n\n    #plt.legend()\n    plt.xlabel('Timestamp')\n    plt.ylabel(ylabel)\n    plt.title(f'{ylabel} over Time with Event Flags - {series_id}')\n    plt.show()\n\ndef plot_series_with_events_scatter(series_id, train_series, train_events):\n    sample_series, sample_onset, sample_wakeup = load_and_process_data(series_id, train_series, train_events)\n\n    # Plot enmo and anglez\n    plot_data_and_events_scatter(sample_series, sample_onset, sample_wakeup, 'enmo', 'ENMO Value', series_id)\n    plot_data_and_events_scatter(sample_series, sample_onset, sample_wakeup, 'enmo_log', 'logENMO Value', series_id)\n    plot_data_and_events_scatter(sample_series, sample_onset, sample_wakeup, 'anglez', 'anglez Value', series_id)\n    plot_data_and_events_scatter(sample_series, sample_onset, sample_wakeup, 'anglez_wma', 'anglez_wma Value', series_id)\n    plot_data_and_events_scatter(sample_series, sample_onset, sample_wakeup, 'anglez_ave', 'anglez_ave Value', series_id)\n    plot_data_and_events_scatter(sample_series, sample_onset, sample_wakeup, 'anglez_der2', 'anglez_der2 Value', series_id)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:56:59.252491Z","iopub.execute_input":"2023-12-02T08:56:59.252951Z","iopub.status.idle":"2023-12-02T08:56:59.266599Z","shell.execute_reply.started":"2023-12-02T08:56:59.252920Z","shell.execute_reply":"2023-12-02T08:56:59.265324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series_with_events_scatter('038441c925bb', train_series, train_events)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:57:13.886121Z","iopub.execute_input":"2023-12-02T08:57:13.886748Z","iopub.status.idle":"2023-12-02T08:57:18.818937Z","shell.execute_reply.started":"2023-12-02T08:57:13.886714Z","shell.execute_reply":"2023-12-02T08:57:18.817636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_eda(train_series, 'anglez_der2')","metadata":{"execution":{"iopub.status.busy":"2023-12-02T08:57:08.367103Z","iopub.execute_input":"2023-12-02T08:57:08.367438Z","iopub.status.idle":"2023-12-02T08:57:08.427400Z","shell.execute_reply.started":"2023-12-02T08:57:08.367416Z","shell.execute_reply":"2023-12-02T08:57:08.426549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series_with_events_scatter('038441c925bb', train_series, train_events)","metadata":{"execution":{"iopub.status.busy":"2023-12-01T20:14:54.932127Z","iopub.execute_input":"2023-12-01T20:14:54.932606Z","iopub.status.idle":"2023-12-01T20:15:26.495117Z","shell.execute_reply.started":"2023-12-01T20:14:54.932569Z","shell.execute_reply":"2023-12-01T20:15:26.494124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series_with_events('038441c925bb', train_series, train_events)","metadata":{"execution":{"iopub.status.busy":"2023-12-01T17:07:13.453816Z","iopub.execute_input":"2023-12-01T17:07:13.454296Z","iopub.status.idle":"2023-12-01T17:08:11.236344Z","shell.execute_reply.started":"2023-12-01T17:07:13.454260Z","shell.execute_reply":"2023-12-01T17:08:11.235216Z"},"trusted":true},"execution_count":null,"outputs":[]}]}