{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Cleaning the Signals Using Fast Fourier Transformations\n\nFirst of all, much of the initial data cleaning and plotting code was adapted from: https://www.kaggle.com/code/veronikasarikaya/it-s-a-wonderful-wonderful-sleep-eda . Please give @veronikasarikaya's notebook an upvote!\n\nWe note that `enmo` and `anglez` appear to be very noisy, and so this is an attempt to explore the use of Fast Fourier Transformations to clean the signal.\n\nHopefully this will translate in better model learning down the road.\n\nIf you find this helpful, ***please upvote!***\n\nUpdates (10/21/2023):\n* Using fixed datasets converting to UTC to avoid errors with daylights savings\n* Added parameters when performing denoising to better see what is being filtered out and the effect on the denoised data\n\nUpdates (10/19/2023):\n* Assume that frequency is 0.2 (1 step every 5 seconds)\n* Filter based on percentile - default to 99.7%","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-10-21T21:47:38.010173Z","iopub.execute_input":"2023-10-21T21:47:38.010638Z","iopub.status.idle":"2023-10-21T21:47:40.543732Z","shell.execute_reply.started":"2023-10-21T21:47:38.010605Z","shell.execute_reply":"2023-10-21T21:47:40.542459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Memory reduction for train_series\n#The following function has been taken from the notebook\n# https://www.kaggle.com/code/renatoreggiani/zzzs-feat-eng-ideas-60-memory-reduction?scriptVersionId=143987308\n\nfrom pandas.api.types import is_datetime64_ns_dtype\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndef reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n\n    df['series_id'] = df['series_id'].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    \n    return df","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-10-21T21:47:40.545979Z","iopub.execute_input":"2023-10-21T21:47:40.546686Z","iopub.status.idle":"2023-10-21T21:47:40.568441Z","shell.execute_reply.started":"2023-10-21T21:47:40.546652Z","shell.execute_reply":"2023-10-21T21:47:40.567200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## create a list 'dt_transforms' with transformations \n## to be applied to a timestamp column in the data\n#dt_transforms = [\n#    pl.col('timestamp').str.to_datetime(), \n#    (pl.col('timestamp').str.to_datetime().dt.date()).alias('date'), \n#    pl.col('timestamp').str.to_datetime().dt.time().alias('time')\n#]\n\n#trn_series = (\n#    pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n#    .with_columns((dt_transforms))\n#    .collect()\n#    .to_pandas()\n#)\n\n#trn_events = (\n#    pl.scan_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n#    .with_columns((dt_transforms))\n#    .collect()\n#    .to_pandas()\n#)\n\n## download test_series data\n#tst_series = (\n#    pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n#    .with_columns((dt_transforms))\n#    .collect()\n#    .to_pandas()\n#)\n\n#reduce_mem_usage(trn_series)\n#reduce_mem_usage(tst_series)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-10-21T21:47:40.570537Z","iopub.execute_input":"2023-10-21T21:47:40.570999Z","iopub.status.idle":"2023-10-21T21:47:40.596980Z","shell.execute_reply.started":"2023-10-21T21:47:40.570959Z","shell.execute_reply":"2023-10-21T21:47:40.595678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Load preprocessed data - this is public. If you find it helpful, please upvote!\ntst_series = pd.read_parquet(\"/kaggle/input/simple-utc-conversion-using-polars/tst_series.parquet\")\ntrn_series = pd.read_parquet(\"/kaggle/input/simple-utc-conversion-using-polars/trn_series.parquet\")\ntrn_events = pd.read_parquet(\"/kaggle/input/simple-utc-conversion-using-polars/trn_events.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:47:40.599644Z","iopub.execute_input":"2023-10-21T21:47:40.600024Z","iopub.status.idle":"2023-10-21T21:48:13.672133Z","shell.execute_reply.started":"2023-10-21T21:47:40.599990Z","shell.execute_reply":"2023-10-21T21:48:13.670890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Normalize anglez\ntst_series['anglez_norm'] = tst_series['anglez'] / 90.\ntrn_series['anglez_norm'] = trn_series['anglez'] / 90.","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:48:13.674236Z","iopub.execute_input":"2023-10-21T21:48:13.675151Z","iopub.status.idle":"2023-10-21T21:48:14.484461Z","shell.execute_reply.started":"2023-10-21T21:48:13.675074Z","shell.execute_reply":"2023-10-21T21:48:14.483289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_sleep(series_df: pd.DataFrame, events_df: pd.DataFrame, series_id: str, series_dates: list[str]):\n    \n    s1 = series_df[series_df['series_id']==series_id]\n    s1_d1 = s1[s1['date'].isin(series_dates)]\n    s1_d1_lbls = events_df[(events_df.series_id==series_id)&(events_df.date.isin(series_dates))]\n    \n    color_enmo = '#F8C471'\n    color_anglez = '#7FB3D5'\n\n    color_onset = '#196F3D'\n    color_wakeup = '#943126'\n\n    fig, ax1 = plt.subplots(figsize=(20, 6))\n    ax2 = ax1.twinx()\n\n    ax1.plot(s1_d1.step, s1_d1.anglez, color= color_anglez)\n    ax2.plot(s1_d1.step, s1_d1.enmo, color= color_enmo)\n\n    # Apply labels\n    for r in s1_d1_lbls[['event', 'step']].itertuples():\n        ax1.axvline(x=r.step, \n                    ymin=-90,\n                    ymax=90,\n                    linewidth=2.,\n                    linestyle='--',\n                    color=color_onset if r.event == 'onset' else color_wakeup)\n\n    ax1.set_xlabel('Step')\n    ax1.set_ylabel('Anglez', color = color_anglez, fontsize = 14)\n    ax2.set_ylabel('ENMO', color = color_enmo, fontsize = 14)\n\n\n    fig.suptitle(f'ENMO & ANGLEZ values for {series_id} from {s1_d1.date.min().strftime(\"%D\")} to {s1_d1.date.max().strftime(\"%D\")}', fontsize = 20)\n    fig.autofmt_xdate()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-10-21T21:48:14.485958Z","iopub.execute_input":"2023-10-21T21:48:14.486465Z","iopub.status.idle":"2023-10-21T21:48:14.500797Z","shell.execute_reply.started":"2023-10-21T21:48:14.486433Z","shell.execute_reply":"2023-10-21T21:48:14.499381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_series(s, s_lbls, column='enmo'):\n    \"\"\"\n    Plot the selected column and the accompanying events\n    \"\"\"\n    assert column in s.columns, \"Invalid column.\"\n    \n    ymin = s[column].min()\n    ymax = s[column].max()\n    \n    color = '#F8C471'\n\n    color_onset = '#196F3D'\n    color_wakeup = '#943126'\n\n    fig, ax = plt.subplots(figsize=(20, 6))\n\n    ax.plot(s.step, s[column], color=color)\n\n    # Apply labels\n    for r in s_lbls[['event', 'step']].itertuples():\n        ax.axvline(x=r.step,\n                   ymin=min(0, ymin),\n                   ymax=max(1, ymax),\n                   linewidth=2.,\n                   linestyle='--',\n                   color=color_onset if r.event == 'onset' else color_wakeup)\n\n    ax.set_xlabel('Step')\n    ax.set_ylabel(column, color=color, fontsize = 14)\n    fig.suptitle(f'{column} for {s.series_id.iloc[0]}', fontsize = 20)\n    fig.autofmt_xdate()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:48:14.503220Z","iopub.execute_input":"2023-10-21T21:48:14.504073Z","iopub.status.idle":"2023-10-21T21:48:14.519923Z","shell.execute_reply.started":"2023-10-21T21:48:14.504030Z","shell.execute_reply":"2023-10-21T21:48:14.518670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fft_signal_clean(signal, filter_percent=99.7, f=0.2, show_plot=False, y_min=0, y_max=1, L_min_idx=0, L_max_idx=-1):\n    \"\"\"\n    Denoising signal using Fast Fourier Transformation\n    Adapted from: https://www.youtube.com/watch?v=s2K1JfNR7Sc\n    \n    All errors are mine.\n    \"\"\"\n    t = np.arange(0,1,f)\n    \n    # Get signal length\n    n = len(signal)\n    \n    # Compute the FFT\n    fhat = np.fft.fft(signal, n)\n    \n    # Compute Power Spectrum\n    PSD = fhat * np.conj(fhat) / n\n    \n    # Create x-axis of frequencies\n    freq = (1 / (f * n)) * np.arange(n)\n    \n    PSD_filter = np.percentile(PSD, filter_percent)\n    \n    # For graphing\n    if show_plot:\n        L = np.arange(1, np.floor(n/2), dtype='int')\n        fig,axs = plt.subplots(1,1)\n        plt.sca(axs)\n        plt.plot(freq[L], PSD[L], color='c', linewidth=2, label='Noisy')\n        axs.axhline(\n            y=PSD_filter,\n            color='red'\n        )\n        ## Limits on y-axis\n        plt.ylim(y_min, y_max)\n        \n        ## Limits on x-axis\n        plt.xlim(freq[L[L_min_idx]]-0.1, freq[L[L_max_idx]]+0.1)\n        plt.legend()\n        plt.title(\"PSD\")\n        plt.show()\n    \n    indices = PSD >= PSD_filter\n    \n    # For graphing\n    if show_plot:\n        PSD_clean = PSD * indices\n    \n    fhat = fhat * indices\n    \n    # Compute inverse FFT\n    signal_clean = np.fft.ifft(fhat)\n    \n    return signal_clean.real","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:57:32.451105Z","iopub.execute_input":"2023-10-21T21:57:32.451522Z","iopub.status.idle":"2023-10-21T21:57:32.465822Z","shell.execute_reply.started":"2023-10-21T21:57:32.451489Z","shell.execute_reply":"2023-10-21T21:57:32.464416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Number of night events\ntrn_events.groupby('series_id').night.last().plot.hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:48:34.481972Z","iopub.execute_input":"2023-10-21T21:48:34.482542Z","iopub.status.idle":"2023-10-21T21:48:34.922037Z","shell.execute_reply.started":"2023-10-21T21:48:34.482497Z","shell.execute_reply":"2023-10-21T21:48:34.920665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"empty_nights = [\n    '0f9e60a8e56d',\n     '390b487231ce',\n     '2fc653ca75c7',\n     'c7b1283bb7eb',\n     '89c7daa72eee',\n     'e11b9d69f856',\n     'c5d08fc3e040',\n     'a3e59c2ce3f6']\n\n## Number of night events\ntrn_events[trn_events['series_id'].isin(empty_nights)].groupby('series_id').night.last().plot.hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:48:34.924011Z","iopub.execute_input":"2023-10-21T21:48:34.924370Z","iopub.status.idle":"2023-10-21T21:48:35.381939Z","shell.execute_reply.started":"2023-10-21T21:48:34.924340Z","shell.execute_reply":"2023-10-21T21:48:35.380565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Series 038441c925bb","metadata":{}},{"cell_type":"code","source":"series_id = trn_events.series_id[0]\n\ns = trn_series.query(f\"`series_id`=='{series_id}'\")\ns_lbls = trn_events.query(f\"`series_id`=='{series_id}'\")","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:48:39.470510Z","iopub.execute_input":"2023-10-21T21:48:39.471004Z","iopub.status.idle":"2023-10-21T21:48:41.235504Z","shell.execute_reply.started":"2023-10-21T21:48:39.470967Z","shell.execute_reply":"2023-10-21T21:48:41.234165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:48:41.240610Z","iopub.execute_input":"2023-10-21T21:48:41.241012Z","iopub.status.idle":"2023-10-21T21:48:42.002258Z","shell.execute_reply.started":"2023-10-21T21:48:41.240978Z","shell.execute_reply":"2023-10-21T21:48:42.001307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.7, show_plot=True, L_min_idx=0, L_max_idx=50)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:58:20.177079Z","iopub.execute_input":"2023-10-21T21:58:20.177538Z","iopub.status.idle":"2023-10-21T21:58:20.724438Z","shell.execute_reply.started":"2023-10-21T21:58:20.177505Z","shell.execute_reply":"2023-10-21T21:58:20.722951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:58:23.896720Z","iopub.execute_input":"2023-10-21T21:58:23.897372Z","iopub.status.idle":"2023-10-21T21:58:24.485941Z","shell.execute_reply.started":"2023-10-21T21:58:23.897310Z","shell.execute_reply":"2023-10-21T21:58:24.484546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Keeping only the top 0.3%.","metadata":{}},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.9, show_plot=True, L_min_idx=0, L_max_idx=50)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:55:14.328456Z","iopub.execute_input":"2023-10-21T21:55:14.329992Z","iopub.status.idle":"2023-10-21T21:55:14.860345Z","shell.execute_reply.started":"2023-10-21T21:55:14.329921Z","shell.execute_reply":"2023-10-21T21:55:14.859041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Based on the power spectral density plot, we can see that a `power_filter` value of 1 is likely valid.","metadata":{}},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:54:03.156338Z","iopub.execute_input":"2023-10-21T21:54:03.156826Z","iopub.status.idle":"2023-10-21T21:54:03.668701Z","shell.execute_reply.started":"2023-10-21T21:54:03.156791Z","shell.execute_reply":"2023-10-21T21:54:03.667380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.95, show_plot=True, L_min_idx=0, L_max_idx=50)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:55:24.765421Z","iopub.execute_input":"2023-10-21T21:55:24.765899Z","iopub.status.idle":"2023-10-21T21:55:25.308727Z","shell.execute_reply.started":"2023-10-21T21:55:24.765864Z","shell.execute_reply":"2023-10-21T21:55:25.307155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Look at how much cleaner the plot is!","metadata":{}},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:55:30.655243Z","iopub.execute_input":"2023-10-21T21:55:30.655697Z","iopub.status.idle":"2023-10-21T21:55:31.228557Z","shell.execute_reply.started":"2023-10-21T21:55:30.655666Z","shell.execute_reply":"2023-10-21T21:55:31.227296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.99, show_plot=True, y_min=0, y_max=2, L_min_idx=0, L_max_idx=50)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:57:43.906343Z","iopub.execute_input":"2023-10-21T21:57:43.907356Z","iopub.status.idle":"2023-10-21T21:57:44.393433Z","shell.execute_reply.started":"2023-10-21T21:57:43.907285Z","shell.execute_reply":"2023-10-21T21:57:44.392259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T21:57:51.426313Z","iopub.execute_input":"2023-10-21T21:57:51.426785Z","iopub.status.idle":"2023-10-21T21:57:51.999269Z","shell.execute_reply.started":"2023-10-21T21:57:51.426751Z","shell.execute_reply":"2023-10-21T21:57:51.998164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Perhaps this is **too** clean.","metadata":{}},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_norm')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:36.971633Z","iopub.execute_input":"2023-10-21T18:27:36.972778Z","iopub.status.idle":"2023-10-21T18:27:38.905282Z","shell.execute_reply.started":"2023-10-21T18:27:36.972738Z","shell.execute_reply":"2023-10-21T18:27:38.903592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Normalized but still very noisy.","metadata":{}},{"cell_type":"code","source":"s['anglez_clean'] = fft_signal_clean(s.anglez_norm.values, filter_percent=99.99, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:38.907505Z","iopub.execute_input":"2023-10-21T18:27:38.908041Z","iopub.status.idle":"2023-10-21T18:27:38.988956Z","shell.execute_reply.started":"2023-10-21T18:27:38.907991Z","shell.execute_reply":"2023-10-21T18:27:38.987906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Again, based on the PSD plot, a `power_filter` of 100 seems to be valid.","metadata":{}},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:38.990706Z","iopub.execute_input":"2023-10-21T18:27:38.991225Z","iopub.status.idle":"2023-10-21T18:27:39.573300Z","shell.execute_reply.started":"2023-10-21T18:27:38.991160Z","shell.execute_reply":"2023-10-21T18:27:39.571948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Much cleaner!","metadata":{}},{"cell_type":"markdown","source":"# Series 038441c925bb","metadata":{}},{"cell_type":"code","source":"series_id = trn_events.series_id[1]\n\ns = trn_series.query(f\"`series_id`=='{series_id}'\")\ns_lbls = trn_events.query(f\"`series_id`=='{series_id}'\")","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:39.575051Z","iopub.execute_input":"2023-10-21T18:27:39.575538Z","iopub.status.idle":"2023-10-21T18:27:41.357363Z","shell.execute_reply.started":"2023-10-21T18:27:39.575476Z","shell.execute_reply":"2023-10-21T18:27:41.356088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:41.359028Z","iopub.execute_input":"2023-10-21T18:27:41.359451Z","iopub.status.idle":"2023-10-21T18:27:42.146921Z","shell.execute_reply.started":"2023-10-21T18:27:41.359417Z","shell.execute_reply":"2023-10-21T18:27:42.145436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.7, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:42.148848Z","iopub.execute_input":"2023-10-21T18:27:42.150090Z","iopub.status.idle":"2023-10-21T18:27:42.245994Z","shell.execute_reply.started":"2023-10-21T18:27:42.150045Z","shell.execute_reply":"2023-10-21T18:27:42.244713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:42.247632Z","iopub.execute_input":"2023-10-21T18:27:42.248016Z","iopub.status.idle":"2023-10-21T18:27:43.028908Z","shell.execute_reply.started":"2023-10-21T18:27:42.247984Z","shell.execute_reply":"2023-10-21T18:27:43.027245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:43.030685Z","iopub.execute_input":"2023-10-21T18:27:43.031114Z","iopub.status.idle":"2023-10-21T18:27:44.721754Z","shell.execute_reply.started":"2023-10-21T18:27:43.031073Z","shell.execute_reply":"2023-10-21T18:27:44.720414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['anglez_clean'] = fft_signal_clean(s.anglez_norm.values, filter_percent=99.99, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:44.723823Z","iopub.execute_input":"2023-10-21T18:27:44.724769Z","iopub.status.idle":"2023-10-21T18:27:44.800780Z","shell.execute_reply.started":"2023-10-21T18:27:44.724728Z","shell.execute_reply":"2023-10-21T18:27:44.799483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:44.802771Z","iopub.execute_input":"2023-10-21T18:27:44.803276Z","iopub.status.idle":"2023-10-21T18:27:45.287646Z","shell.execute_reply.started":"2023-10-21T18:27:44.803230Z","shell.execute_reply":"2023-10-21T18:27:45.286611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Series 390b487231ce","metadata":{}},{"cell_type":"code","source":"series_id = \"390b487231ce\"\n\ns = trn_series.query(f\"`series_id`=='{series_id}'\")\ns_lbls = trn_events.query(f\"`series_id`=='{series_id}'\")","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:31:03.897172Z","iopub.execute_input":"2023-10-21T18:31:03.897655Z","iopub.status.idle":"2023-10-21T18:31:05.727159Z","shell.execute_reply.started":"2023-10-21T18:31:03.897620Z","shell.execute_reply":"2023-10-21T18:31:05.725718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:31:09.012088Z","iopub.execute_input":"2023-10-21T18:31:09.013523Z","iopub.status.idle":"2023-10-21T18:31:09.687651Z","shell.execute_reply.started":"2023-10-21T18:31:09.013459Z","shell.execute_reply":"2023-10-21T18:31:09.686293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.7, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:31:16.558328Z","iopub.execute_input":"2023-10-21T18:31:16.558789Z","iopub.status.idle":"2023-10-21T18:31:16.680420Z","shell.execute_reply.started":"2023-10-21T18:31:16.558755Z","shell.execute_reply":"2023-10-21T18:31:16.678856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:31:21.524656Z","iopub.execute_input":"2023-10-21T18:31:21.525140Z","iopub.status.idle":"2023-10-21T18:31:22.186588Z","shell.execute_reply.started":"2023-10-21T18:31:21.525103Z","shell.execute_reply":"2023-10-21T18:31:22.185162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.99, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:31:43.023737Z","iopub.execute_input":"2023-10-21T18:31:43.024993Z","iopub.status.idle":"2023-10-21T18:31:43.214783Z","shell.execute_reply.started":"2023-10-21T18:31:43.024920Z","shell.execute_reply":"2023-10-21T18:31:43.213489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:31:50.229868Z","iopub.execute_input":"2023-10-21T18:31:50.230385Z","iopub.status.idle":"2023-10-21T18:31:50.895009Z","shell.execute_reply.started":"2023-10-21T18:31:50.230346Z","shell.execute_reply":"2023-10-21T18:31:50.894010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_norm')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:32:22.882495Z","iopub.execute_input":"2023-10-21T18:32:22.882937Z","iopub.status.idle":"2023-10-21T18:32:24.748272Z","shell.execute_reply.started":"2023-10-21T18:32:22.882904Z","shell.execute_reply":"2023-10-21T18:32:24.746896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['anglez_clean'] = fft_signal_clean(s.anglez_norm.values, filter_percent=99.7, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:32:33.205634Z","iopub.execute_input":"2023-10-21T18:32:33.206129Z","iopub.status.idle":"2023-10-21T18:32:33.303315Z","shell.execute_reply.started":"2023-10-21T18:32:33.206091Z","shell.execute_reply":"2023-10-21T18:32:33.301894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:32:41.856778Z","iopub.execute_input":"2023-10-21T18:32:41.857266Z","iopub.status.idle":"2023-10-21T18:32:42.531120Z","shell.execute_reply.started":"2023-10-21T18:32:41.857230Z","shell.execute_reply":"2023-10-21T18:32:42.529566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['anglez_clean'] = fft_signal_clean(s.anglez_norm.values, filter_percent=99.99, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:33:12.057862Z","iopub.execute_input":"2023-10-21T18:33:12.058381Z","iopub.status.idle":"2023-10-21T18:33:12.162309Z","shell.execute_reply.started":"2023-10-21T18:33:12.058339Z","shell.execute_reply":"2023-10-21T18:33:12.160897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:33:12.481987Z","iopub.execute_input":"2023-10-21T18:33:12.482539Z","iopub.status.idle":"2023-10-21T18:33:13.023993Z","shell.execute_reply.started":"2023-10-21T18:33:12.482498Z","shell.execute_reply":"2023-10-21T18:33:13.022614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Series 0f9e60a8e56d\n\n* This is an interesting series because it contains no annotations","metadata":{}},{"cell_type":"code","source":"series_id = \"0f9e60a8e56d\"\n\ns = trn_series.query(f\"`series_id`=='{series_id}'\")\ns_lbls = trn_events.query(f\"`series_id`=='{series_id}'\")","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:45.288963Z","iopub.execute_input":"2023-10-21T18:27:45.289567Z","iopub.status.idle":"2023-10-21T18:27:47.048509Z","shell.execute_reply.started":"2023-10-21T18:27:45.289534Z","shell.execute_reply":"2023-10-21T18:27:47.047036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:47.050340Z","iopub.execute_input":"2023-10-21T18:27:47.050825Z","iopub.status.idle":"2023-10-21T18:27:47.620971Z","shell.execute_reply.started":"2023-10-21T18:27:47.050779Z","shell.execute_reply":"2023-10-21T18:27:47.619496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.7, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:47.622533Z","iopub.execute_input":"2023-10-21T18:27:47.622911Z","iopub.status.idle":"2023-10-21T18:27:47.674301Z","shell.execute_reply.started":"2023-10-21T18:27:47.622877Z","shell.execute_reply":"2023-10-21T18:27:47.672794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:47.676639Z","iopub.execute_input":"2023-10-21T18:27:47.677210Z","iopub.status.idle":"2023-10-21T18:27:48.450625Z","shell.execute_reply.started":"2023-10-21T18:27:47.677136Z","shell.execute_reply":"2023-10-21T18:27:48.449268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['enmo_clean'] = fft_signal_clean(s.enmo.values, filter_percent=99.99, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:48.452613Z","iopub.execute_input":"2023-10-21T18:27:48.453103Z","iopub.status.idle":"2023-10-21T18:27:48.510111Z","shell.execute_reply.started":"2023-10-21T18:27:48.453059Z","shell.execute_reply":"2023-10-21T18:27:48.508696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'enmo_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:48.516707Z","iopub.execute_input":"2023-10-21T18:27:48.517158Z","iopub.status.idle":"2023-10-21T18:27:48.995491Z","shell.execute_reply.started":"2023-10-21T18:27:48.517121Z","shell.execute_reply":"2023-10-21T18:27:48.994241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_norm')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:48.997149Z","iopub.execute_input":"2023-10-21T18:27:48.997560Z","iopub.status.idle":"2023-10-21T18:27:50.116924Z","shell.execute_reply.started":"2023-10-21T18:27:48.997527Z","shell.execute_reply":"2023-10-21T18:27:50.115368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['anglez_clean'] = fft_signal_clean(s.anglez_norm.values, filter_percent=99.7, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:50.119312Z","iopub.execute_input":"2023-10-21T18:27:50.119847Z","iopub.status.idle":"2023-10-21T18:27:50.170667Z","shell.execute_reply.started":"2023-10-21T18:27:50.119788Z","shell.execute_reply":"2023-10-21T18:27:50.169548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:50.172560Z","iopub.execute_input":"2023-10-21T18:27:50.173225Z","iopub.status.idle":"2023-10-21T18:27:50.715589Z","shell.execute_reply.started":"2023-10-21T18:27:50.173190Z","shell.execute_reply":"2023-10-21T18:27:50.714263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s['anglez_clean'] = fft_signal_clean(s.anglez_norm.values, filter_percent=99.99, show_plot=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:50.717689Z","iopub.execute_input":"2023-10-21T18:27:50.718170Z","iopub.status.idle":"2023-10-21T18:27:50.769285Z","shell.execute_reply.started":"2023-10-21T18:27:50.718126Z","shell.execute_reply":"2023-10-21T18:27:50.768116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_series(s, s_lbls, 'anglez_clean')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T18:27:50.771313Z","iopub.execute_input":"2023-10-21T18:27:50.771834Z","iopub.status.idle":"2023-10-21T18:27:51.283532Z","shell.execute_reply.started":"2023-10-21T18:27:50.771789Z","shell.execute_reply":"2023-10-21T18:27:51.282247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For both denoised `enmo` and `anglez` we can see that there is a very regular pattern that appears. As suggested in https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/438438 perhaps these are patterns that can be learned and further eliminated from the signals once they appear.\n\nMy hypothesis is that when the user takes the device off, there is some internal process inherent to the device that accounts for these signals.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}