{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-22T02:44:42.265464Z","iopub.execute_input":"2023-10-22T02:44:42.266637Z","iopub.status.idle":"2023-10-22T02:44:42.706834Z","shell.execute_reply.started":"2023-10-22T02:44:42.266539Z","shell.execute_reply":"2023-10-22T02:44:42.705680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport warnings \nwarnings.filterwarnings('ignore')\nimport os\nfrom matplotlib import font_manager","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:44:42.709187Z","iopub.execute_input":"2023-10-22T02:44:42.709670Z","iopub.status.idle":"2023-10-22T02:44:42.716027Z","shell.execute_reply.started":"2023-10-22T02:44:42.709640Z","shell.execute_reply":"2023-10-22T02:44:42.715186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_train_series = '/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet'\nfile_sample_submission = '/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv'\nfile_train_events = '/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv'\nfile_test_series = '/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet'","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:44:42.717825Z","iopub.execute_input":"2023-10-22T02:44:42.718602Z","iopub.status.idle":"2023-10-22T02:44:42.731833Z","shell.execute_reply.started":"2023-10-22T02:44:42.718567Z","shell.execute_reply":"2023-10-22T02:44:42.730342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =============================================================================\n# \n# temporal moving \"mean\" and \"std\"\n#\n# =============================================================================\n\ndef MA_mean(OneInterval,steps):\n    the_mean=OneInterval.rolling(steps).mean()\n    the_mean.dropna(inplace=True)\n    return the_mean\n\ndef MA_std(OneInterval,steps):\n    the_std=OneInterval.rolling(steps).std()\n    the_std.dropna(inplace=True)\n    return the_std","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:44:42.734880Z","iopub.execute_input":"2023-10-22T02:44:42.735378Z","iopub.status.idle":"2023-10-22T02:44:42.745458Z","shell.execute_reply.started":"2023-10-22T02:44:42.735348Z","shell.execute_reply":"2023-10-22T02:44:42.744082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =============================================================================\n# \n# plot original and processed data\n#\n# =============================================================================\n\ndef PlotProcessedData(df_x,df_y,the_id,close_fig):\n        \n    df_y['flag'] = 0\n    \n    for j in range (0,len(df_answer)):\n        if (df_y['event'].iloc[j] == 'onset'): \n            df_y['flag'].iloc[j] = 1\n        if (df_y['event'].iloc[j] == 'wakeup'): \n            df_y['flag'].iloc[j] = 0\n    \n    df1['y'] = 0\n    length_of_answer = len(df_answer)\n    \n    for j in range (0, length_of_answer-1):   \n        r1 = df_y.step.iloc[j].astype(int)\n        r2 = df_y.step.iloc[j+1].astype(int)\n        df_x['y'].iloc[r1:r2] = df_y.flag.iloc[j]\n    \n    # =============================================================================\n    # \n    # plot the data and answer\n    # \n    # =============================================================================\n    \n    fig = plt.figure(figsize=(12,8))\n    fig.suptitle(the_id, fontsize=20)\n    \n    ax1 = plt.subplot2grid((3,1), (0,0), colspan=1)\n    ax2 = plt.subplot2grid((3,1), (1,0), colspan=1)\n    ax3 = plt.subplot2grid((3,1), (2,0), colspan=1)\n    \n    df_x.plot(x='step',y='anglez', color = 'b',ax=ax1)\n    df_x.plot(x='step',y='y', color = 'r',ax=ax2)\n    df_x.plot(x='step',y='enmo', color = 'g',ax=ax3)\n    \n    outputfile = '../Plot/' + the_id + '_1.png' \n    fig.savefig(outputfile)\n    \n    if (close_fig == 'Y'):\n        plt.close(fig)\n        \n    # =============================================================================\n    # \n    # process the data\n    # \n    # =============================================================================\n    \n    moving_average = 300\n    \n    df2 = df_x.copy()\n        \n    df2_anglez = df2['anglez'].copy()\n    df2['anglez_std']  = MA_std(df2_anglez,moving_average)\n    \n    df2_anglez_std = df2['anglez_std'].copy()\n    df2['anglez_std_std']  = MA_std(df2_anglez_std,moving_average)\n     \n    df2_enmo = df2['enmo'].copy()\n    df2['enmo_ma']  = MA_mean(df2_enmo,moving_average)\n    \n    df2['combine'] = df2['anglez_std_std']  - 100*df2['enmo_ma']   \n    df2['combine_thre']  = df2['combine'] > 0\n    df2['combine_thre']  = df2['combine_thre'].astype(int)\n    df2_combine_thre = df2['combine_thre'].copy()\n    df2['combine_thre_ma']  = 100*MA_mean(df2_combine_thre,1000)\n        \n    # =============================================================================\n    # \n    # plot processed data and answer\n    # \n    # =============================================================================\n    \n    fig = plt.figure(figsize=(12,8))\n    fig.suptitle(the_id, fontsize=20)\n    \n    ax1 = plt.subplot2grid((6,1), (0,0), colspan=1)\n    ax2 = plt.subplot2grid((6,1), (1,0), colspan=1)\n    ax3 = plt.subplot2grid((6,1), (2,0), colspan=1)\n    ax4 = plt.subplot2grid((6,1), (3,0), colspan=1)\n    ax5 = plt.subplot2grid((6,1), (4,0), colspan=1)\n    ax6 = plt.subplot2grid((6,1), (5,0), colspan=1)\n\n    df2.plot(x='step',y='anglez_std_std', color = 'b',ax=ax1)\n    df2.plot(x='step',y='enmo_ma', color = 'g',ax=ax2)\n    df2.plot(x='step',y='combine', color = 'k',ax=ax3)\n    df2.plot(x='step',y='combine_thre', color = 'c',ax=ax4)\n    df2.plot(x='step',y='combine_thre_ma', color = 'r',ax=ax5)\n    df2.plot(x='step',y='y', color = 'r',ax=ax6)\n    \n    outputfile = '../Plot/' + the_id + '_2.png' \n    fig.savefig(outputfile)\n    if (close_fig == 'Y'):\n        plt.close(fig)\n        ","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:44:42.747866Z","iopub.execute_input":"2023-10-22T02:44:42.748500Z","iopub.status.idle":"2023-10-22T02:44:42.775330Z","shell.execute_reply.started":"2023-10-22T02:44:42.748326Z","shell.execute_reply":"2023-10-22T02:44:42.773663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =============================================================================\n# \n# read the input data and answer CSV files\n# \n# =============================================================================\n\ndf_all = pd.read_parquet(file_train_series)\nprint(df_all.shape)\n\ndf_answer_all = pd.read_csv(file_train_events)\nprint(df_answer_all.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:44:42.777456Z","iopub.execute_input":"2023-10-22T02:44:42.777804Z","iopub.status.idle":"2023-10-22T02:45:59.787165Z","shell.execute_reply.started":"2023-10-22T02:44:42.777773Z","shell.execute_reply":"2023-10-22T02:45:59.786013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =============================================================================\n# \n# create a directory for savng plots\n# \n# =============================================================================\n\nFileLocation_data = '../Plot' \nprint(' ')\ntry:\n    os.mkdir(FileLocation_data)\nexcept OSError:\n    print (\"Creation of the directory %s failed\" % FileLocation_data)\nelse:\n    print (\"Successfully created the directory %s \" % FileLocation_data)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:45:59.788771Z","iopub.execute_input":"2023-10-22T02:45:59.789956Z","iopub.status.idle":"2023-10-22T02:45:59.798900Z","shell.execute_reply.started":"2023-10-22T02:45:59.789906Z","shell.execute_reply":"2023-10-22T02:45:59.797337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =============================================================================\n# \n# select one data set, and insert answer to the input data\n# good case: series_id = 'f6d2cc003183'\n#\n# =============================================================================\n\nstr_id = 'f6d2cc003183'\n\ndf1 = df_all[ df_all['series_id'] == str_id]\ndf_answer = df_answer_all[ df_answer_all['series_id'] == str_id]    \ndf_answer = df_answer.dropna()\n    \nPlotProcessedData(df1,df_answer,str_id,'N')    ","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:45:59.800830Z","iopub.execute_input":"2023-10-22T02:45:59.801239Z","iopub.status.idle":"2023-10-22T02:46:43.463210Z","shell.execute_reply.started":"2023-10-22T02:45:59.801207Z","shell.execute_reply":"2023-10-22T02:46:43.461663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =============================================================================\n# \n# select one data set, and insert answer to the input data\n# another case: series_id = '87a6cbb7c4ed'\n#\n# =============================================================================\n\nstr_id = '87a6cbb7c4ed'\n\ndf1 = df_all[ df_all['series_id'] == str_id]\ndf_answer = df_answer_all[ df_answer_all['series_id'] == str_id]    \ndf_answer = df_answer.dropna()\n    \nPlotProcessedData(df1,df_answer,str_id,'N')    ","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:46:43.464738Z","iopub.execute_input":"2023-10-22T02:46:43.465116Z","iopub.status.idle":"2023-10-22T02:47:18.989911Z","shell.execute_reply.started":"2023-10-22T02:46:43.465082Z","shell.execute_reply":"2023-10-22T02:47:18.988728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =============================================================================\n# \n# plot processed data for all series_id\n# \n# =============================================================================\n\nlist_series = df_all['series_id'].unique()\n\n#for idx in range (0,len(list_series)):\nfor idx in range (0,5):\n        \n    print(idx, list_series[idx])\n    str_id = list_series[idx]\n    df1 = df_all[ df_all['series_id'] == str_id]\n    df_answer = df_answer_all[ df_answer_all['series_id'] == str_id]    \n    df_answer = df_answer.dropna()\n    PlotProcessedData(df1,df_answer,str_id, 'Y')    \n","metadata":{"execution":{"iopub.status.busy":"2023-10-22T02:47:18.993931Z","iopub.execute_input":"2023-10-22T02:47:18.994422Z","iopub.status.idle":"2023-10-22T02:49:54.763752Z","shell.execute_reply.started":"2023-10-22T02:47:18.994379Z","shell.execute_reply":"2023-10-22T02:49:54.762363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}