{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-07T04:36:58.361241Z","iopub.execute_input":"2023-09-07T04:36:58.361580Z","iopub.status.idle":"2023-09-07T04:36:58.695495Z","shell.execute_reply.started":"2023-09-07T04:36:58.361547Z","shell.execute_reply":"2023-09-07T04:36:58.693810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Load the training data\ntrain_data = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n\n# Display the first few rows to understand the data structure\nprint(train_data.head())\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:36:58.696958Z","iopub.execute_input":"2023-09-07T04:36:58.697290Z","iopub.status.idle":"2023-09-07T04:37:51.575860Z","shell.execute_reply.started":"2023-09-07T04:36:58.697270Z","shell.execute_reply":"2023-09-07T04:37:51.574540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:37:51.578436Z","iopub.execute_input":"2023-09-07T04:37:51.578780Z","iopub.status.idle":"2023-09-07T04:37:51.598684Z","shell.execute_reply.started":"2023-09-07T04:37:51.578754Z","shell.execute_reply":"2023-09-07T04:37:51.597246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the sleep event logs\nevent_logs = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n\n# Display the first few rows to understand the structure of event logs\nprint(event_logs.head())\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:37:51.601009Z","iopub.execute_input":"2023-09-07T04:37:51.601302Z","iopub.status.idle":"2023-09-07T04:37:51.645271Z","shell.execute_reply.started":"2023-09-07T04:37:51.601279Z","shell.execute_reply":"2023-09-07T04:37:51.644034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()\nevent_logs.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:37:51.646769Z","iopub.execute_input":"2023-09-07T04:37:51.647037Z","iopub.status.idle":"2023-09-07T04:37:51.665067Z","shell.execute_reply.started":"2023-09-07T04:37:51.647011Z","shell.execute_reply":"2023-09-07T04:37:51.663763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values = train_data.isnull().sum()\nprint(missing_values)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:37:51.666852Z","iopub.execute_input":"2023-09-07T04:37:51.667174Z","iopub.status.idle":"2023-09-07T04:37:59.378565Z","shell.execute_reply.started":"2023-09-07T04:37:51.667146Z","shell.execute_reply":"2023-09-07T04:37:59.377112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Examine Data for Noise/Outliers","metadata":{}},{"cell_type":"code","source":"import random\nimport matplotlib.pyplot as plt\n\n\n# Sample a subset of the data for visualization (adjust the sample size as needed)\nsample_size = 1000\nsampled_data = train_data.sample(sample_size)\n\n# Visualize anglez over time for the sampled data\nplt.figure(figsize=(12, 6))\nplt.plot(sampled_data['timestamp'], sampled_data['anglez'], label='anglez')\nplt.xlabel('Timestamp')\nplt.ylabel('anglez')\nplt.title('Anglez Over Time (Sampled Data)')\nplt.legend()\nplt.grid(True)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:38:28.431662Z","iopub.execute_input":"2023-09-07T04:38:28.432029Z","iopub.status.idle":"2023-09-07T04:38:40.861064Z","shell.execute_reply.started":"2023-09-07T04:38:28.432002Z","shell.execute_reply":"2023-09-07T04:38:40.859960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculate moving averages over a window of data points to smooth out fluctuations. Moving averages can help remove high-frequency noise from the data.","metadata":{}},{"cell_type":"code","source":"# Compute moving averages for anglez and enmo\nwindow_size = 10  # Adjust the window size as needed\ntrain_data['anglez_ma'] = train_data['anglez'].rolling(window=window_size).mean()\ntrain_data['enmo_ma'] = train_data['enmo'].rolling(window=window_size).mean()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:38:46.833574Z","iopub.execute_input":"2023-09-07T04:38:46.834426Z","iopub.status.idle":"2023-09-07T04:38:51.442129Z","shell.execute_reply.started":"2023-09-07T04:38:46.834395Z","shell.execute_reply":"2023-09-07T04:38:51.440629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.signal import savgol_filter\n\n# Apply Savitzky-Golay filter to anglez and enmo\ntrain_data['anglez_filtered'] = savgol_filter(train_data['anglez'], window_length=5, polyorder=3)\ntrain_data['enmo_filtered'] = savgol_filter(train_data['enmo'], window_length=5, polyorder=3)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:38:51.444582Z","iopub.execute_input":"2023-09-07T04:38:51.444939Z","iopub.status.idle":"2023-09-07T04:38:54.430741Z","shell.execute_reply.started":"2023-09-07T04:38:51.444911Z","shell.execute_reply":"2023-09-07T04:38:54.429466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import stats\n\n# Compute Z-scores for anglez and enmo\nz_scores_anglez = stats.zscore(train_data['anglez'])\nz_scores_enmo = stats.zscore(train_data['enmo'])\n\n# Threshold for outlier detection (e.g., z-score > 3)\noutlier_threshold = 3\n\n# Remove rows with outliers\ntrain_data_no_outliers = train_data[(abs(z_scores_anglez) <= outlier_threshold) & (abs(z_scores_enmo) <= outlier_threshold)]\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:38:54.432114Z","iopub.execute_input":"2023-09-07T04:38:54.432430Z","iopub.status.idle":"2023-09-07T04:39:14.847975Z","shell.execute_reply.started":"2023-09-07T04:38:54.432404Z","shell.execute_reply":"2023-09-07T04:39:14.845554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}