{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Load the training data\ntrain_series = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n\n# Load the test data\ntest_series = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n\n# Load the event annotations for training data\ntrain_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n\n# Display the first few rows to understand the data structure\nprint(\"Train Series Data:\")\nprint(train_series.head())\nprint(\"Test Series Data:\")\nprint(test_series.head())\nprint(\"Train Events Data:\")\nprint(train_events.head())\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-07T09:38:17.205887Z","iopub.execute_input":"2023-09-07T09:38:17.206295Z","iopub.status.idle":"2023-09-07T09:39:08.854958Z","shell.execute_reply.started":"2023-09-07T09:38:17.206264Z","shell.execute_reply":"2023-09-07T09:39:08.853894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:22:21.829683Z","iopub.execute_input":"2023-09-07T09:22:21.830029Z","iopub.status.idle":"2023-09-07T09:22:21.837083Z","shell.execute_reply.started":"2023-09-07T09:22:21.830003Z","shell.execute_reply":"2023-09-07T09:22:21.836005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"NOTE: Considering that signal data is large, the code for creating the plots needs a lot of time to compile.","metadata":{}},{"cell_type":"code","source":"# Generate summary statistics for the train dataset\ntrain_series.describe()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:49:32.73896Z","iopub.execute_input":"2023-09-07T07:49:32.739443Z","iopub.status.idle":"2023-09-07T07:49:43.86387Z","shell.execute_reply.started":"2023-09-07T07:49:32.739409Z","shell.execute_reply":"2023-09-07T07:49:43.862499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for missing data in the train and test datasets\ntrain_series.isnull().sum(), test_series.isnull().sum()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:49:43.865863Z","iopub.execute_input":"2023-09-07T07:49:43.866346Z","iopub.status.idle":"2023-09-07T07:50:10.650332Z","shell.execute_reply.started":"2023-09-07T07:49:43.866314Z","shell.execute_reply":"2023-09-07T07:50:10.649068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Filter data for a few subjects\nsubject_ids = train_series['series_id'].unique()[:3]  # Change this to select different subjects\n\nfor subject in subject_ids:\n    subject_data = train_series[train_series['series_id'] == subject]\n    \n    plt.figure(figsize=(14, 6))\n    \n    plt.subplot(1, 2, 1)\n    plt.plot(subject_data['timestamp'], subject_data['anglez'])\n    plt.title(f'Subject {subject} - AngleZ over Time')\n    plt.xlabel('Timestamp')\n    plt.ylabel('AngleZ')\n    \n    plt.subplot(1, 2, 2)\n    plt.plot(subject_data['timestamp'], subject_data['enmo'])\n    plt.title(f'Subject {subject} - ENMO over Time')\n    plt.xlabel('Timestamp')\n    plt.ylabel('ENMO')\n    \n    plt.tight_layout()\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-06T11:59:07.320227Z","iopub.execute_input":"2023-09-06T11:59:07.320643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample 10% of the data (you can adjust the fraction as needed)\nsampled_data = train_series.sample(frac=0.1)\n\nplt.figure(figsize=(14, 6))\n\nplt.subplot(1, 2, 1)\nsns.histplot(sampled_data['anglez'], bins=50, kde=True)\nplt.title('Distribution of AngleZ')\nplt.xlabel('AngleZ')\nplt.ylabel('Frequency')\n\nplt.subplot(1, 2, 2)\nsns.histplot(sampled_data['enmo'], bins=50, kde=True)\nplt.title('Distribution of ENMO')\nplt.xlabel('ENMO')\nplt.ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:26:41.079788Z","iopub.execute_input":"2023-09-07T07:26:41.080304Z","iopub.status.idle":"2023-09-07T07:29:04.317924Z","shell.execute_reply.started":"2023-09-07T07:26:41.080259Z","shell.execute_reply":"2023-09-07T07:29:04.316873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from statsmodels.graphics.tsaplots import plot_acf, plot_pacf\n\n# Filter data for a sample subject\nsample_subject = train_series['series_id'].unique()[0]  # You can change this to select a different subject\nsample_data = train_series[train_series['series_id'] == sample_subject]\n\n# Autocorrelation for anglez\nplot_acf(sample_data['anglez'], lags=50)\nplt.title('Autocorrelation for AngleZ')\nplt.xlabel('Lags')\nplt.ylabel('Correlation')\nplt.show()\n\n# Partial Autocorrelation for anglez\nplot_pacf(sample_data['anglez'], lags=50)\nplt.title('Partial Autocorrelation for AngleZ')\nplt.xlabel('Lags')\nplt.ylabel('Correlation')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:33:21.172579Z","iopub.execute_input":"2023-09-07T07:33:21.172987Z","iopub.status.idle":"2023-09-07T07:34:34.189102Z","shell.execute_reply.started":"2023-09-07T07:33:21.172954Z","shell.execute_reply":"2023-09-07T07:34:34.187955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Autocorrelation for enmo\nplot_acf(sample_data['enmo'], lags=50)\nplt.title('Autocorrelation for ENMO')\nplt.xlabel('Lags')\nplt.ylabel('Correlation')\nplt.show()\n\n# Partial Autocorrelation for enmo\nplot_pacf(sample_data['enmo'], lags=50)\nplt.title('Partial Autocorrelation for ENMO')\nplt.xlabel('Lags')\nplt.ylabel('Correlation')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:34:59.659967Z","iopub.execute_input":"2023-09-07T07:34:59.660413Z","iopub.status.idle":"2023-09-07T07:35:38.273512Z","shell.execute_reply.started":"2023-09-07T07:34:59.660376Z","shell.execute_reply":"2023-09-07T07:35:38.272579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.close('all')","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:50:10.652456Z","iopub.execute_input":"2023-09-07T07:50:10.65286Z","iopub.status.idle":"2023-09-07T07:50:10.752456Z","shell.execute_reply.started":"2023-09-07T07:50:10.652828Z","shell.execute_reply":"2023-09-07T07:50:10.751444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:36:35.164264Z","iopub.execute_input":"2023-09-07T07:36:35.164699Z","iopub.status.idle":"2023-09-07T07:36:35.186063Z","shell.execute_reply.started":"2023-09-07T07:36:35.164662Z","shell.execute_reply":"2023-09-07T07:36:35.18488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events[\"event\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:36:55.753307Z","iopub.execute_input":"2023-09-07T07:36:55.753711Z","iopub.status.idle":"2023-09-07T07:36:55.770376Z","shell.execute_reply.started":"2023-09-07T07:36:55.753672Z","shell.execute_reply":"2023-09-07T07:36:55.769291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events[\"series_id\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:37:16.01958Z","iopub.execute_input":"2023-09-07T07:37:16.019999Z","iopub.status.idle":"2023-09-07T07:37:16.032357Z","shell.execute_reply.started":"2023-09-07T07:37:16.019968Z","shell.execute_reply":"2023-09-07T07:37:16.031042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge the train_series and train_events on series_id and timestamp\nmerged_data = pd.merge(train_series, train_events, on=['series_id', 'timestamp'], how='left')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T07:50:10.754116Z","iopub.execute_input":"2023-09-07T07:50:10.755265Z","iopub.status.idle":"2023-09-07T07:53:38.569257Z","shell.execute_reply.started":"2023-09-07T07:50:10.755209Z","shell.execute_reply":"2023-09-07T07:53:38.568077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T08:02:04.688792Z","iopub.execute_input":"2023-09-07T08:02:04.689627Z","iopub.status.idle":"2023-09-07T08:02:04.710972Z","shell.execute_reply.started":"2023-09-07T08:02:04.689579Z","shell.execute_reply":"2023-09-07T08:02:04.709647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-07T08:47:31.653842Z","iopub.execute_input":"2023-09-07T08:47:31.654316Z","iopub.status.idle":"2023-09-07T08:47:31.662609Z","shell.execute_reply.started":"2023-09-07T08:47:31.654279Z","shell.execute_reply":"2023-09-07T08:47:31.661303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_data[\"event\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T08:03:15.293098Z","iopub.execute_input":"2023-09-07T08:03:15.293578Z","iopub.status.idle":"2023-09-07T08:03:19.927079Z","shell.execute_reply.started":"2023-09-07T08:03:15.293514Z","shell.execute_reply":"2023-09-07T08:03:19.925693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check for duplicates in 'series_id', 'night', and 'event' columns\nduplicates = merged_data.duplicated(subset=['series_id', 'night', 'event'], keep=False)\n\n# Display rows where duplicates are found\nmerged_data[duplicates]\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T08:45:09.512403Z","iopub.execute_input":"2023-09-07T08:45:09.512865Z","iopub.status.idle":"2023-09-07T08:45:57.153961Z","shell.execute_reply.started":"2023-09-07T08:45:09.51283Z","shell.execute_reply":"2023-09-07T08:45:57.152984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run Python's garbage collection\nimport gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:08:44.814091Z","iopub.execute_input":"2023-09-07T09:08:44.814586Z","iopub.status.idle":"2023-09-07T09:08:45.083485Z","shell.execute_reply.started":"2023-09-07T09:08:44.814551Z","shell.execute_reply":"2023-09-07T09:08:45.082595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}