{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Step 1. Import Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-07T09:20:49.316035Z","iopub.execute_input":"2023-09-07T09:20:49.316584Z","iopub.status.idle":"2023-09-07T09:20:49.322725Z","shell.execute_reply.started":"2023-09-07T09:20:49.316542Z","shell.execute_reply":"2023-09-07T09:20:49.321553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\n# Set the random seed for reproducibility\nrandom.seed(42)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:20:49.329050Z","iopub.execute_input":"2023-09-07T09:20:49.329774Z","iopub.status.idle":"2023-09-07T09:20:49.335853Z","shell.execute_reply.started":"2023-09-07T09:20:49.329727Z","shell.execute_reply":"2023-09-07T09:20:49.334641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 2: Load the Data","metadata":{}},{"cell_type":"code","source":"# Load the training data\ntrain = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n\n# Load the sleep event logs\nevents = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n\n# Load the test data (if available)\ntest = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:20:49.338140Z","iopub.execute_input":"2023-09-07T09:20:49.339138Z","iopub.status.idle":"2023-09-07T09:21:56.805665Z","shell.execute_reply.started":"2023-09-07T09:20:49.339093Z","shell.execute_reply":"2023-09-07T09:21:56.804551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sample 10% of the data\nsample_fraction = 0.005\ntrain_series = train.sample(frac=sample_fraction)\ntrain_events = events.sample(frac=sample_fraction)\ntest_series = test.sample(frac=sample_fraction)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:21:56.807595Z","iopub.execute_input":"2023-09-07T09:21:56.807934Z","iopub.status.idle":"2023-09-07T09:22:05.644962Z","shell.execute_reply.started":"2023-09-07T09:21:56.807906Z","shell.execute_reply":"2023-09-07T09:22:05.643885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 3: Data Exploration\n\nLet's start by exploring the training data:","metadata":{}},{"cell_type":"markdown","source":"## 3.1. Overview of the Data","metadata":{}},{"cell_type":"code","source":"# Display the first few rows of the training series data\nprint(train_series.head())\nprint('='*90)\n\n# Display basic information about the data\nprint(train_series.info())\nprint('='*90)\n\n# Display basic statistics for numerical columns\nprint(train_series.describe())\nprint('='*90)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:22:05.646241Z","iopub.execute_input":"2023-09-07T09:22:05.646588Z","iopub.status.idle":"2023-09-07T09:22:06.009535Z","shell.execute_reply.started":"2023-09-07T09:22:05.646560Z","shell.execute_reply":"2023-09-07T09:22:06.008239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the first few rows of the training series data\nprint(test_series.head())\nprint('='*90)\n\n# Display basic information about the data\nprint(test_series.info())\nprint('='*90)\n\n# Display basic statistics for numerical columns\nprint(test_series.describe())\nprint('='*90)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:23:21.196332Z","iopub.execute_input":"2023-09-07T09:23:21.196853Z","iopub.status.idle":"2023-09-07T09:23:21.228289Z","shell.execute_reply.started":"2023-09-07T09:23:21.196815Z","shell.execute_reply":"2023-09-07T09:23:21.226949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2. Visualization\nLet's create some visualizations to better understand the data:","metadata":{}},{"cell_type":"code","source":"# Plot a histogram of 'enmo' values\nplt.figure(figsize=(8, 6))\nsns.histplot(train_series['enmo'], bins=50, kde=True)\nplt.xlabel('ENMO Values')\nplt.ylabel('Frequency')\nplt.title('Distribution of ENMO Values')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:22:06.012176Z","iopub.execute_input":"2023-09-07T09:22:06.012529Z","iopub.status.idle":"2023-09-07T09:22:09.568803Z","shell.execute_reply.started":"2023-09-07T09:22:06.012498Z","shell.execute_reply":"2023-09-07T09:22:09.567558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.3. Sleep Event Logs\nNow, let's explore the sleep event logs:","metadata":{}},{"cell_type":"code","source":"# Display the first few rows of the sleep event logs\nprint(train_events.head(5))","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:22:09.570245Z","iopub.execute_input":"2023-09-07T09:22:09.570640Z","iopub.status.idle":"2023-09-07T09:22:09.580202Z","shell.execute_reply.started":"2023-09-07T09:22:09.570608Z","shell.execute_reply":"2023-09-07T09:22:09.578917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# Check the distribution of event types\nsns.countplot(data=train_events, x='event')\nplt.xlabel('Event Type')\nplt.ylabel('Count')\nplt.title('Distribution of Sleep Events')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:22:09.582048Z","iopub.execute_input":"2023-09-07T09:22:09.582564Z","iopub.status.idle":"2023-09-07T09:22:09.856075Z","shell.execute_reply.started":"2023-09-07T09:22:09.582520Z","shell.execute_reply":"2023-09-07T09:22:09.854843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 4: Data Preprocessing\n## Step 4.1: Data Cleaning","metadata":{}},{"cell_type":"code","source":"# Check for missing values\nmissing_values = train_series.isnull().sum()\nprint(\"Missing Values:\\n\", missing_values)\n\n# Remove duplicates\ntrain_series = train_series.drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:22:09.857846Z","iopub.execute_input":"2023-09-07T09:22:09.858658Z","iopub.status.idle":"2023-09-07T09:22:11.073257Z","shell.execute_reply.started":"2023-09-07T09:22:09.858612Z","shell.execute_reply":"2023-09-07T09:22:11.071849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 4.2: Data Transformation","metadata":{}},{"cell_type":"code","source":"# Convert timestamp column to datetime\ntrain_series['timestamp'] = pd.to_datetime(train_series['timestamp'])\n\n# Encode categorical variables (if applicable)\n# Example: One-hot encoding for 'event' column\ntrain_events_encoded = pd.get_dummies(train_events, columns=['event'], prefix=['event'])\n\n# Feature scaling (if needed)\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ntrain_series['enmo_scaled'] = scaler.fit_transform(train_series[['enmo']])","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:22:11.074842Z","iopub.execute_input":"2023-09-07T09:22:11.075963Z","iopub.status.idle":"2023-09-07T09:22:16.202513Z","shell.execute_reply.started":"2023-09-07T09:22:11.075926Z","shell.execute_reply":"2023-09-07T09:22:16.201391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 5: Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Example: Create rolling mean of 'enmo' over a window of 5 time steps\ntrain_series['enmo_rolling_mean'] = train_series['enmo'].rolling(window=5).mean()\n\n# Select relevant features (if needed)\n# Example: Select specific columns as features\nselected_features = train_series[['enmo', 'anglez']]","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:22:16.204257Z","iopub.execute_input":"2023-09-07T09:22:16.205346Z","iopub.status.idle":"2023-09-07T09:22:16.231137Z","shell.execute_reply.started":"2023-09-07T09:22:16.205311Z","shell.execute_reply":"2023-09-07T09:22:16.230192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 6: Data Splitting","metadata":{}},{"cell_type":"code","source":"# Assuming you have a target variable 'sleep_event' in your dataset\ntarget_variable = train_series['enmo']\n\n# Select relevant features (excluding the target variable)\nselected_features = train_series.drop(columns=['enmo'])","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:27:25.499840Z","iopub.execute_input":"2023-09-07T09:27:25.500923Z","iopub.status.idle":"2023-09-07T09:27:25.561363Z","shell.execute_reply.started":"2023-09-07T09:27:25.500877Z","shell.execute_reply":"2023-09-07T09:27:25.560169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(selected_features, target_variable, test_size=0.2, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:27:27.987116Z","iopub.execute_input":"2023-09-07T09:27:27.987550Z","iopub.status.idle":"2023-09-07T09:27:28.262375Z","shell.execute_reply.started":"2023-09-07T09:27:27.987519Z","shell.execute_reply":"2023-09-07T09:27:28.261113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Step 8: Data Normalization","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Select only numeric columns for scaling\nnumeric_columns = X_train.select_dtypes(include=['number'])\n\n# Initialize the StandardScaler\nscaler = StandardScaler()\n\n# Fit and transform the scaler on the training data\nX_train_scaled = scaler.fit_transform(numeric_columns)\n\n# Transform the validation data using the same scaler\nX_val_scaled = scaler.transform(X_val[numeric_columns.columns])","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:32:20.475894Z","iopub.execute_input":"2023-09-07T09:32:20.476419Z","iopub.status.idle":"2023-09-07T09:32:20.579608Z","shell.execute_reply.started":"2023-09-07T09:32:20.476382Z","shell.execute_reply":"2023-09-07T09:32:20.578481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}