{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-08T09:06:00.835895Z","iopub.execute_input":"2023-12-08T09:06:00.836982Z","iopub.status.idle":"2023-12-08T09:06:02.61861Z","shell.execute_reply.started":"2023-12-08T09:06:00.836938Z","shell.execute_reply":"2023-12-08T09:06:02.617608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:02.62061Z","iopub.execute_input":"2023-12-08T09:06:02.621441Z","iopub.status.idle":"2023-12-08T09:06:02.629256Z","shell.execute_reply.started":"2023-12-08T09:06:02.621401Z","shell.execute_reply":"2023-12-08T09:06:02.62832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:02.630358Z","iopub.execute_input":"2023-12-08T09:06:02.630652Z","iopub.status.idle":"2023-12-08T09:06:17.35388Z","shell.execute_reply.started":"2023-12-08T09:06:02.630626Z","shell.execute_reply":"2023-12-08T09:06:17.352601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\ntqdm.pandas()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:17.357074Z","iopub.execute_input":"2023-12-08T09:06:17.357429Z","iopub.status.idle":"2023-12-08T09:06:17.383821Z","shell.execute_reply.started":"2023-12-08T09:06:17.3574Z","shell.execute_reply":"2023-12-08T09:06:17.382801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Understanding","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")\ntrain_data\n\n# parquet_file = pq.ParquetFile(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")\n# train_data = parquet_file.read()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:40.867037Z","iopub.execute_input":"2023-12-08T09:06:40.867968Z","iopub.status.idle":"2023-12-08T09:07:20.64653Z","shell.execute_reply.started":"2023-12-08T09:06:40.867933Z","shell.execute_reply":"2023-12-08T09:07:20.64499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\n# train_events","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:26.271709Z","iopub.execute_input":"2023-12-08T09:07:26.272099Z","iopub.status.idle":"2023-12-08T09:07:26.323158Z","shell.execute_reply.started":"2023-12-08T09:07:26.272063Z","shell.execute_reply":"2023-12-08T09:07:26.322212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv\")\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:30.09216Z","iopub.execute_input":"2023-12-08T09:07:30.092938Z","iopub.status.idle":"2023-12-08T09:07:30.111665Z","shell.execute_reply.started":"2023-12-08T09:07:30.092899Z","shell.execute_reply":"2023-12-08T09:07:30.110677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events[\"series_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:30.55294Z","iopub.execute_input":"2023-12-08T09:07:30.553275Z","iopub.status.idle":"2023-12-08T09:07:30.562877Z","shell.execute_reply.started":"2023-12-08T09:07:30.553246Z","shell.execute_reply":"2023-12-08T09:07:30.561965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pre_process(data):\n    ts = data.copy()\n    ts[\"timestamp\"] = pd.to_datetime(ts[\"timestamp\"], utc=True)\n#     ts.set_index(\"timestamp\", inplace=True)\n    ts[\"step\"] = ts[\"step\"].astype(float)\n    \n#     ts = ts.resample(\"1min\").max()\n#     ts.reset_index(inplace=True)\n#     print(ts.isna().sum())\n#     ts.set_index(\"timestamp\", inplace=True)\n    return ts \n\nprocessed_ts = train_data.iloc[:10000,:].\\\ngroupby(\"series_id\").progress_apply(pre_process).reset_index(level=0,drop=True)\n\n# processed_ts.groupby(\"series_id\").apply(plot_timeseries)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:40.643225Z","iopub.execute_input":"2023-12-08T09:07:40.644171Z","iopub.status.idle":"2023-12-08T09:07:40.827376Z","shell.execute_reply.started":"2023-12-08T09:07:40.644132Z","shell.execute_reply":"2023-12-08T09:07:40.826278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_ts","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:41.278959Z","iopub.execute_input":"2023-12-08T09:07:41.279357Z","iopub.status.idle":"2023-12-08T09:07:41.296528Z","shell.execute_reply.started":"2023-12-08T09:07:41.279304Z","shell.execute_reply":"2023-12-08T09:07:41.295501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:41.765756Z","iopub.execute_input":"2023-12-08T09:07:41.766863Z","iopub.status.idle":"2023-12-08T09:07:41.782519Z","shell.execute_reply.started":"2023-12-08T09:07:41.766812Z","shell.execute_reply":"2023-12-08T09:07:41.781567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_df = train_events[[\"series_id\",\"event\",\"step\"]]\nevent_df","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:44.321276Z","iopub.execute_input":"2023-12-08T09:07:44.321721Z","iopub.status.idle":"2023-12-08T09:07:44.33964Z","shell.execute_reply.started":"2023-12-08T09:07:44.321685Z","shell.execute_reply":"2023-12-08T09:07:44.338656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Analysis Base Table\nmerged_ts = pd.merge(processed_ts, event_df, on=[\"step\", \"series_id\"],how='left')\nmerged_ts","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:44.958437Z","iopub.execute_input":"2023-12-08T09:07:44.959435Z","iopub.status.idle":"2023-12-08T09:07:44.995186Z","shell.execute_reply.started":"2023-12-08T09:07:44.959395Z","shell.execute_reply":"2023-12-08T09:07:44.994266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_ts[merged_ts[\"step\"] == 4992.0]","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:46.553008Z","iopub.execute_input":"2023-12-08T09:07:46.553433Z","iopub.status.idle":"2023-12-08T09:07:46.567626Z","shell.execute_reply.started":"2023-12-08T09:07:46.553398Z","shell.execute_reply":"2023-12-08T09:07:46.566603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation\n","metadata":{}},{"cell_type":"code","source":"# Pre-process the data\ndef pre_process(data):\n    ts = data.copy()\n    ts[\"timestamp\"] = pd.to_datetime(ts[\"timestamp\"], utc=True)\n    ts[\"step\"] = ts[\"step\"].astype(float)\n    return ts ","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:50.541408Z","iopub.execute_input":"2023-12-08T09:07:50.542292Z","iopub.status.idle":"2023-12-08T09:07:50.548819Z","shell.execute_reply.started":"2023-12-08T09:07:50.542247Z","shell.execute_reply":"2023-12-08T09:07:50.547532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_ts = train_data.groupby(\"series_id\").head(100).groupby(\"series_id\").apply(pre_process).reset_index(level=0, drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:07:51.863042Z","iopub.execute_input":"2023-12-08T09:07:51.863793Z","iopub.status.idle":"2023-12-08T09:08:10.385574Z","shell.execute_reply.started":"2023-12-08T09:07:51.863755Z","shell.execute_reply":"2023-12-08T09:08:10.384543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge data with events\nevent_df = train_events[[\"series_id\", \"event\", \"step\"]]\nmerged_ts = pd.merge(processed_ts, event_df, on=[\"step\", \"series_id\"], how='left')","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:08:19.74797Z","iopub.execute_input":"2023-12-08T09:08:19.748395Z","iopub.status.idle":"2023-12-08T09:08:19.770214Z","shell.execute_reply.started":"2023-12-08T09:08:19.74836Z","shell.execute_reply":"2023-12-08T09:08:19.76925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create labels\ndef create_labels(data):\n    ts = data.copy()\n    ts[\"sleep\"] = np.nan\n    ts.loc[ts[\"event\"] == \"onset\", \"sleep\"] = 1\n    ts.loc[ts[\"event\"] == \"wakeup\", \"sleep\"] = 0\n    ts[\"sleep\"] = ts[\"sleep\"].ffill().fillna(value=0)\n    \n    if ts[\"sleep\"].mean() < 0.2:\n        return pd.DataFrame()\n    \n    return ts\n\nlabelled_ts = merged_ts.groupby(\"series_id\").progress_apply(create_labels).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:08:21.927418Z","iopub.execute_input":"2023-12-08T09:08:21.928242Z","iopub.status.idle":"2023-12-08T09:08:22.505084Z","shell.execute_reply.started":"2023-12-08T09:08:21.928201Z","shell.execute_reply":"2023-12-08T09:08:22.504219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resample to 1 minute\ndef resample_fn(ts):\n    ts_resampled = ts.copy()\n    ts_resampled.set_index(\"timestamp\", inplace=True)\n    ts_resampled = ts_resampled.resample(\"1min\").max(numeric_only=True)\n    ts_resampled.reset_index(inplace=True)\n    return ts_resampled","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:20:13.127211Z","iopub.execute_input":"2023-12-08T09:20:13.128603Z","iopub.status.idle":"2023-12-08T09:20:13.134647Z","shell.execute_reply.started":"2023-12-08T09:20:13.128561Z","shell.execute_reply":"2023-12-08T09:20:13.133425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resampled_ts = labelled_ts.groupby(\"series_id\", as_index=False).progress_apply(resample_fn).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:25:06.973107Z","iopub.execute_input":"2023-12-08T09:25:06.973516Z","iopub.status.idle":"2023-12-08T09:25:07.074778Z","shell.execute_reply.started":"2023-12-08T09:25:06.973482Z","shell.execute_reply":"2023-12-08T09:25:07.073374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add time features\ndef add_time_features(ts):\n    ts[\"hour\"] = ts[\"timestamp\"].dt.hour\n    return ts","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:20:43.623599Z","iopub.execute_input":"2023-12-08T09:20:43.624467Z","iopub.status.idle":"2023-12-08T09:20:43.629105Z","shell.execute_reply.started":"2023-12-08T09:20:43.624433Z","shell.execute_reply":"2023-12-08T09:20:43.628092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:31:33.265043Z","iopub.execute_input":"2023-12-08T09:31:33.265483Z","iopub.status.idle":"2023-12-08T09:31:48.15663Z","shell.execute_reply.started":"2023-12-08T09:31:33.265449Z","shell.execute_reply":"2023-12-08T09:31:48.155455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data for modeling\ntrain_cols = [col for col in df_prepared.columns if col not in [\"series_id\", \"timestamp\", \"step\", \"sleep\"]]\nX = df_prepared[train_cols]\ny = df_prepared[\"sleep\"]","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:31:54.321013Z","iopub.execute_input":"2023-12-08T09:31:54.321894Z","iopub.status.idle":"2023-12-08T09:31:54.721709Z","shell.execute_reply.started":"2023-12-08T09:31:54.321853Z","shell.execute_reply":"2023-12-08T09:31:54.720295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data into train and test sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.179848Z","iopub.status.idle":"2023-12-08T09:06:31.180288Z","shell.execute_reply.started":"2023-12-08T09:06:31.180061Z","shell.execute_reply":"2023-12-08T09:06:31.180082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Standardize the data\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.181392Z","iopub.status.idle":"2023-12-08T09:06:31.181903Z","shell.execute_reply.started":"2023-12-08T09:06:31.181646Z","shell.execute_reply":"2023-12-08T09:06:31.181676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build the neural network model\nmodel = Sequential()\nmodel.add(Dense(64, input_dim=X_train_scaled.shape[1], activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(32, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.188259Z","iopub.status.idle":"2023-12-08T09:06:31.188973Z","shell.execute_reply.started":"2023-12-08T09:06:31.188719Z","shell.execute_reply":"2023-12-08T09:06:31.188741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.1903Z","iopub.status.idle":"2023-12-08T09:06:31.191071Z","shell.execute_reply.started":"2023-12-08T09:06:31.190795Z","shell.execute_reply":"2023-12-08T09:06:31.190817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nmodel.fit(X_train_scaled, y_train, epochs=10, batch_size=32, validation_split=0.2, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.192395Z","iopub.status.idle":"2023-12-08T09:06:31.193146Z","shell.execute_reply.started":"2023-12-08T09:06:31.192865Z","shell.execute_reply":"2023-12-08T09:06:31.192889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"# Evaluate the model\ny_pred = model.predict_classes(X_test_scaled)\naccuracy = accuracy_score(y_test, y_pred)\nprint(f\"Accuracy: {accuracy:.4f}\")\n\nclassification_report_result = classification_report(y_test, y_pred)\nprint(\"Classification Report:\\n\", classification_report_result)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.194515Z","iopub.status.idle":"2023-12-08T09:06:31.19527Z","shell.execute_reply.started":"2023-12-08T09:06:31.194966Z","shell.execute_reply":"2023-12-08T09:06:31.195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_labels(data):\n    ts = data.copy()\n    \n#     ts[\"sleep\"] = 0\n    ts[\"sleep\"] = np.nan\n    \n    ts.loc[ts[\"event\"]==\"onset\", \"sleep\"] = 1\n    ts.loc[ts[\"event\"]==\"wakeup\", \"sleep\"] = 0\n            \n    ts[\"sleep\"] = ts[\"sleep\"].ffill().fillna(value=0)\n    \n    if ts[\"sleep\"].mean() < 0.2:\n        return pd.DataFrame()\n    \n    print(ts[\"sleep\"].mean())\n        \n#     plt.plot(ts[\"sleep\"])\n#     plt.show()\n                \n#     first_index = ts[\"event\"].first_valid_index()\n        \n#     fill_value = 1 - ts.loc[first_index, \"sleep\"]\n    \n#     ts[\"sleep\"] = ts[\"sleep\"].fillna(value=fill_value)\n\n    return ts\n\nlabelled_ts = merged_ts.groupby(\"series_id\").\\\nprogress_apply(create_labels).reset_index(drop=True)\n\nlabelled_ts","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.196698Z","iopub.status.idle":"2023-12-08T09:06:31.197453Z","shell.execute_reply.started":"2023-12-08T09:06:31.197153Z","shell.execute_reply":"2023-12-08T09:06:31.197176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resample to 1 minute\n\ndef resample_fn(ts):\n    ts_resampled = ts.copy()\n    ts_resampled.set_index(\"timestamp\", inplace=True)\n    ts_resampled = ts_resampled.resample(\"1min\").max(numeric_only=True)\n\n    ts_resampled.reset_index(inplace=True)\n    return ts_resampled\n\nresampled_ts = labelled_ts.groupby(\"series_id\").\\\nprogress_apply(resample_fn).reset_index().drop([\"level_1\"], axis=1)\n\nresampled_ts","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.198839Z","iopub.status.idle":"2023-12-08T09:06:31.199606Z","shell.execute_reply.started":"2023-12-08T09:06:31.199316Z","shell.execute_reply":"2023-12-08T09:06:31.199352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_time_features(ts):\n    ts[\"hour\"] = ts[\"timestamp\"].dt.hour\n#     ts[\"minute\"] = ts[\"timestamp\"].dt.minute\n    \n    return ts\n\ndf_prepared = add_time_features(resampled_ts.copy())\ndf_prepared","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.200936Z","iopub.status.idle":"2023-12-08T09:06:31.201316Z","shell.execute_reply.started":"2023-12-08T09:06:31.201134Z","shell.execute_reply":"2023-12-08T09:06:31.201152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cols = [col for col in df_prepared.columns if col not in [\"series_id\", \"timestamp\",\"step\",\"sleep\"]]\nX = df_prepared[train_cols]\ny = df_prepared[\"sleep\"]","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.2029Z","iopub.status.idle":"2023-12-08T09:06:31.203618Z","shell.execute_reply.started":"2023-12-08T09:06:31.203341Z","shell.execute_reply":"2023-12-08T09:06:31.203375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import accuracy_score, classification_report\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, LSTM, Dropout\n\n# Load the data\ntrain_data = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")\ntrain_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\n\n# Pre-process the data\ndef pre_process(data):\n    ts = data.copy()\n    ts[\"timestamp\"] = pd.to_datetime(ts[\"timestamp\"], utc=True)\n    ts[\"step\"] = ts[\"step\"].astype(float)\n    return ts \n\nprocessed_ts = train_data.groupby(\"series_id\").progress_apply(pre_process).reset_index(level=0, drop=True)\n\n# Merge data with events\nevent_df = train_events[[\"series_id\", \"event\", \"step\"]]\nmerged_ts = pd.merge(processed_ts, event_df, on=[\"step\", \"series_id\"], how='left')\n\n# Create labels\ndef create_labels(data):\n    ts = data.copy()\n    ts[\"sleep\"] = np.nan\n    ts.loc[ts[\"event\"] == \"onset\", \"sleep\"] = 1\n    ts.loc[ts[\"event\"] == \"wakeup\", \"sleep\"] = 0\n    ts[\"sleep\"] = ts[\"sleep\"].ffill().fillna(value=0)\n    \n    if ts[\"sleep\"].mean() < 0.2:\n        return pd.DataFrame()\n    \n    return ts\n\nlabelled_ts = merged_ts.groupby(\"series_id\").progress_apply(create_labels).reset_index(drop=True)\n\n# Resample to 1 minute\ndef resample_fn(ts):\n    ts_resampled = ts.copy()\n    ts_resampled.set_index(\"timestamp\", inplace=True)\n    ts_resampled = ts_resampled.resample(\"1min\").max(numeric_only=True)\n    ts_resampled.reset_index(inplace=True)\n    return ts_resampled\n\nresampled_ts = labelled_ts.groupby(\"series_id\").progress_apply(resample_fn).reset_index().drop([\"level_1\"], axis=1)\n\n# Add time features\ndef add_time_features(ts):\n    ts[\"hour\"] = ts[\"timestamp\"].dt.hour\n    return ts\n\ndf_prepared = add_time_features(resampled_ts.copy())\n\n# Prepare data for modeling\ntrain_cols = [col for col in df_prepared.columns if col not in [\"series_id\", \"timestamp\", \"step\", \"sleep\"]]\nX = df_prepared[train_cols]\ny = df_prepared[\"sleep\"]\n\n# Split data into train and test sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Standardize the data\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\n\n# Build the neural network model\nmodel = Sequential()\nmodel.add(Dense(64, input_dim=X_train_scaled.shape[1], activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(32, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n\n# Train the model\nmodel.fit(X_train_scaled, y_train, epochs=10, batch_size=32, validation_split=0.2, verbose=1)\n\n# Evaluate the model\ny_pred = model.predict_classes(X_test_scaled)\naccuracy = accuracy_score(y_test, y_pred)\nprint(f\"Accuracy: {accuracy:.4f}\")\n\nclassification_report_result = classification_report(y_test, y_pred)\nprint(\"Classification Report:\\n\", classification_report_result)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.204898Z","iopub.status.idle":"2023-12-08T09:06:31.20562Z","shell.execute_reply.started":"2023-12-08T09:06:31.205361Z","shell.execute_reply":"2023-12-08T09:06:31.205384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\n\nX_strd = scaler.fit_transform(X)\n\nX_strd.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.206896Z","iopub.status.idle":"2023-12-08T09:06:31.207626Z","shell.execute_reply.started":"2023-12-08T09:06:31.207367Z","shell.execute_reply":"2023-12-08T09:06:31.20739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_valid, y_train, y_valid = train_test_split(X_strd, y, train_size=0.75)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.20908Z","iopub.status.idle":"2023-12-08T09:06:31.209829Z","shell.execute_reply.started":"2023-12-08T09:06:31.209536Z","shell.execute_reply":"2023-12-08T09:06:31.209575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, y_train.shape, X_valid.shape, y_valid.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.211257Z","iopub.status.idle":"2023-12-08T09:06:31.212034Z","shell.execute_reply.started":"2023-12-08T09:06:31.21174Z","shell.execute_reply":"2023-12-08T09:06:31.211765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import layers\nmodel = keras.Sequential([\n    layers.BatchNormalization(input_shape=[X_train.shape[1]]),\n    layers.Dense(256, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    layers.Dense(256, activation='relu'), \n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    layers.Dense(1, activation='sigmoid'),\n])","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.213413Z","iopub.status.idle":"2023-12-08T09:06:31.214194Z","shell.execute_reply.started":"2023-12-08T09:06:31.213893Z","shell.execute_reply":"2023-12-08T09:06:31.213921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer='adam',\n    loss='binary_crossentropy',\n    metrics=['binary_accuracy'],\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.215581Z","iopub.status.idle":"2023-12-08T09:06:31.21636Z","shell.execute_reply.started":"2023-12-08T09:06:31.21606Z","shell.execute_reply":"2023-12-08T09:06:31.216085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping = keras.callbacks.EarlyStopping(\n    patience=5,\n    min_delta=0.001,\n    restore_best_weights=True,\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.217742Z","iopub.status.idle":"2023-12-08T09:06:31.218502Z","shell.execute_reply.started":"2023-12-08T09:06:31.218204Z","shell.execute_reply":"2023-12-08T09:06:31.218229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    X_train, y_train,\n    validation_data=(X_valid, y_valid),\n    batch_size=512,\n    epochs=200,\n    callbacks=[early_stopping],\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.219857Z","iopub.status.idle":"2023-12-08T09:06:31.220626Z","shell.execute_reply.started":"2023-12-08T09:06:31.220298Z","shell.execute_reply":"2023-12-08T09:06:31.220349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = model.predict(X_strd)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.221931Z","iopub.status.idle":"2023-12-08T09:06:31.222662Z","shell.execute_reply.started":"2023-12-08T09:06:31.222388Z","shell.execute_reply":"2023-12-08T09:06:31.222414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.mean()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.223975Z","iopub.status.idle":"2023-12-08T09:06:31.224705Z","shell.execute_reply.started":"2023-12-08T09:06:31.224422Z","shell.execute_reply":"2023-12-08T09:06:31.224446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fitted_data = X.copy()\nfitted_data[\"sleep\"] = y\nfitted_data[\"sleep_pred\"] = pred.round()\n\nfitted_data","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.226005Z","iopub.status.idle":"2023-12-08T09:06:31.226757Z","shell.execute_reply.started":"2023-12-08T09:06:31.226448Z","shell.execute_reply":"2023-12-08T09:06:31.226482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = 0\nwindow = 10000\n\nfor i in range(int(len(fitted_data)/window)):\n    end = start+window\n    \n    plot_df = fitted_data[start:end].copy()\n\n    plt.plot(plot_df[\"sleep\"].values, label = \"actual\")\n    plt.plot(plot_df[\"sleep_pred\"].values, label = \"pred\", alpha = 0.5)\n    plt.legend()\n    # plt.show()\n\n    plt.plot(X_strd[start:end,1], label = \"enmo\", alpha = 0.3)\n\n    plt.show()\n    \n    start = end","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.228084Z","iopub.status.idle":"2023-12-08T09:06:31.228838Z","shell.execute_reply.started":"2023-12-08T09:06:31.228543Z","shell.execute_reply":"2023-12-08T09:06:31.228573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_strd.max()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.230141Z","iopub.status.idle":"2023-12-08T09:06:31.230897Z","shell.execute_reply.started":"2023-12-08T09:06:31.230621Z","shell.execute_reply":"2023-12-08T09:06:31.230644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2023-12-08T09:06:31.232209Z","iopub.status.idle":"2023-12-08T09:06:31.232959Z","shell.execute_reply.started":"2023-12-08T09:06:31.23268Z","shell.execute_reply":"2023-12-08T09:06:31.232703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}