{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-21T19:18:09.555440Z","iopub.execute_input":"2023-11-21T19:18:09.555819Z","iopub.status.idle":"2023-11-21T19:18:09.566313Z","shell.execute_reply.started":"2023-11-21T19:18:09.555791Z","shell.execute_reply":"2023-11-21T19:18:09.565390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:18:09.578649Z","iopub.execute_input":"2023-11-21T19:18:09.579014Z","iopub.status.idle":"2023-11-21T19:18:09.587227Z","shell.execute_reply.started":"2023-11-21T19:18:09.578986Z","shell.execute_reply":"2023-11-21T19:18:09.586176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyarrow","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:18:09.592779Z","iopub.execute_input":"2023-11-21T19:18:09.593458Z","iopub.status.idle":"2023-11-21T19:18:21.844201Z","shell.execute_reply.started":"2023-11-21T19:18:09.593422Z","shell.execute_reply":"2023-11-21T19:18:21.843130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\ntqdm.pandas()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:18:21.846351Z","iopub.execute_input":"2023-11-21T19:18:21.846718Z","iopub.status.idle":"2023-11-21T19:18:21.853480Z","shell.execute_reply.started":"2023-11-21T19:18:21.846684Z","shell.execute_reply":"2023-11-21T19:18:21.852454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")\ntrain_data\n\n# parquet_file = pq.ParquetFile(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")\n# train_data = parquet_file.read()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:18:21.854903Z","iopub.execute_input":"2023-11-21T19:18:21.855274Z","iopub.status.idle":"2023-11-21T19:19:00.856427Z","shell.execute_reply.started":"2023-11-21T19:18:21.855243Z","shell.execute_reply":"2023-11-21T19:19:00.855337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\n# train_events","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:19:00.858760Z","iopub.execute_input":"2023-11-21T19:19:00.859120Z","iopub.status.idle":"2023-11-21T19:19:00.886700Z","shell.execute_reply.started":"2023-11-21T19:19:00.859091Z","shell.execute_reply":"2023-11-21T19:19:00.885870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv\")\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:19:00.888072Z","iopub.execute_input":"2023-11-21T19:19:00.888486Z","iopub.status.idle":"2023-11-21T19:19:00.904370Z","shell.execute_reply.started":"2023-11-21T19:19:00.888448Z","shell.execute_reply":"2023-11-21T19:19:00.903227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events[\"series_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:19:00.905525Z","iopub.execute_input":"2023-11-21T19:19:00.905800Z","iopub.status.idle":"2023-11-21T19:19:00.914062Z","shell.execute_reply.started":"2023-11-21T19:19:00.905776Z","shell.execute_reply":"2023-11-21T19:19:00.913019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pre_process(data):\n    ts = data.copy()\n    ts[\"timestamp\"] = pd.to_datetime(ts[\"timestamp\"], utc=True)\n#     ts.set_index(\"timestamp\", inplace=True)\n    ts[\"step\"] = ts[\"step\"].astype(float)\n    \n#     ts = ts.resample(\"1min\").max()\n#     ts.reset_index(inplace=True)\n#     print(ts.isna().sum())\n#     ts.set_index(\"timestamp\", inplace=True)\n    return ts \n\nprocessed_ts = train_data.iloc[:10000000,:].\\\ngroupby(\"series_id\").progress_apply(pre_process).reset_index(level=0,drop=True)\n\n# processed_ts.groupby(\"series_id\").apply(plot_timeseries)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:19:00.915879Z","iopub.execute_input":"2023-11-21T19:19:00.916902Z","iopub.status.idle":"2023-11-21T19:20:27.787429Z","shell.execute_reply.started":"2023-11-21T19:19:00.916871Z","shell.execute_reply":"2023-11-21T19:20:27.786390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_ts","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:20:27.788703Z","iopub.execute_input":"2023-11-21T19:20:27.789015Z","iopub.status.idle":"2023-11-21T19:20:27.805978Z","shell.execute_reply.started":"2023-11-21T19:20:27.788990Z","shell.execute_reply":"2023-11-21T19:20:27.804876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:20:27.807171Z","iopub.execute_input":"2023-11-21T19:20:27.807757Z","iopub.status.idle":"2023-11-21T19:20:27.824358Z","shell.execute_reply.started":"2023-11-21T19:20:27.807729Z","shell.execute_reply":"2023-11-21T19:20:27.823469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_df = train_events[[\"series_id\",\"event\",\"step\"]]\nevent_df","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:20:27.828272Z","iopub.execute_input":"2023-11-21T19:20:27.828626Z","iopub.status.idle":"2023-11-21T19:20:27.842927Z","shell.execute_reply.started":"2023-11-21T19:20:27.828601Z","shell.execute_reply":"2023-11-21T19:20:27.841918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_ts = pd.merge(processed_ts, event_df, on=[\"step\", \"series_id\"],how='left')\nmerged_ts","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:20:27.844097Z","iopub.execute_input":"2023-11-21T19:20:27.844390Z","iopub.status.idle":"2023-11-21T19:20:32.953294Z","shell.execute_reply.started":"2023-11-21T19:20:27.844360Z","shell.execute_reply":"2023-11-21T19:20:32.952242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_ts[merged_ts[\"step\"] == 4992.0]","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:20:32.954662Z","iopub.execute_input":"2023-11-21T19:20:32.954947Z","iopub.status.idle":"2023-11-21T19:20:32.988213Z","shell.execute_reply.started":"2023-11-21T19:20:32.954924Z","shell.execute_reply":"2023-11-21T19:20:32.987240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_labels(data):\n    ts = data.copy()\n    \n#     ts[\"sleep\"] = 0\n    ts[\"sleep\"] = np.nan\n    \n    ts.loc[ts[\"event\"]==\"onset\", \"sleep\"] = 1\n    ts.loc[ts[\"event\"]==\"wakeup\", \"sleep\"] = 0\n            \n    ts[\"sleep\"] = ts[\"sleep\"].ffill().fillna(value=0)\n    \n    if ts[\"sleep\"].mean() < 0.2:\n        return pd.DataFrame()\n    \n    print(ts[\"sleep\"].mean())\n        \n#     plt.plot(ts[\"sleep\"])\n#     plt.show()\n                \n#     first_index = ts[\"event\"].first_valid_index()\n        \n#     fill_value = 1 - ts.loc[first_index, \"sleep\"]\n    \n#     ts[\"sleep\"] = ts[\"sleep\"].fillna(value=fill_value)\n\n    return ts\n\nlabelled_ts = merged_ts.groupby(\"series_id\").\\\nprogress_apply(create_labels).reset_index(drop=True)\n\nlabelled_ts","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:20:32.989410Z","iopub.execute_input":"2023-11-21T19:20:32.989701Z","iopub.status.idle":"2023-11-21T19:20:39.420625Z","shell.execute_reply.started":"2023-11-21T19:20:32.989676Z","shell.execute_reply":"2023-11-21T19:20:39.419647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resample to 1 minute\n\ndef resample_fn(ts):\n    ts_resampled = ts.copy()\n    ts_resampled.set_index(\"timestamp\", inplace=True)\n    ts_resampled = ts_resampled.resample(\"1min\").max(numeric_only=True)\n\n    ts_resampled.reset_index(inplace=True)\n    return ts_resampled\n\nresampled_ts = labelled_ts.groupby(\"series_id\").\\\nprogress_apply(resample_fn).reset_index().drop([\"level_1\"], axis=1)\n\nresampled_ts","metadata":{"execution":{"iopub.status.busy":"2023-11-21T19:49:21.705640Z","iopub.execute_input":"2023-11-21T19:49:21.706031Z","iopub.status.idle":"2023-11-21T19:49:23.702133Z","shell.execute_reply.started":"2023-11-21T19:49:21.706000Z","shell.execute_reply":"2023-11-21T19:49:23.701065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_time_features(ts):\n#     ts[\"hour\"] = ts[\"timestamp\"].dt.hour\n#     ts[\"minute\"] = ts[\"timestamp\"].dt.minute\n    \n    return ts\n\ndf_prepared = add_time_features(resampled_ts.copy())\ndf_prepared","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:05:50.001226Z","iopub.execute_input":"2023-11-21T20:05:50.001667Z","iopub.status.idle":"2023-11-21T20:05:50.028530Z","shell.execute_reply.started":"2023-11-21T20:05:50.001635Z","shell.execute_reply":"2023-11-21T20:05:50.027642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cols = [col for col in df_prepared.columns if col not in [\"series_id\", \"timestamp\",\"step\",\"sleep\"]]\nX = df_prepared[train_cols]\ny = df_prepared[\"sleep\"]","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:05:51.460122Z","iopub.execute_input":"2023-11-21T20:05:51.460500Z","iopub.status.idle":"2023-11-21T20:05:51.467454Z","shell.execute_reply.started":"2023-11-21T20:05:51.460464Z","shell.execute_reply":"2023-11-21T20:05:51.466265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\n\nX_strd = scaler.fit_transform(X)\n\nX_strd.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:05:51.777994Z","iopub.execute_input":"2023-11-21T20:05:51.778539Z","iopub.status.idle":"2023-11-21T20:05:51.809003Z","shell.execute_reply.started":"2023-11-21T20:05:51.778487Z","shell.execute_reply":"2023-11-21T20:05:51.808053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_valid, y_train, y_valid = train_test_split(X_strd, y, train_size=0.75)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:05:52.775466Z","iopub.execute_input":"2023-11-21T20:05:52.775880Z","iopub.status.idle":"2023-11-21T20:05:52.832369Z","shell.execute_reply.started":"2023-11-21T20:05:52.775847Z","shell.execute_reply":"2023-11-21T20:05:52.831337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, y_train.shape, X_valid.shape, y_valid.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:05:53.428879Z","iopub.execute_input":"2023-11-21T20:05:53.430065Z","iopub.status.idle":"2023-11-21T20:05:53.436341Z","shell.execute_reply.started":"2023-11-21T20:05:53.430012Z","shell.execute_reply":"2023-11-21T20:05:53.435327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Model","metadata":{}},{"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import layers\nmodel = keras.Sequential([\n    layers.BatchNormalization(input_shape=[X_train.shape[1]]),\n    layers.Dense(256, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    layers.Dense(256, activation='relu'), \n    layers.BatchNormalization(),\n    layers.Dense(256, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    layers.Dense(256, activation='relu'), \n    layers.BatchNormalization(),\n    layers.Dropout(0.3),\n    layers.Dense(1, activation='sigmoid'),\n])","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:16:17.970790Z","iopub.execute_input":"2023-11-21T20:16:17.971221Z","iopub.status.idle":"2023-11-21T20:16:18.154344Z","shell.execute_reply.started":"2023-11-21T20:16:17.971186Z","shell.execute_reply":"2023-11-21T20:16:18.153125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer='adam',\n    loss='binary_crossentropy',\n    metrics=['binary_accuracy'],\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:16:18.412871Z","iopub.execute_input":"2023-11-21T20:16:18.413311Z","iopub.status.idle":"2023-11-21T20:16:18.428131Z","shell.execute_reply.started":"2023-11-21T20:16:18.413269Z","shell.execute_reply":"2023-11-21T20:16:18.427009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"early_stopping = keras.callbacks.EarlyStopping(\n    patience=5,\n    min_delta=0.001,\n    restore_best_weights=True,\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:16:18.751263Z","iopub.execute_input":"2023-11-21T20:16:18.751996Z","iopub.status.idle":"2023-11-21T20:16:18.756750Z","shell.execute_reply.started":"2023-11-21T20:16:18.751961Z","shell.execute_reply":"2023-11-21T20:16:18.755590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    X_train, y_train,\n    validation_data=(X_valid, y_valid),\n    batch_size=512,\n    epochs=200,\n    callbacks=[early_stopping],\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:16:19.106218Z","iopub.execute_input":"2023-11-21T20:16:19.106633Z","iopub.status.idle":"2023-11-21T20:17:35.757191Z","shell.execute_reply.started":"2023-11-21T20:16:19.106600Z","shell.execute_reply":"2023-11-21T20:17:35.756065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = model.predict(X_strd)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:19:00.739788Z","iopub.execute_input":"2023-11-21T20:19:00.740550Z","iopub.status.idle":"2023-11-21T20:19:40.342194Z","shell.execute_reply.started":"2023-11-21T20:19:00.740518Z","shell.execute_reply":"2023-11-21T20:19:40.341006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.mean()","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:19:40.345231Z","iopub.execute_input":"2023-11-21T20:19:40.345583Z","iopub.status.idle":"2023-11-21T20:19:40.354231Z","shell.execute_reply.started":"2023-11-21T20:19:40.345555Z","shell.execute_reply":"2023-11-21T20:19:40.353256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fitted_data = X.copy()\nfitted_data[\"sleep\"] = y\nfitted_data[\"sleep_pred\"] = pred.round()\n\nfitted_data","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:19:40.355535Z","iopub.execute_input":"2023-11-21T20:19:40.355836Z","iopub.status.idle":"2023-11-21T20:19:40.379506Z","shell.execute_reply.started":"2023-11-21T20:19:40.355810Z","shell.execute_reply":"2023-11-21T20:19:40.378524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = 0\nwindow = 10000\n\nfor i in range(int(len(fitted_data)/window)):\n    end = start+window\n    plt.figure(figsize = (15,10))\n    \n    plot_df = fitted_data[start:end].copy()\n\n    plt.plot(plot_df[\"sleep\"].values, label = \"actual\",c=\"k\",linewidth=2)\n    plt.plot(plot_df[\"sleep_pred\"].values, label = \"pred\",c=\"grey\", alpha = 0.7)\n    # plt.show()\n\n    plt.plot(X_strd[start:end,1], label = \"enmo\", alpha = 0.4)\n#     plt.plot(X_strd[start:end,0], label = \"anglwz\", alpha = 0.3)\n    \n    plt.legend()\n\n    plt.show()\n    \n    start = end","metadata":{"execution":{"iopub.status.busy":"2023-11-21T20:19:40.382035Z","iopub.execute_input":"2023-11-21T20:19:40.382790Z","iopub.status.idle":"2023-11-21T20:20:00.514784Z","shell.execute_reply.started":"2023-11-21T20:19:40.382746Z","shell.execute_reply":"2023-11-21T20:20:00.513650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}