{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport polars as pl\nimport numpy as np\nimport pandas as pd\nfrom datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:49:40.438238Z","iopub.execute_input":"2024-07-20T15:49:40.438620Z","iopub.status.idle":"2024-07-20T15:49:43.259265Z","shell.execute_reply.started":"2024-07-20T15:49:40.438588Z","shell.execute_reply":"2024-07-20T15:49:43.258057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_transforms = [\n    pl.col('timestamp').str.to_datetime(),\n    pl.col('timestamp').str.to_datetime().dt.date().alias('date'),\n    pl.col('timestamp').str.to_datetime().dt.time().alias('time')\n    ]\n\n#load train series data\ntrain_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n                .with_columns((dt_transforms))\n                .collect()\n                .to_pandas()\n               )       ","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:49:52.234882Z","iopub.execute_input":"2024-07-20T15:49:52.235278Z","iopub.status.idle":"2024-07-20T15:50:57.272939Z","shell.execute_reply.started":"2024-07-20T15:49:52.235248Z","shell.execute_reply":"2024-07-20T15:50:57.271593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_nulls(df):\n    NAN_SERIES_IDS = [\n    '0f9e60a8e56d',\n    '390b487231ce',\n    '2fc653ca75c7',\n    'c7b1283bb7eb',\n    '89c7daa72eee',\n    'e11b9d69f856',\n    'c5d08fc3e040',\n    'a3e59c2ce3f6',\n    ]\n    df = df[~df['series_id'].isin(NAN_SERIES_IDS)]\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:51:20.763275Z","iopub.execute_input":"2024-07-20T15:51:20.764249Z","iopub.status.idle":"2024-07-20T15:51:20.770683Z","shell.execute_reply.started":"2024-07-20T15:51:20.764201Z","shell.execute_reply":"2024-07-20T15:51:20.769220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = drop_nulls(train_series)","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:51:21.215691Z","iopub.execute_input":"2024-07-20T15:51:21.216713Z","iopub.status.idle":"2024-07-20T15:51:40.487788Z","shell.execute_reply.started":"2024-07-20T15:51:21.216676Z","shell.execute_reply":"2024-07-20T15:51:40.485730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas.api.types import is_datetime64_any_dtype, is_integer_dtype, is_float_dtype\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:51:40.491339Z","iopub.execute_input":"2024-07-20T15:51:40.491873Z","iopub.status.idle":"2024-07-20T15:51:40.500457Z","shell.execute_reply.started":"2024-07-20T15:51:40.491810Z","shell.execute_reply":"2024-07-20T15:51:40.499171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimize_dataframe(df):\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if is_datetime64_any_dtype(df[col]):\n            continue  # Skip datetime columns\n\n        c_min = df[col].min()\n        c_max = df[col].max()\n\n        if is_integer_dtype(df[col]):\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                df[col] = df[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                df[col] = df[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                df[col] = df[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                df[col] = df[col].astype(np.int64)\n        elif is_float_dtype(df[col]):\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                df[col] = df[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                df[col] = df[col].astype(np.float32)\n            else:\n                df[col] = df[col].astype(np.float64)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:51:42.308912Z","iopub.execute_input":"2024-07-20T15:51:42.309297Z","iopub.status.idle":"2024-07-20T15:51:42.322554Z","shell.execute_reply.started":"2024-07-20T15:51:42.309257Z","shell.execute_reply":"2024-07-20T15:51:42.321256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_select = [col for col in train_data.columns if col != 'timestamp']\n\n#  use this list to select the columns\nf_train_data = train_data[columns_to_select]\n\n# optimize DataFrame reduce memory\nfull_train_data = optimize_dataframe(f_train_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:51:42.324212Z","iopub.execute_input":"2024-07-20T15:51:42.324551Z","iopub.status.idle":"2024-07-20T15:52:28.986490Z","shell.execute_reply.started":"2024-07-20T15:51:42.324521Z","shell.execute_reply":"2024-07-20T15:52:28.985529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_data.info()","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:53:32.667977Z","iopub.execute_input":"2024-07-20T15:53:32.668844Z","iopub.status.idle":"2024-07-20T15:53:32.690678Z","shell.execute_reply.started":"2024-07-20T15:53:32.668795Z","shell.execute_reply":"2024-07-20T15:53:32.689117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_data","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count the number of unique values in train series\nk = full_train_data.nunique()\nprint(f'Unique values in train series are: {k}')","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:53:54.088456Z","iopub.execute_input":"2024-07-20T15:53:54.088867Z","iopub.status.idle":"2024-07-20T15:54:14.011410Z","shell.execute_reply.started":"2024-07-20T15:53:54.088818Z","shell.execute_reply":"2024-07-20T15:54:14.010100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_max_tr = full_train_data.date.max()\nprint(f'The recent date of the train data is {date_max_tr}')","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:54:19.293673Z","iopub.execute_input":"2024-07-20T15:54:19.294057Z","iopub.status.idle":"2024-07-20T15:54:19.626043Z","shell.execute_reply.started":"2024-07-20T15:54:19.294029Z","shell.execute_reply":"2024-07-20T15:54:19.624765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check values of the first rows for each series_id\nid_first_rows = full_train_data.groupby('series_id').first()\nid_first_rows","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:54:50.549897Z","iopub.execute_input":"2024-07-20T15:54:50.550650Z","iopub.status.idle":"2024-07-20T15:55:15.305614Z","shell.execute_reply.started":"2024-07-20T15:54:50.550614Z","shell.execute_reply":"2024-07-20T15:55:15.304485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count number of steps for every series_id in train_series data frame grouped by date\ntrain_series_steps_per_day = full_train_data.groupby(\n    ['series_id', 'date'],\n    as_index = False).step.count()\n","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:55:27.414381Z","iopub.execute_input":"2024-07-20T15:55:27.414746Z","iopub.status.idle":"2024-07-20T15:55:42.970474Z","shell.execute_reply.started":"2024-07-20T15:55:27.414718Z","shell.execute_reply":"2024-07-20T15:55:42.969272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_steps_per_day","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:55:42.972128Z","iopub.execute_input":"2024-07-20T15:55:42.972441Z","iopub.status.idle":"2024-07-20T15:55:42.985479Z","shell.execute_reply.started":"2024-07-20T15:55:42.972414Z","shell.execute_reply":"2024-07-20T15:55:42.984390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count unique values of 'step' column for every 'series_id'\nsteps2 = train_series_steps_per_day.groupby(\n    ['series_id'],\n    as_index=False).step.value_counts()\n\nsteps2","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:59:01.499425Z","iopub.execute_input":"2024-07-20T15:59:01.501424Z","iopub.status.idle":"2024-07-20T15:59:01.529175Z","shell.execute_reply.started":"2024-07-20T15:59:01.501376Z","shell.execute_reply":"2024-07-20T15:59:01.528105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count the sum of observed days for each series_id\ndays_per_id = steps2.groupby('series_id')['count'].sum()\ndays_per_id = days_per_id.reset_index()\ndays_per_id.columns = ['series_id', 'total_days']\ndays_per_id","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:59:21.530059Z","iopub.execute_input":"2024-07-20T15:59:21.530997Z","iopub.status.idle":"2024-07-20T15:59:21.547132Z","shell.execute_reply.started":"2024-07-20T15:59:21.530959Z","shell.execute_reply":"2024-07-20T15:59:21.545665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make a subset of no_zero_steps with steps = 17280 (24-hour recorded days)\nfull_day = steps2[steps2['step'] == 17280 ]\nfull_day.columns = ['series_id', 'step', 'full_day_steps_count']\nfull_day\n","metadata":{"execution":{"iopub.status.busy":"2024-07-20T15:59:44.182362Z","iopub.execute_input":"2024-07-20T15:59:44.182817Z","iopub.status.idle":"2024-07-20T15:59:44.197825Z","shell.execute_reply.started":"2024-07-20T15:59:44.182779Z","shell.execute_reply":"2024-07-20T15:59:44.196765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_max= full_train_data.enmo.max()\nenmo_min = full_train_data.enmo.min()\nenmo_std = full_train_data.enmo.std()\nenmo_mean = full_train_data.enmo.mean()\nenmo_perc75 = full_train_data.enmo.quantile(0.75)\nenmo_perc90 = full_train_data.enmo.quantile(0.90)\nenmo_perc99 = full_train_data.enmo.quantile(0.99)","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:00:44.265155Z","iopub.execute_input":"2024-07-20T16:00:44.265578Z","iopub.status.idle":"2024-07-20T16:01:02.929431Z","shell.execute_reply.started":"2024-07-20T16:00:44.265548Z","shell.execute_reply":"2024-07-20T16:01:02.928228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'enmo max is {enmo_max}')\nprint(f'enmo min is {enmo_min}')\nprint(f'enmo standard deviation is {enmo_std}')\nprint(f'enmo mean is {enmo_mean}')\nprint(f'enmo third quantile is {enmo_perc75}')\nprint(f'enmo 90 percentile is {enmo_perc90}')\nprint(f'enmo 99 percentile is {enmo_perc99}')","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:01:34.630812Z","iopub.execute_input":"2024-07-20T16:01:34.631580Z","iopub.status.idle":"2024-07-20T16:01:34.638384Z","shell.execute_reply.started":"2024-07-20T16:01:34.631537Z","shell.execute_reply":"2024-07-20T16:01:34.637272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the distribution of enmo-values\nplt.figure(figsize=(10, 6))\nax = sns.distplot(full_train_data.enmo)\nplt.title('Distribution of ENMO values in train_series')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:03:44.020250Z","iopub.execute_input":"2024-07-20T16:03:44.020722Z","iopub.status.idle":"2024-07-20T16:11:48.430429Z","shell.execute_reply.started":"2024-07-20T16:03:44.020686Z","shell.execute_reply":"2024-07-20T16:11:48.429118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # fix huge enmo values (enmo > 99th percentile) at the level = 0.4\n\n# full_train_data.loc[full_train_data.enmo > enmo_perc99, 'enmo'] = 0.4","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # plot the distribution of enmo-values\n# plt.figure(figsize=(10, 6))\n# ax = sns.distplot(full_train_data.enmo)\n# plt.title('Distribution of ENMO values in train_series')\n# plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mxa= full_train_data.anglez.max()\nmna = full_train_data.anglez.min()\nmeana = full_train_data.anglez.mean()\nsta = full_train_data.anglez.std()\nperc75a = full_train_data.anglez.quantile(0.75)\nperc90a = full_train_data.anglez.quantile(0.90)\nperc99a = full_train_data.anglez.quantile(0.99)\nprint(f'anglez max is {mxa}')\nprint(f'anglez min is {mna}')\nprint(f'anglez mean is {meana}')\nprint(f'anglez standard deviation is {sta}')\nprint(f'anglez third quantile is {perc75a}')\nprint(f'anglez 90 percentile is {perc90a}')\nprint(f'anglez 99 percentile is {perc99a}')","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:11:48.432440Z","iopub.execute_input":"2024-07-20T16:11:48.432775Z","iopub.status.idle":"2024-07-20T16:12:04.282005Z","shell.execute_reply.started":"2024-07-20T16:11:48.432744Z","shell.execute_reply":"2024-07-20T16:12:04.280903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot the distribution of anglez-values\nplt.figure(figsize=(10, 6))\nax = sns.distplot(full_train_data.anglez)\nplt.title('Distribution of anglez values in train_series')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:12:59.993382Z","iopub.execute_input":"2024-07-20T16:12:59.993874Z","iopub.status.idle":"2024-07-20T16:21:43.247791Z","shell.execute_reply.started":"2024-07-20T16:12:59.993816Z","shell.execute_reply":"2024-07-20T16:21:43.246586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:21:43.250214Z","iopub.execute_input":"2024-07-20T16:21:43.250621Z","iopub.status.idle":"2024-07-20T16:21:44.247404Z","shell.execute_reply.started":"2024-07-20T16:21:43.250585Z","shell.execute_reply":"2024-07-20T16:21:44.246114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\ntrain_events.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:21:44.248878Z","iopub.execute_input":"2024-07-20T16:21:44.249249Z","iopub.status.idle":"2024-07-20T16:21:44.300751Z","shell.execute_reply.started":"2024-07-20T16:21:44.249220Z","shell.execute_reply":"2024-07-20T16:21:44.299637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:21:44.303007Z","iopub.execute_input":"2024-07-20T16:21:44.303346Z","iopub.status.idle":"2024-07-20T16:21:44.315146Z","shell.execute_reply.started":"2024-07-20T16:21:44.303317Z","shell.execute_reply":"2024-07-20T16:21:44.313814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = drop_nulls(train_events)","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:22:39.761016Z","iopub.execute_input":"2024-07-20T16:22:39.761530Z","iopub.status.idle":"2024-07-20T16:22:39.773471Z","shell.execute_reply.started":"2024-07-20T16:22:39.761483Z","shell.execute_reply":"2024-07-20T16:22:39.771886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:22:56.013022Z","iopub.execute_input":"2024-07-20T16:22:56.013436Z","iopub.status.idle":"2024-07-20T16:22:56.332406Z","shell.execute_reply.started":"2024-07-20T16:22:56.013404Z","shell.execute_reply":"2024-07-20T16:22:56.331069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_example(id):\n    df_example= train_data.loc[train_data[\"series_id\"]==id].reset_index(drop = True).copy()\n    df_example[\"timestamp\"] = pd.to_datetime(df_example[\"timestamp\"],utc = True)\n    df_example[\"date\"] = df_example[\"timestamp\"].dt.date\n    for date, data in df_example.groupby(\"date\"):\n        step_min = data[\"step\"].min()\n        step_max = data[\"step\"].max()\n        series_id = data[\"series_id\"].values[0]\n        fig, ax = plt.subplots(figsize=(15,5))\n        data.set_index(\"step\")[[\"anglez\"]].plot(ax= ax,color = \"blue\", title = f\"date:{date}\")\n        ax2 = ax.twinx()\n        data.set_index(\"step\")[[\"enmo\"]].plot(ax= ax2, color = \"orange\")\n        my_events = events.query(\n        \"series_id == @series_id and step >= @step_min and step <= @step_max\"\n        )\n        for i, e in my_events.iterrows():\n            ax.axvline(e[\"step\"],color = \"red\")\n        plt.show()\n\nplot_example(\"fcca183903b7\")","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:22:58.322446Z","iopub.execute_input":"2024-07-20T16:22:58.322869Z","iopub.status.idle":"2024-07-20T16:23:31.951317Z","shell.execute_reply.started":"2024-07-20T16:22:58.322817Z","shell.execute_reply":"2024-07-20T16:23:31.950206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_example(\"038441c925bb\")","metadata":{"execution":{"iopub.status.busy":"2024-07-20T16:25:21.968063Z","iopub.execute_input":"2024-07-20T16:25:21.968604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}