{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport polars as pl\nimport numpy as np\nimport pandas as pd\nfrom datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:47:50.585954Z","iopub.execute_input":"2024-07-09T14:47:50.586347Z","iopub.status.idle":"2024-07-09T14:47:53.375837Z","shell.execute_reply.started":"2024-07-09T14:47:50.586317Z","shell.execute_reply":"2024-07-09T14:47:53.374547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_transforms = [\n    pl.col('timestamp').str.to_datetime(),\n    pl.col('timestamp').str.to_datetime().dt.date().alias('date'),\n    pl.col('timestamp').str.to_datetime().dt.time().alias('time'),\n    pl.col(\"timestamp\").str.to_datetime().dt.weekday().alias(\"weekday\"),\n    pl.col(\"timestamp\").str.to_datetime().dt.hour().alias(\"hour\"),\n    pl.col(\"timestamp\").str.to_datetime().dt.minute().alias(\"minute\")\n    ]\n\n#load train series data\ntrain_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n                .with_columns((dt_transforms))\n                .collect()\n                .to_pandas()\n               )       ","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:49:52.714898Z","iopub.execute_input":"2024-07-09T14:49:52.715320Z","iopub.status.idle":"2024-07-09T14:51:37.697531Z","shell.execute_reply.started":"2024-07-09T14:49:52.715288Z","shell.execute_reply":"2024-07-09T14:51:37.696056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_series.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def drop_nulls(df):\n    NAN_SERIES_IDS = [\n    '0f9e60a8e56d',\n    '390b487231ce',\n    '2fc653ca75c7',\n    'c7b1283bb7eb',\n    '89c7daa72eee',\n    'e11b9d69f856',\n    'c5d08fc3e040',\n    'a3e59c2ce3f6',\n    ]\n    df = df[~df['series_id'].isin(NAN_SERIES_IDS)]\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:51:37.699734Z","iopub.execute_input":"2024-07-09T14:51:37.700194Z","iopub.status.idle":"2024-07-09T14:51:37.707902Z","shell.execute_reply.started":"2024-07-09T14:51:37.700157Z","shell.execute_reply":"2024-07-09T14:51:37.706619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = drop_nulls(train_series)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:51:37.709188Z","iopub.execute_input":"2024-07-09T14:51:37.709559Z","iopub.status.idle":"2024-07-09T14:52:02.852330Z","shell.execute_reply.started":"2024-07-09T14:51:37.709528Z","shell.execute_reply":"2024-07-09T14:52:02.850713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas.api.types import is_datetime64_any_dtype, is_integer_dtype, is_float_dtype\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:52:02.855598Z","iopub.execute_input":"2024-07-09T14:52:02.856662Z","iopub.status.idle":"2024-07-09T14:52:02.862906Z","shell.execute_reply.started":"2024-07-09T14:52:02.856617Z","shell.execute_reply":"2024-07-09T14:52:02.861740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_series\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:52:02.864270Z","iopub.execute_input":"2024-07-09T14:52:02.864574Z","iopub.status.idle":"2024-07-09T14:52:03.927269Z","shell.execute_reply.started":"2024-07-09T14:52:02.864547Z","shell.execute_reply":"2024-07-09T14:52:03.926083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimize_dataframe(df):\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if is_datetime64_any_dtype(df[col]):\n            continue  # Skip datetime columns\n\n        c_min = df[col].min()\n        c_max = df[col].max()\n\n        if is_integer_dtype(df[col]):\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                df[col] = df[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                df[col] = df[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                df[col] = df[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                df[col] = df[col].astype(np.int64)\n        elif is_float_dtype(df[col]):\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                df[col] = df[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                df[col] = df[col].astype(np.float32)\n            else:\n                df[col] = df[col].astype(np.float64)\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:52:03.928903Z","iopub.execute_input":"2024-07-09T14:52:03.929349Z","iopub.status.idle":"2024-07-09T14:52:03.941865Z","shell.execute_reply.started":"2024-07-09T14:52:03.929310Z","shell.execute_reply":"2024-07-09T14:52:03.940654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_select = [col for col in train_data.columns if col != 'timestamp']\n\n#  use this list to select the columns\nf_train_data = train_data[columns_to_select]\n\n# optimize DataFrame reduce memory\nfull_train_data = optimize_dataframe(f_train_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:52:03.943203Z","iopub.execute_input":"2024-07-09T14:52:03.943534Z","iopub.status.idle":"2024-07-09T14:52:45.117723Z","shell.execute_reply.started":"2024-07-09T14:52:03.943502Z","shell.execute_reply":"2024-07-09T14:52:45.116541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# full_train_data.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# full_train_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # count the number of unique values in train series\n# k = full_train_data.nunique()\n# print(f'Unique values in train series are: {k}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# date_max_tr = full_train_data.date.max()\n# print(f'The recent date of the train data is {date_max_tr}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # check the latest date of train_series\n# date_min_tr = full_train_data.date.min()\n# print(f'The earliest date of the train data is {date_min_tr}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # check values of the first rows for each series_id\n# id_first_rows = full_train_data.groupby('series_id').first()\n# id_first_rows","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # count number of steps for every series_id in train_series data frame grouped by date\n# train_series_steps_per_day = full_train_data.groupby(\n#     ['series_id', 'date'],\n#     as_index = False).step.count()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_series_steps_per_day","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # count unique values of 'step' column for every 'series_id'\n# steps2 = train_series_steps_per_day.groupby(\n#     ['series_id'],\n#     as_index=False).step.value_counts()\n\n# steps2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # count the sum of observed days for each series_id\n# days_per_id = steps2.groupby('series_id')['count'].sum()\n# days_per_id = days_per_id.reset_index()\n# days_per_id.columns = ['series_id', 'total_days']\n# days_per_id","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # make a subset of no_zero_steps with steps = 17280 (24-hour recorded days)\n# full_day = steps2[steps2['step'] == 17280 ]\n# full_day.columns = ['series_id', 'step', 'full_day_steps_count']\n# full_day","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def extract_full_day_data(df):\n    # Count the number of steps for each series_id and date\n    steps_per_day = df.groupby(['series_id', 'date'], as_index=False)['step'].count()\n    \n    # Filter to keep only the series_id and date combinations with 17280 steps\n    valid_days = steps_per_day[steps_per_day['step'] == 17280]\n    \n    # Merge the original DataFrame with the valid_days to filter the rows\n    filtered_df = pd.merge(df, valid_days[['series_id', 'date']], on=['series_id', 'date'], how='inner')\n    \n    return filtered_df\n","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:52:45.119210Z","iopub.execute_input":"2024-07-09T14:52:45.119635Z","iopub.status.idle":"2024-07-09T14:52:45.127065Z","shell.execute_reply.started":"2024-07-09T14:52:45.119598Z","shell.execute_reply":"2024-07-09T14:52:45.125979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_data\ndel f_train_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:52:45.128312Z","iopub.execute_input":"2024-07-09T14:52:45.128677Z","iopub.status.idle":"2024-07-09T14:52:46.316010Z","shell.execute_reply.started":"2024-07-09T14:52:45.128649Z","shell.execute_reply":"2024-07-09T14:52:46.314789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_data = extract_full_day_data(full_train_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:52:46.319393Z","iopub.execute_input":"2024-07-09T14:52:46.319929Z","iopub.status.idle":"2024-07-09T14:53:38.345885Z","shell.execute_reply.started":"2024-07-09T14:52:46.319886Z","shell.execute_reply":"2024-07-09T14:53:38.344800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# full_train_data.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# enmo_max= full_train_data.enmo.max()\n# enmo_min = full_train_data.enmo.min()\n# enmo_std = full_train_data.enmo.std()\n# enmo_perc75 = full_train_data.enmo.quantile(0.75)\n# enmo_perc90 = full_train_data.enmo.quantile(0.90)\n# enmo_perc99 = full_train_data.enmo.quantile(0.99)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f'enmo max is {enmo_max}')\n# print(f'enmo min is {enmo_min}')\n# print(f'enmo standard deviation is {enmo_std}')\n# print(f'enmo third quantile is {enmo_perc75}')\n# print(f'enmo 90 percentile is {enmo_perc90}')\n# print(f'enmo 99 percentile is {enmo_perc99}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # plot the distribution of enmo-values\n# plt.figure(figsize=(10, 6))\n# ax = sns.distplot(full_train_data.enmo)\n# plt.title('Distribution of ENMO values in train_series')\n# plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # fix huge enmo values (enmo > 99th percentile) at the level = 0.4\n\n# full_train_data.loc[full_train_data.enmo > enmo_perc99, 'enmo'] = 0.4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # plot the distribution of enmo-values\n# plt.figure(figsize=(10, 6))\n# ax = sns.distplot(full_train_data.enmo)\n# plt.title('Distribution of ENMO values in train_series')\n# plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mxa= full_train_data.anglez.max()\n# mna = full_train_data.anglez.min()\n# meana = full_train_data.anglez.mean()\n# sta = full_train_data.anglez.std()\n# perc75a = full_train_data.anglez.quantile(0.75)\n# perc90a = full_train_data.anglez.quantile(0.90)\n# perc99a = full_train_data.anglez.quantile(0.99)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f'anglez max is {mxa}')\n# print(f'anglez min is {mna}')\n# print(f'anglez mean is {meana}')\n# print(f'anglez standard deviation is {sta}')\n# print(f'anglez third quantile is {perc75a}')\n# print(f'anglez 90 percentile is {perc90a}')\n# print(f'anglez 99 percentile is {perc99a}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # plot the distribution of anglez-values\n# plt.figure(figsize=(10, 6))\n# ax = sns.distplot(full_train_data.anglez)\n# plt.title('Distribution of anglez values in train_series')\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-08T12:47:19.734589Z","iopub.execute_input":"2024-07-08T12:47:19.734973Z","iopub.status.idle":"2024-07-08T12:47:19.827852Z","shell.execute_reply.started":"2024-07-08T12:47:19.734941Z","shell.execute_reply":"2024-07-08T12:47:19.826741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:53:38.347227Z","iopub.execute_input":"2024-07-09T14:53:38.347570Z","iopub.status.idle":"2024-07-09T14:53:38.393204Z","shell.execute_reply.started":"2024-07-09T14:53:38.347539Z","shell.execute_reply":"2024-07-09T14:53:38.392081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_events.isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find series_id with no missing values\n# series_full = train_events[~train_events.isnull().any(axis=1)]","metadata":{"execution":{"iopub.status.busy":"2024-07-08T12:47:19.888258Z","iopub.execute_input":"2024-07-08T12:47:19.888595Z","iopub.status.idle":"2024-07-08T12:47:19.901317Z","shell.execute_reply.started":"2024-07-08T12:47:19.888567Z","shell.execute_reply":"2024-07-08T12:47:19.900230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_full = drop_nulls(train_events)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:53:38.394729Z","iopub.execute_input":"2024-07-09T14:53:38.395091Z","iopub.status.idle":"2024-07-09T14:53:38.403894Z","shell.execute_reply.started":"2024-07-09T14:53:38.395061Z","shell.execute_reply":"2024-07-09T14:53:38.402681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_data = full_train_data.merge(series_full[[\"event\",\"step\"]],on=\"step\",how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:53:38.424116Z","iopub.execute_input":"2024-07-09T14:53:38.424665Z","iopub.status.idle":"2024-07-09T14:54:17.842093Z","shell.execute_reply.started":"2024-07-09T14:53:38.424620Z","shell.execute_reply":"2024-07-09T14:54:17.840996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def event_encode(df):\n    df[\"event\"] = df[\"event\"].ffill()\n    df[\"event\"] = df[\"event\"].fillna(\"wakeup\")\n    df[\"event\"] = df[\"event\"].map({\"wakeup\":0, \"onset\":1})\n    return df\n\nall_data = event_encode(full_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:53:38.407180Z","iopub.execute_input":"2024-07-09T14:53:38.407718Z","iopub.status.idle":"2024-07-09T14:53:38.422542Z","shell.execute_reply.started":"2024-07-09T14:53:38.407659Z","shell.execute_reply":"2024-07-09T14:53:38.421314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del series_full\ndel train_events\ndel full_data\ndel full_train_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:54:17.843636Z","iopub.execute_input":"2024-07-09T14:54:17.844114Z","iopub.status.idle":"2024-07-09T14:54:18.848650Z","shell.execute_reply.started":"2024-07-09T14:54:17.844070Z","shell.execute_reply":"2024-07-09T14:54:18.847571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating lagged absolute difference features\nall_data['anglez'] = all_data['anglez'].abs()","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:54:18.849974Z","iopub.execute_input":"2024-07-09T14:54:18.850267Z","iopub.status.idle":"2024-07-09T14:54:19.035322Z","shell.execute_reply.started":"2024-07-09T14:54:18.850243Z","shell.execute_reply":"2024-07-09T14:54:19.034104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating lagged features\nall_data['anglez_diff'] = all_data.groupby('series_id')['anglez'].diff().abs().fillna(0)\nall_data['enmo_diff'] = all_data.groupby('series_id')['enmo'].diff().abs().fillna(0)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:54:29.334767Z","iopub.execute_input":"2024-07-09T14:54:29.335155Z","iopub.status.idle":"2024-07-09T14:55:01.790972Z","shell.execute_reply.started":"2024-07-09T14:54:29.335124Z","shell.execute_reply":"2024-07-09T14:55:01.789618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute rolling standard deviation and assign to new columns using .loc\nall_data['anglez_roll_std_1m'] = all_data.groupby('series_id')['anglez'].rolling(12,center=True, min_periods=1).std().reset_index(level=0, drop=True)\nall_data['enmo_roll_std_1m'] = all_data.groupby('series_id')['enmo'].rolling(12,center=True, min_periods=1).std().reset_index(level=0, drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T14:55:41.743208Z","iopub.execute_input":"2024-07-09T14:55:41.744099Z","iopub.status.idle":"2024-07-09T14:58:13.309250Z","shell.execute_reply.started":"2024-07-09T14:55:41.744059Z","shell.execute_reply":"2024-07-09T14:58:13.308057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating rolling mean features for accelerometer data\nall_data['anglez_roll_mean_1m'] = all_data.groupby('series_id')['anglez'].rolling(12,center=True, min_periods=1).mean().reset_index(level=0, drop=True)\nall_data['enmo_roll_mean_1m'] = all_data.groupby('series_id')['enmo'].rolling(12,center=True, min_periods=1).mean().reset_index(level=0, drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:00:05.770047Z","iopub.execute_input":"2024-07-09T15:00:05.770557Z","iopub.status.idle":"2024-07-09T15:02:35.266419Z","shell.execute_reply.started":"2024-07-09T15:00:05.770511Z","shell.execute_reply":"2024-07-09T15:02:35.264889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n# optimize DataFrame reduce memory\nall_data = optimize_dataframe(all_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:02:35.268778Z","iopub.execute_input":"2024-07-09T15:02:35.269162Z","iopub.status.idle":"2024-07-09T15:03:30.846110Z","shell.execute_reply.started":"2024-07-09T15:02:35.269131Z","shell.execute_reply":"2024-07-09T15:03:30.845004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute rolling standard deviation and assign to new columns using .loc\nall_data['anglez_roll_std_30m'] = all_data.groupby('series_id')['anglez'].rolling(360,center=True, min_periods=1).std().reset_index(level=0, drop=True)\nall_data['enmo_roll_std_30m'] = all_data.groupby('series_id')['enmo'].rolling(360,center=True, min_periods=1).std().reset_index(level=0, drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:10:11.329462Z","iopub.execute_input":"2024-07-09T15:10:11.329925Z","iopub.status.idle":"2024-07-09T15:12:46.091094Z","shell.execute_reply.started":"2024-07-09T15:10:11.329886Z","shell.execute_reply":"2024-07-09T15:12:46.089602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating rolling mean features for accelerometer data\nall_data['anglez_roll_mean_30m'] = all_data.groupby('series_id')['anglez'].rolling(360,center=True, min_periods=1).mean().reset_index(level=0, drop=True)\nall_data['enmo_roll_mean_30m'] = all_data.groupby('series_id')['enmo'].rolling(360,center=True, min_periods=1).mean().reset_index(level=0, drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:14:42.383566Z","iopub.execute_input":"2024-07-09T15:14:42.384009Z","iopub.status.idle":"2024-07-09T15:17:09.407450Z","shell.execute_reply.started":"2024-07-09T15:14:42.383974Z","shell.execute_reply":"2024-07-09T15:17:09.405892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n# optimize DataFrame reduce memory\nall_data = optimize_dataframe(all_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:17:54.532027Z","iopub.execute_input":"2024-07-09T15:17:54.533232Z","iopub.status.idle":"2024-07-09T15:18:56.554245Z","shell.execute_reply.started":"2024-07-09T15:17:54.533188Z","shell.execute_reply":"2024-07-09T15:18:56.553220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute rolling standard deviation and assign to new columns using .loc\nall_data['anglez_roll_std_1h'] = all_data.groupby('series_id')['anglez'].rolling(720,center=True, min_periods=1).std().reset_index(level=0, drop=True)\nall_data['enmo_roll_std_1h'] = all_data.groupby('series_id')['enmo'].rolling(720,center=True, min_periods=1).std().reset_index(level=0, drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:22:47.744632Z","iopub.execute_input":"2024-07-09T15:22:47.745551Z","iopub.status.idle":"2024-07-09T15:25:21.793747Z","shell.execute_reply.started":"2024-07-09T15:22:47.745508Z","shell.execute_reply":"2024-07-09T15:25:21.792248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating rolling mean features for accelerometer data\nall_data['anglez_roll_mean_1h'] = all_data.groupby('series_id')['anglez'].rolling(720,center=True, min_periods=1).mean().reset_index(level=0, drop=True)\nall_data['enmo_roll_mean_1h'] = all_data.groupby('series_id')['enmo'].rolling(720,center=True, min_periods=1).mean().reset_index(level=0, drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:25:36.421086Z","iopub.execute_input":"2024-07-09T15:25:36.421502Z","iopub.status.idle":"2024-07-09T15:28:05.354465Z","shell.execute_reply.started":"2024-07-09T15:25:36.421468Z","shell.execute_reply":"2024-07-09T15:28:05.352563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n# optimize DataFrame reduce memory\nall_data = optimize_dataframe(all_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:28:23.627054Z","iopub.execute_input":"2024-07-09T15:28:23.627447Z","iopub.status.idle":"2024-07-09T15:29:44.028863Z","shell.execute_reply.started":"2024-07-09T15:28:23.627400Z","shell.execute_reply":"2024-07-09T15:29:44.027608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert hours to radians\n# Calculate sine and cosine of the hour column directly\nall_data['sin_hour'] = np.sin(all_data['hour'] * (2 * np.pi / 24))\nall_data['cos_hour'] = np.cos(all_data['hour'] * (2 * np.pi / 24))","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:20:09.183789Z","iopub.execute_input":"2024-07-09T15:20:09.184245Z","iopub.status.idle":"2024-07-09T15:20:14.170724Z","shell.execute_reply.started":"2024-07-09T15:20:09.184214Z","shell.execute_reply":"2024-07-09T15:20:14.169709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert hours to radians\n# Calculate sine and cosine of the hour column directly\nall_data['sin_minute'] = np.sin(all_data['minute'] * (2 * np.pi / 60))\nall_data['cos_minute'] = np.cos(all_data['minute'] * (2 * np.pi / 60))","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:21:21.061651Z","iopub.execute_input":"2024-07-09T15:21:21.062080Z","iopub.status.idle":"2024-07-09T15:21:26.120268Z","shell.execute_reply.started":"2024-07-09T15:21:21.062048Z","shell.execute_reply":"2024-07-09T15:21:26.119030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data = all_data.drop(columns=['hour','minute'])","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:29:51.548892Z","iopub.execute_input":"2024-07-09T15:29:51.549280Z","iopub.status.idle":"2024-07-09T15:30:12.056946Z","shell.execute_reply.started":"2024-07-09T15:29:51.549251Z","shell.execute_reply":"2024-07-09T15:30:12.055770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n# optimize DataFrame reduce memory\nall_data = optimize_dataframe(all_data)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:30:15.872348Z","iopub.execute_input":"2024-07-09T15:30:15.872773Z","iopub.status.idle":"2024-07-09T15:31:38.097717Z","shell.execute_reply.started":"2024-07-09T15:30:15.872739Z","shell.execute_reply":"2024-07-09T15:31:38.096567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.to_parquet(\"all_data.parquet\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-09T15:31:45.368352Z","iopub.execute_input":"2024-07-09T15:31:45.368812Z","iopub.status.idle":"2024-07-09T15:34:52.470345Z","shell.execute_reply.started":"2024-07-09T15:31:45.368778Z","shell.execute_reply":"2024-07-09T15:34:52.468847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}