{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6507308,"sourceType":"datasetVersion","datasetId":3706667}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nimport gc\nfrom tqdm.auto import tqdm \n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-09T09:37:50.083433Z","iopub.execute_input":"2023-11-09T09:37:50.083952Z","iopub.status.idle":"2023-11-09T09:37:50.925993Z","shell.execute_reply.started":"2023-11-09T09:37:50.0839Z","shell.execute_reply":"2023-11-09T09:37:50.924709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = pl.read_parquet('/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T09:37:53.315451Z","iopub.execute_input":"2023-11-09T09:37:53.316027Z","iopub.status.idle":"2023-11-09T09:37:55.615058Z","shell.execute_reply.started":"2023-11-09T09:37:53.315991Z","shell.execute_reply":"2023-11-09T09:37:55.614201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.replace(\"awake\", train[\"awake\"].cast(pl.Int8))","metadata":{"execution":{"iopub.status.busy":"2023-11-09T09:37:56.942427Z","iopub.execute_input":"2023-11-09T09:37:56.943177Z","iopub.status.idle":"2023-11-09T09:37:57.07471Z","shell.execute_reply.started":"2023-11-09T09:37:56.94314Z","shell.execute_reply":"2023-11-09T09:37:57.073386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train[\"step\"][1:]-train[\"step\"][:-1]).ne(1).any()","metadata":{"execution":{"iopub.status.busy":"2023-11-06T13:02:56.480225Z","iopub.execute_input":"2023-11-06T13:02:56.480662Z","iopub.status.idle":"2023-11-06T13:02:56.547078Z","shell.execute_reply.started":"2023-11-06T13:02:56.480626Z","shell.execute_reply":"2023-11-06T13:02:56.545923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series_id in train[\"series_id\"].unique():\n    temp = train.filter(pl.col(\"series_id\") == series_id)\n    print((temp[\"step\"][1:]-temp[\"step\"][:-1]).ne(1).any())","metadata":{"execution":{"iopub.status.busy":"2023-11-09T09:38:04.0281Z","iopub.execute_input":"2023-11-09T09:38:04.028544Z","iopub.status.idle":"2023-11-09T09:38:08.589459Z","shell.execute_reply.started":"2023-11-09T09:38:04.028511Z","shell.execute_reply":"2023-11-09T09:38:08.588066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train.clone()\ngrouped_df = df.group_by(\"series_id\").agg(pl.col(\"step\").diff())\ngrouped_df","metadata":{"execution":{"iopub.status.busy":"2023-11-05T22:21:49.248843Z","iopub.execute_input":"2023-11-05T22:21:49.249223Z","iopub.status.idle":"2023-11-05T22:21:49.848265Z","shell.execute_reply.started":"2023-11-05T22:21:49.249193Z","shell.execute_reply":"2023-11-05T22:21:49.847177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def luan_unique(seq):\n    seen = set()\n    seen_add = seen.add\n    return [x for x in seq if not (x in seen or seen_add(x))]\n\n\nids = luan_unique(train['series_id'])\n\nprint(ids)\n\nlen(ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-05T21:33:09.234976Z","iopub.execute_input":"2023-11-05T21:33:09.235601Z","iopub.status.idle":"2023-11-05T21:33:11.207657Z","shell.execute_reply.started":"2023-11-05T21:33:09.235567Z","shell.execute_reply":"2023-11-05T21:33:11.206813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"smooth data","metadata":{}},{"cell_type":"code","source":"# %%time\n\n# dict_ids2data = {}\n\n# for i in tqdm(range(len(ids))):\n# # for i in range(2):\n#     print (i,': ', ids[i])\n#     dict_ids2data[ids[i]] = train.filter(pl.col(\"series_id\") == ids[i])\n    \n#     dict_ids2data[ids[i]] = dict_ids2data[ids[i]].with_columns(\n#         anglez = pl.col(\"anglez\").rolling_mean(window_size=21, center = True),\n#         enmo = pl.col(\"enmo\").rolling_mean(window_size=21, center = True),\n#     )\n    \n#     dict_ids2data[ids[i]] = dict_ids2data[ids[i]].fill_null(strategy=\"forward\").fill_null(strategy=\"backward\")\n    \n\n\n# train_smooth = pl.concat([i for i in dict_ids2data.values()], rechunk=True)\n\n\n# del dict_ids2data\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-05T16:59:18.399336Z","iopub.execute_input":"2023-11-05T16:59:18.399759Z","iopub.status.idle":"2023-11-05T16:59:18.405311Z","shell.execute_reply.started":"2023-11-05T16:59:18.399719Z","shell.execute_reply":"2023-11-05T16:59:18.40421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smooth = train\n","metadata":{"execution":{"iopub.status.busy":"2023-11-09T09:38:21.125673Z","iopub.execute_input":"2023-11-09T09:38:21.126085Z","iopub.status.idle":"2023-11-09T09:38:21.131561Z","shell.execute_reply.started":"2023-11-09T09:38:21.126053Z","shell.execute_reply":"2023-11-09T09:38:21.130026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_reduce = train_smooth.take_every(5)","metadata":{"execution":{"iopub.status.busy":"2023-11-05T16:59:18.428616Z","iopub.execute_input":"2023-11-05T16:59:18.429053Z","iopub.status.idle":"2023-11-05T16:59:18.433725Z","shell.execute_reply.started":"2023-11-05T16:59:18.429008Z","shell.execute_reply":"2023-11-05T16:59:18.432606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smooth[\"timestamp\"].str.slice(-5,).unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-06T21:19:42.119679Z","iopub.execute_input":"2023-11-06T21:19:42.120094Z","iopub.status.idle":"2023-11-06T21:19:43.412872Z","shell.execute_reply.started":"2023-11-06T21:19:42.120063Z","shell.execute_reply":"2023-11-06T21:19:43.411734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smooth[\"timestamp\"]","metadata":{"execution":{"iopub.status.busy":"2023-11-06T21:20:15.740669Z","iopub.execute_input":"2023-11-06T21:20:15.741057Z","iopub.status.idle":"2023-11-06T21:20:15.74837Z","shell.execute_reply.started":"2023-11-06T21:20:15.741028Z","shell.execute_reply":"2023-11-06T21:20:15.747457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef process_timestamp(df):\n    df = df.with_columns(\n        #year = (df[\"timestamp\"].str.slice(0,4).cast(pl.Int16) - 2000).cast(pl.UInt8),\n        #month = df[\"timestamp\"].str.slice(5,2).cast(pl.Int8),\n        #day = df[\"timestamp\"].str.slice(8,2).cast(pl.Int8),\n        #hour = df[\"timestamp\"].str.slice(11,2).cast(pl.Int8),\n        #minute = df[\"timestamp\"].str.slice(14,2).cast(pl.Int8),\n        #second = df[\"timestamp\"].str.slice(17,2).cast(pl.Int8),\n        #time_zone = df[\"timestamp\"].str.slice(-5,3).cast(pl.Int32),\n        enmo =  (pl.col('enmo')*1000).cast(pl.UInt16),\n        date_time =pl.col('timestamp').str.strptime(pl.Datetime),\n    )\n    return df\n    \n    \n# print (process_timestamp(train_smooth))\n\ntrain_time_extract = process_timestamp(train_smooth)\n\ntrain_time_extract","metadata":{"execution":{"iopub.status.busy":"2023-11-09T09:38:28.894773Z","iopub.execute_input":"2023-11-09T09:38:28.895236Z","iopub.status.idle":"2023-11-09T09:38:40.136192Z","shell.execute_reply.started":"2023-11-09T09:38:28.895201Z","shell.execute_reply":"2023-11-09T09:38:40.134832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_time_extract[\"date_time\"][1:]-train_time_extract[\"date_time\"][:-1]","metadata":{"execution":{"iopub.status.busy":"2023-11-07T19:32:29.971531Z","iopub.execute_input":"2023-11-07T19:32:29.971863Z","iopub.status.idle":"2023-11-07T19:32:30.069503Z","shell.execute_reply.started":"2023-11-07T19:32:29.971836Z","shell.execute_reply":"2023-11-07T19:32:30.068668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_time_extract[\"date_time\"][1:]-train_time_extract[\"date_time\"][:-1]).map_elements(lambda x: (x.seconds  / 60) > 1).unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T09:40:07.142005Z","iopub.execute_input":"2023-11-09T09:40:07.142452Z","iopub.status.idle":"2023-11-09T09:40:40.119114Z","shell.execute_reply.started":"2023-11-09T09:40:07.14241Z","shell.execute_reply":"2023-11-09T09:40:40.11794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series_id in train[\"series_id\"].unique():\n    temp = train_time_extract.filter(pl.col(\"series_id\") == series_id)\n    # print((temp[\"date_time\"][1:]-temp[\"date_time\"][:-1]).map_elements(lambda x: x.days > 1).any())\n    my_series = (temp[\"date_time\"][1:]-temp[\"date_time\"][:-1]).map_elements(lambda x: x.seconds < 5)\n    \"\"\"\n    seconds = (temp[\"date_time\"][1:]-temp[\"date_time\"][:-1]).map_elements(lambda x: x.seconds )\n    days = (temp[\"date_time\"][1:]-temp[\"date_time\"][:-1]).map_elements(lambda x: x.days )\n    print(seconds.unique())\n    print(days.unique())\n    continue\n    \"\"\"\n    b = pl.Series(\"b\", [False])\n    assign_series = my_series.extend(b) \n    temp = temp.with_columns(\n        time_stamp = assign_series,\n    )\n    #filtered_df = \n    print(temp.filter(pl.col(\"time_stamp\")))","metadata":{"execution":{"iopub.status.busy":"2023-11-09T09:54:50.528783Z","iopub.execute_input":"2023-11-09T09:54:50.529268Z","iopub.status.idle":"2023-11-09T09:54:50.957456Z","shell.execute_reply.started":"2023-11-09T09:54:50.529234Z","shell.execute_reply":"2023-11-09T09:54:50.95571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(train_time_extract)","metadata":{"execution":{"iopub.status.busy":"2023-11-05T21:33:51.537402Z","iopub.execute_input":"2023-11-05T21:33:51.537725Z","iopub.status.idle":"2023-11-05T21:33:51.543686Z","shell.execute_reply.started":"2023-11-05T21:33:51.537702Z","shell.execute_reply":"2023-11-05T21:33:51.542731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_time_extract.select(pl.corr([\"anglez\",\"enmo\",\"hour\"]))","metadata":{"execution":{"iopub.status.busy":"2023-11-06T22:44:43.602991Z","iopub.execute_input":"2023-11-06T22:44:43.60349Z","iopub.status.idle":"2023-11-06T22:44:43.640089Z","shell.execute_reply.started":"2023-11-06T22:44:43.603448Z","shell.execute_reply":"2023-11-06T22:44:43.63826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfeatures, feature_cols = [], []\n\n\nfor mins in [5, 30, 60*2, 60*8] :\n    for var in ['enmo', 'anglez'] :\n        features += [\n            pl.col(var).rolling_mean(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_mean'),\n            pl.col(var).rolling_max(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_max'),\n            pl.col(var).rolling_std(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_std')\n        ]\n\n        feature_cols += [ \n            f'{var}_{mins}m_mean', f'{var}_{mins}m_max', f'{var}_{mins}m_std'\n        ]\n\n        # Getting first variations\n        features += [\n            (pl.col(var).diff().abs().rolling_mean(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_mean'),\n            (pl.col(var).diff().abs().rolling_max(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_max'),\n            (pl.col(var).diff().abs().rolling_std(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_std')\n        ]\n\n        feature_cols += [ \n            f'{var}_1v_{mins}m_mean', f'{var}_1v_{mins}m_max', f'{var}_1v_{mins}m_std'\n        ]\n\n# id_cols = ['series_id', 'step', 'timestamp']\n\n\ndict_ids2data = {}\n\nfor i in tqdm(range(len(ids))):\n# for i in range(2):\n    print (i,': ', ids[i])\n    dict_ids2data[ids[i]] = train_time_extract.filter(pl.col(\"series_id\") == ids[i])\n    \n    dict_ids2data[ids[i]] = dict_ids2data[ids[i]].with_columns(\n        features\n    )\n    \n    dict_ids2data[ids[i]] = dict_ids2data[ids[i]].fill_null(strategy=\"forward\").fill_null(strategy=\"backward\")\n    \n\n\ntrain_data = pl.concat([i for i in dict_ids2data.values()], rechunk=True)\n\n\ndel dict_ids2data\ngc.collect()\n\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2023-11-05T21:33:22.642709Z","iopub.execute_input":"2023-11-05T21:33:22.643021Z","iopub.status.idle":"2023-11-05T21:33:22.741728Z","shell.execute_reply.started":"2023-11-05T21:33:22.642999Z","shell.execute_reply":"2023-11-05T21:33:22.74091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c_types = train_data.dtypes\nc_name = train_data.columns\n\nprint(len(c_name))\nprint('\\n'.join([\"{}\\t-\\t{}\".format(str(a_), str(b_)) for a_, b_ in zip(c_name, c_types)]))","metadata":{"execution":{"iopub.status.busy":"2023-11-05T17:36:47.463204Z","iopub.execute_input":"2023-11-05T17:36:47.463611Z","iopub.status.idle":"2023-11-05T17:36:47.471304Z","shell.execute_reply.started":"2023-11-05T17:36:47.46358Z","shell.execute_reply":"2023-11-05T17:36:47.469924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\njoblib.dump(train_data, 'train_luan_fe.pkl')\n","metadata":{"execution":{"iopub.status.busy":"2023-11-05T16:59:25.218368Z","iopub.execute_input":"2023-11-05T16:59:25.21873Z","iopub.status.idle":"2023-11-05T16:59:31.65304Z","shell.execute_reply.started":"2023-11-05T16:59:25.218688Z","shell.execute_reply":"2023-11-05T16:59:31.652203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_data.write_csv('train_luan.csv')\ntrain_data.write_parquet('train_luan.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-11-05T17:40:11.275393Z","iopub.execute_input":"2023-11-05T17:40:11.275937Z","iopub.status.idle":"2023-11-05T17:41:21.647435Z","shell.execute_reply.started":"2023-11-05T17:40:11.275898Z","shell.execute_reply":"2023-11-05T17:41:21.646552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_dtype = {\n#         \"series_id\": \"category\",\n#         \"step\": \"uint32\",\n#         \"timestamp\": \"str\",\n#         \"anglez\": \"float32\",\n#         \"enmo\": \"float32\",\n#     }\n# data = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet').astype(data_dtype, copy=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-05T16:59:50.543073Z","iopub.status.idle":"2023-11-05T16:59:50.543729Z","shell.execute_reply.started":"2023-11-05T16:59:50.543513Z","shell.execute_reply":"2023-11-05T16:59:50.543534Z"},"trusted":true},"execution_count":null,"outputs":[]}]}