{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nimport gc\nfrom tqdm.auto import tqdm \n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-10T08:20:24.998771Z","iopub.execute_input":"2023-11-10T08:20:24.999100Z","iopub.status.idle":"2023-11-10T08:20:25.576185Z","shell.execute_reply.started":"2023-11-10T08:20:24.999074Z","shell.execute_reply":"2023-11-10T08:20:25.574876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = pl.read_parquet('/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:25.578067Z","iopub.execute_input":"2023-11-10T08:20:25.578500Z","iopub.status.idle":"2023-11-10T08:20:27.754866Z","shell.execute_reply.started":"2023-11-10T08:20:25.578472Z","shell.execute_reply":"2023-11-10T08:20:27.753936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.replace(\"awake\", train[\"awake\"].cast(pl.Int8))","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:27.756213Z","iopub.execute_input":"2023-11-10T08:20:27.757189Z","iopub.status.idle":"2023-11-10T08:20:27.868551Z","shell.execute_reply.started":"2023-11-10T08:20:27.757150Z","shell.execute_reply":"2023-11-10T08:20:27.867554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train[\"step\"][1:]-train[\"step\"][:-1]).ne(1).any()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:27.870573Z","iopub.execute_input":"2023-11-10T08:20:27.871524Z","iopub.status.idle":"2023-11-10T08:20:27.940930Z","shell.execute_reply.started":"2023-11-10T08:20:27.871495Z","shell.execute_reply":"2023-11-10T08:20:27.939686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train[\"step\"][1:]-train[\"step\"][:-1]).ne(1).unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:27.942008Z","iopub.execute_input":"2023-11-10T08:20:27.942316Z","iopub.status.idle":"2023-11-10T08:20:28.044767Z","shell.execute_reply.started":"2023-11-10T08:20:27.942290Z","shell.execute_reply":"2023-11-10T08:20:28.043861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series_id in train[\"series_id\"].unique():\n    temp = train.filter(pl.col(\"series_id\") == series_id)\n    print((temp[\"step\"][1:]-temp[\"step\"][:-1]).ne(1).any())","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:28.045712Z","iopub.execute_input":"2023-11-10T08:20:28.046059Z","iopub.status.idle":"2023-11-10T08:20:30.161342Z","shell.execute_reply.started":"2023-11-10T08:20:28.046031Z","shell.execute_reply":"2023-11-10T08:20:30.160417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train.clone()\ngrouped_df = df.group_by(\"series_id\").agg(pl.col(\"step\").diff())\ngrouped_df","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:30.163191Z","iopub.execute_input":"2023-11-10T08:20:30.163647Z","iopub.status.idle":"2023-11-10T08:20:30.531136Z","shell.execute_reply.started":"2023-11-10T08:20:30.163603Z","shell.execute_reply":"2023-11-10T08:20:30.529979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def luan_unique(seq):\n    seen = set()\n    seen_add = seen.add\n    return [x for x in seq if not (x in seen or seen_add(x))]\n\n\nids = luan_unique(train['series_id'])\n\nprint(ids)\n\nlen(ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:30.532449Z","iopub.execute_input":"2023-11-10T08:20:30.532917Z","iopub.status.idle":"2023-11-10T08:20:32.569208Z","shell.execute_reply.started":"2023-11-10T08:20:30.532893Z","shell.execute_reply":"2023-11-10T08:20:32.568310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_smooth = train\n","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:32.570779Z","iopub.execute_input":"2023-11-10T08:20:32.573275Z","iopub.status.idle":"2023-11-10T08:20:32.579542Z","shell.execute_reply.started":"2023-11-10T08:20:32.573223Z","shell.execute_reply":"2023-11-10T08:20:32.578091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef process_timestamp(df):\n    df = df.with_columns(\n        #year = (df[\"timestamp\"].str.slice(0,4).cast(pl.Int16) - 2000).cast(pl.UInt8),\n        #month = df[\"timestamp\"].str.slice(5,2).cast(pl.Int8),\n        #day = df[\"timestamp\"].str.slice(8,2).cast(pl.Int8),\n        hour = df[\"timestamp\"].str.slice(11,2).cast(pl.Int8),\n        #minute = df[\"timestamp\"].str.slice(14,2).cast(pl.Int8),\n        #second = df[\"timestamp\"].str.slice(17,2).cast(pl.Int8),\n        #time_zone = df[\"timestamp\"].str.slice(-5,3).cast(pl.Int32),\n        enmo =  (pl.col('enmo')*1000).cast(pl.UInt16),\n        date_time =pl.col('timestamp').str.strptime(pl.Datetime),\n    )\n    return df\n    \n    \n# print (process_timestamp(train_smooth))\n\ntrain_time_extract = process_timestamp(train_smooth)\n\ntrain_time_extract","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:21:36.258208Z","iopub.execute_input":"2023-11-10T08:21:36.258640Z","iopub.status.idle":"2023-11-10T08:21:41.240182Z","shell.execute_reply.started":"2023-11-10T08:21:36.258610Z","shell.execute_reply":"2023-11-10T08:21:41.239183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_time_extract[\"date_time\"][1:]-train_time_extract[\"date_time\"][:-1]","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:20:37.030604Z","iopub.execute_input":"2023-11-10T08:20:37.032591Z","iopub.status.idle":"2023-11-10T08:20:37.079773Z","shell.execute_reply.started":"2023-11-10T08:20:37.032549Z","shell.execute_reply":"2023-11-10T08:20:37.078138Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(13 in range(12,15))","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:04:26.614728Z","iopub.execute_input":"2023-11-10T09:04:26.615040Z","iopub.status.idle":"2023-11-10T09:04:26.621812Z","shell.execute_reply.started":"2023-11-10T09:04:26.615014Z","shell.execute_reply":"2023-11-10T09:04:26.620183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_df = train_time_extract.group_by(\"series_id\").agg(pl.col(\"hour\").diff().alias('diff_hour'), pl.col(\"hour\"), pl.col(\"date_time\").diff().alias('diff_time')\n                                                          , pl.col(\"step\"), pl.col(\"date_time\"))\n# grouped_df = train_time_extract.group_by(\"series_id\").agg(pl.col(\"date_time\").diff())\n\n# grouped_df\n\nfor i,l in zip(range(len(grouped_df)), grouped_df['diff_hour']):\n# for l in grouped_df['date_time']:\n    if (str(grouped_df['series_id'][i]) != '1087d7b0ff2e'): continue\n    print (i,' ',grouped_df['series_id'][i],' ',list(l.unique()),' ', list(grouped_df['diff_time'][i].unique()))\n    \n    for i_l, e_l in zip(range(len(l)), l):\n        if e_l == 2: \n            print(i_l)\n            print (grouped_df['step'][i][i_l])\n            print (grouped_df['hour'][i][i_l-3:i_l+3])\n            print (grouped_df['date_time'][i][i_l-3:i_l+3])\n            \n            \nprint (list(train_time_extract.filter((pl.col('series_id') == '1087d7b0ff2e') & (pl.col('step') > 78300-3) & (pl.col('step') < 78300+3))['timestamp']))\n\ntrain_time_extract.filter((pl.col('series_id') == '1087d7b0ff2e') & (pl.col('step') > 78300-3) & (pl.col('step') < 78300+3)).select('timestamp', 'date_time')","metadata":{"execution":{"iopub.status.busy":"2023-11-10T09:11:04.485668Z","iopub.execute_input":"2023-11-10T09:11:04.485993Z","iopub.status.idle":"2023-11-10T09:11:05.086560Z","shell.execute_reply.started":"2023-11-10T09:11:04.485967Z","shell.execute_reply":"2023-11-10T09:11:05.085691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(train_time_extract)","metadata":{"execution":{"iopub.status.busy":"2023-11-10T08:21:10.897040Z","iopub.execute_input":"2023-11-10T08:21:10.898347Z","iopub.status.idle":"2023-11-10T08:21:10.906088Z","shell.execute_reply.started":"2023-11-10T08:21:10.898298Z","shell.execute_reply":"2023-11-10T08:21:10.904549Z"},"trusted":true},"execution_count":null,"outputs":[]}]}