{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6507308,"sourceType":"datasetVersion","datasetId":3706667}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nimport gc\nfrom tqdm.auto import tqdm \nimport joblib\nfrom datetime import timedelta, date\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:19:26.036169Z","iopub.execute_input":"2023-11-19T17:19:26.036565Z","iopub.status.idle":"2023-11-19T17:19:26.047086Z","shell.execute_reply.started":"2023-11-19T17:19:26.036536Z","shell.execute_reply":"2023-11-19T17:19:26.045976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def feature_extraction(df, istrain = True, step_rolling = 5):\n#     ids = df['series_id'].unique(maintain_order = True)\n    \n#     if (istrain):\n#         df = df.with_columns(\n#             awake = df[\"awake\"].cast(pl.Int8)\n#         )\n\n    \n#     df = df.with_columns(        \n#         year = (df[\"timestamp\"].str.slice(0,4).cast(pl.Int16) - 2000).cast(pl.UInt8),\n#         month = df[\"timestamp\"].str.slice(5,2).cast(pl.Int8),\n#         day = df[\"timestamp\"].str.slice(8,2).cast(pl.Int8),\n#         hour = df[\"timestamp\"].str.slice(11,2).cast(pl.Int8),\n#         minute = df[\"timestamp\"].str.slice(14,2).cast(pl.Int8),\n# #         second = df[\"timestamp\"].str.slice(17,2).cast(pl.Int8),\n#         time_zone = df[\"timestamp\"].str.slice(-5,3).cast(pl.Int32),\n        \n#         enmo =  (pl.col('enmo')*1000).cast(pl.UInt16),\n        \n#         weekday = train[\"timestamp\"].str.slice(0,10).str.to_date().dt.weekday()\n        \n#     )\n    \n#     signal_awake = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.208 * np.pi) ** 24))\n#     signal_onset = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.555 * np.pi) ** 24))\n#     df = df.with_columns(\n#         (pl.col(\"hour\").cast(pl.Int32)*60 + pl.col(\"minute\").cast(pl.Int32))\n#         .apply(signal_onset.get, return_dtype=pl.Float32)\n#         .alias(\"signal_onset\")\n#     )    \n#     df = df.with_columns(\n#         (pl.col(\"hour\").cast(pl.Int32)*60 + pl.col(\"minute\").cast(pl.Int32))\n#         .apply(signal_awake.get, return_dtype=pl.Float32)\n#         .alias(\"signal_awake\")\n#     )\n\n#     df = df.with_columns(lids = np.maximum(0., tmp['enmo'] - 0.02))\n#     df = df.with_columns(\n#         lids = pl.col('lids').rolling_sum(121, center=True, min_periods=1)\n#     )\n#     df = df.with_columns(lids = 100 / (pl.col('lids') + 1))\n#     df = df.with_columns(\n#         lids = pl.col('lids').rolling_mean(361, center=True, min_periods=1).cast(pl.Float32)\n#     )\n#     gc.collect()\n#     return df\n\n# train = feature_extraction(train)\n# train\n# # list(train['signal_onset'][:10])","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:19:26.049835Z","iopub.execute_input":"2023-11-19T17:19:26.050648Z","iopub.status.idle":"2023-11-19T17:19:26.058448Z","shell.execute_reply.started":"2023-11-19T17:19:26.050612Z","shell.execute_reply":"2023-11-19T17:19:26.057317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feature_extraction(df, istrain = True, step_rolling = 5):    \n# def feature_extraction(df, istrain = True):    \n\n    ids = df['series_id'].unique(maintain_order = True)\n    \n    if (istrain):\n        df = df.with_columns(\n            awake = df[\"awake\"].cast(pl.Int8)\n        )\n\n    df = df.with_columns(\n        lids = np.maximum(0., df['enmo'] - 0.02),\n        year = (df[\"timestamp\"].str.slice(0,4).cast(pl.Int16) - 2000).cast(pl.UInt8),\n        month = df[\"timestamp\"].str.slice(5,2).cast(pl.Int8),\n        day = df[\"timestamp\"].str.slice(8,2).cast(pl.Int8),\n        hour = df[\"timestamp\"].str.slice(11,2).cast(pl.Int8),\n        minute = df[\"timestamp\"].str.slice(14,2).cast(pl.Int8),\n        second = df[\"timestamp\"].str.slice(17,2).cast(pl.Int8),\n        time_zone = df[\"timestamp\"].str.slice(-5,3).cast(pl.Int32),\n        weekday = df[\"timestamp\"].str.slice(0,10).str.to_date().dt.weekday(),\n\n        enmo =  (pl.col('enmo')*1000).cast(pl.UInt16),\n        )\n    \n    signal_awake = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.208 * np.pi) ** 24))\n    signal_onset = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.555 * np.pi) ** 24))\n    df = df.with_columns(\n        (pl.col(\"hour\").cast(pl.Int32)*60 + pl.col(\"minute\").cast(pl.Int32))\n        .apply(signal_onset.get, return_dtype=pl.Float32)\n        .alias(\"signal_onset\"),\n        (pl.col(\"hour\").cast(pl.Int32)*60 + pl.col(\"minute\").cast(pl.Int32))\n        .apply(signal_awake.get, return_dtype=pl.Float32)\n        .alias(\"signal_awake\"),\n        (df[\"hour\"].cast(pl.Int32)*60 + df[\"minute\"].cast(pl.Int32)).alias('hour_minute')\n    )\n    \n    features, feature_cols = [], []\n\n    for mins in [1, 3, 5, 30, 60*2, 60*8] :\n        for var in ['enmo', 'anglez'] :\n            features += [\n                pl.col(var).rolling_mean(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_mean'),\n                pl.col(var).rolling_max(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_max'),\n                pl.col(var).rolling_std(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'{var}_{mins}m_std')\n            ]\n            feature_cols += [ \n                f'{var}_{mins}m_mean', f'{var}_{mins}m_max', f'{var}_{mins}m_std'\n            ]\n            # Getting first variations\n            features += [\n                (pl.col(var).diff().abs().rolling_mean(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_mean'),\n                (pl.col(var).diff().abs().rolling_max(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_max'),\n                (pl.col(var).diff().abs().rolling_std(12 * mins, center=True, min_periods=1)*10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_std')\n            ]\n            feature_cols += [ \n                f'{var}_1v_{mins}m_mean', f'{var}_1v_{mins}m_max', f'{var}_1v_{mins}m_std'\n            ]\n    # id_cols = ['series_id', 'step', 'timestamp']\n\n    dict_ids2data = {}    \n    for i in tqdm(range(len(ids))):\n    # for i in range(2):\n        print (i,': ', ids[i])\n        dict_ids2data[ids[i]] = df.filter(pl.col(\"series_id\") == ids[i])\n        dict_ids2data[ids[i]] = dict_ids2data[ids[i]].with_columns(\n            features,\n            lids = pl.col('lids').rolling_sum(121, center=True, min_periods=1)\n        )\n                \n        dict_ids2data[ids[i]] = dict_ids2data[ids[i]].with_columns(\n            (100 / (pl.col('lids') + 1)).alias('lids'),\n            pl.col(\"anglez\").rolling_mean(window_size = step_rolling, center = True).cast(pl.Float32).alias(f'anglez_{step_rolling*5}s_smooth'),\n            pl.col(\"enmo\").rolling_mean(window_size = step_rolling, center = True).cast(pl.Float32).alias(f'enmo_{step_rolling*5}s_smooth')\n        )\n        \n        dict_ids2data[ids[i]] = dict_ids2data[ids[i]].fill_null(strategy=\"forward\").fill_null(strategy=\"backward\")\n        \n        dict_ids2data[ids[i]] = dict_ids2data[ids[i]].with_columns(\n            lids = pl.col('lids').rolling_mean(361, center=True, min_periods=1).cast(pl.Float32),\n#             anglez_lag_5s = dict_ids2data[ids[i]]['anglez'].shift(periods=1),\n#             anglez_lag_10s = dict_ids2data[ids[i]]['anglez'].shift(periods=2),\n#             anglez_lag_15s = dict_ids2data[ids[i]]['anglez'].shift(periods=3),\n#             anglez_lag_30s = dict_ids2data[ids[i]]['anglez'].shift(periods=6),\n#             anglez_lag_60s = dict_ids2data[ids[i]]['anglez'].shift(periods=12),\n            anglez_smooth_lag_30s = dict_ids2data[ids[i]][f'anglez_{step_rolling*5}s_smooth'].cast(pl.Float32).shift(periods=6),\n            anglez_smooth_lag_60s = dict_ids2data[ids[i]][f'anglez_{step_rolling*5}s_smooth'].cast(pl.Float32).shift(periods=12),\n\n#             enmo_lag_5s = dict_ids2data[ids[i]]['enmo'].shift(periods=1),\n#             enmo_lag_10s = dict_ids2data[ids[i]]['enmo'].shift(periods=2),\n#             enmo_lag_15s = dict_ids2data[ids[i]]['enmo'].shift(periods=3),\n#             enmo_lag_30s = dict_ids2data[ids[i]]['enmo'].shift(periods=6),\n#             enmo_lag_60s = dict_ids2data[ids[i]]['enmo'].shift(periods=12),\n            enmo_smooth_lag_30s = dict_ids2data[ids[i]][f'enmo_{step_rolling*5}s_smooth'].cast(pl.Float32).shift(periods=6),\n            enmo_smooth_lag_60s = dict_ids2data[ids[i]][f'enmo_{step_rolling*5}s_smooth'].cast(pl.Float32).shift(periods=12)\n        )\n        \n        dict_ids2data[ids[i]] = dict_ids2data[ids[i]].fill_null(strategy=\"forward\").fill_null(strategy=\"backward\")\n\n        \n    df = pl.concat([i for i in dict_ids2data.values()], rechunk=True)\n\n    del dict_ids2data\n    gc.collect()\n\n    return df\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:19:26.064957Z","iopub.execute_input":"2023-11-19T17:19:26.065751Z","iopub.status.idle":"2023-11-19T17:19:26.092545Z","shell.execute_reply.started":"2023-11-19T17:19:26.065704Z","shell.execute_reply":"2023-11-19T17:19:26.091382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = pl.read_parquet('/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:19:26.222999Z","iopub.execute_input":"2023-11-19T17:19:26.223422Z","iopub.status.idle":"2023-11-19T17:19:27.304813Z","shell.execute_reply.started":"2023-11-19T17:19:26.223390Z","shell.execute_reply":"2023-11-19T17:19:27.303713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fe = feature_extraction(train)\n\ntrain_fe","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:19:27.306794Z","iopub.execute_input":"2023-11-19T17:19:27.307327Z","iopub.status.idle":"2023-11-19T17:20:09.717005Z","shell.execute_reply.started":"2023-11-19T17:19:27.307296Z","shell.execute_reply":"2023-11-19T17:20:09.715795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c_types = train_fe.dtypes\nc_name = train_fe.columns\n\nprint(len(c_name))\nprint('\\n'.join([\"{}\\t-\\t{}\".format(str(a_), str(b_)) for a_, b_ in zip(c_name, c_types)]))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:20:09.720885Z","iopub.execute_input":"2023-11-19T17:20:09.721243Z","iopub.status.idle":"2023-11-19T17:20:09.727994Z","shell.execute_reply.started":"2023-11-19T17:20:09.721215Z","shell.execute_reply":"2023-11-19T17:20:09.726537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_raw = pl.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\ntest_raw.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:20:09.729768Z","iopub.execute_input":"2023-11-19T17:20:09.730892Z","iopub.status.idle":"2023-11-19T17:20:09.746155Z","shell.execute_reply.started":"2023-11-19T17:20:09.730819Z","shell.execute_reply":"2023-11-19T17:20:09.744941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_fe = feature_extraction(test_raw, False)\n\ntest_fe","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:20:09.748928Z","iopub.execute_input":"2023-11-19T17:20:09.749297Z","iopub.status.idle":"2023-11-19T17:20:09.959999Z","shell.execute_reply.started":"2023-11-19T17:20:09.749265Z","shell.execute_reply":"2023-11-19T17:20:09.958902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:20:09.961254Z","iopub.execute_input":"2023-11-19T17:20:09.961578Z","iopub.status.idle":"2023-11-19T17:20:10.084584Z","shell.execute_reply.started":"2023-11-19T17:20:09.961552Z","shell.execute_reply":"2023-11-19T17:20:10.083690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nplt.rcParams[\"figure.figsize\"] = [12.00, 5]\nplt.rcParams[\"figure.autolayout\"] = True\n\ndf = pd.DataFrame(dict(day=np.array(train_fe[\"day\"].take_every(12*60))))\n\nax = sns.countplot(x=\"day\", data=df)\n\nfor p in ax.patches:\n   ax.annotate('{:.1f}'.format(p.get_height()), (p.get_x()+0.25, p.get_height()+0.01))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:20:10.086320Z","iopub.execute_input":"2023-11-19T17:20:10.086689Z","iopub.status.idle":"2023-11-19T17:20:10.768973Z","shell.execute_reply.started":"2023-11-19T17:20:10.086625Z","shell.execute_reply":"2023-11-19T17:20:10.767832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nplt.rcParams[\"figure.figsize\"] = [12.00, 5]\nplt.rcParams[\"figure.autolayout\"] = True\n\ndf = pd.DataFrame(dict(month=np.array(train_fe[\"month\"].take_every(12*60*24))))\n\nax = sns.countplot(x=\"month\", data=df)\n\nfor p in ax.patches:\n   ax.annotate('{:.1f}'.format(p.get_height()), (p.get_x()+0.25, p.get_height()+0.01))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:20:10.770448Z","iopub.execute_input":"2023-11-19T17:20:10.770956Z","iopub.status.idle":"2023-11-19T17:20:11.182665Z","shell.execute_reply.started":"2023-11-19T17:20:10.770924Z","shell.execute_reply":"2023-11-19T17:20:11.181483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_fe.write_csv('test_series.csv')\ntest_fe.write_parquet('test_series.parquet')\ntrain_fe.write_parquet('train_series.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:20:11.184187Z","iopub.execute_input":"2023-11-19T17:20:11.185291Z","iopub.status.idle":"2023-11-19T17:20:39.973622Z","shell.execute_reply.started":"2023-11-19T17:20:11.185253Z","shell.execute_reply":"2023-11-19T17:20:39.972210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_dtype = {\n#         \"series_id\": \"category\",\n#         \"step\": \"uint32\",\n#         \"timestamp\": \"str\",\n#         \"anglez\": \"float32\",\n#         \"enmo\": \"float32\",\n#     }\n# data = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet').astype(data_dtype, copy=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T17:20:39.975642Z","iopub.execute_input":"2023-11-19T17:20:39.976637Z","iopub.status.idle":"2023-11-19T17:20:39.982089Z","shell.execute_reply.started":"2023-11-19T17:20:39.976586Z","shell.execute_reply":"2023-11-19T17:20:39.980833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}