{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Statistical Feature Engineering\n\nThis notebook aims to create some features as a starter, such as backward difference and rolling average.\n\nAll of the credits for the inspirations goes to:\n- [Zzzs: Random Forest model starter](https://www.kaggle.com/code/carlmcbrideellis/zzzs-random-forest-model-starter)\n- [LightGBM starter - CMI](https://www.kaggle.com/code/renatoreggiani/lightgbm-starter-cmi)\n- [Zzzs: Make clean starter dataset + target](https://www.kaggle.com/code/carlmcbrideellis/zzzs-make-clean-starter-dataset-target) for the lightweight dataset","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport xgboost as xgb\nimport optuna\n\nfrom time import time\nfrom itertools import groupby, product\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-09T18:49:11.594750Z","iopub.execute_input":"2023-09-09T18:49:11.595582Z","iopub.status.idle":"2023-09-09T18:49:16.409033Z","shell.execute_reply.started":"2023-09-09T18:49:11.595539Z","shell.execute_reply":"2023-09-09T18:49:16.407701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet\")\ntest  = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\nevents = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\n\n# train = train.set_index(['series_id', 'step']).sort_index()\n# test = test.set_index(['series_id', 'step']).sort_index()\n\nst = time()\ntrain[\"timestamp\"] = pd.to_datetime(train[\"timestamp\"], utc=True)\ntest[\"timestamp\"] = pd.to_datetime(test[\"timestamp\"], utc=True)\nevents[\"timestamp\"] = pd.to_datetime(events[\"timestamp\"], utc=True)\nen = time()\nprint(f'Done in {int(en - st)}s')\n\nprint(train.shape, test.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-09T19:03:30.351211Z","iopub.execute_input":"2023-09-09T19:03:30.351692Z","iopub.status.idle":"2023-09-09T19:05:29.085257Z","shell.execute_reply.started":"2023-09-09T19:03:30.351652Z","shell.execute_reply":"2023-09-09T19:05:29.083664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats = ['anglez', 'enmo', 'anglez_x_enmo', 'abs_anglez_x_enmo', 'relu_anglez_x_enmo']\nstats = ['rolling_mean', 'diff_1st', 'diff_2nd', 'rolling_mean_diff_1st', 'rolling_mean_diff_2nd']\nminutes_periods = zip(\n    ['1min', '3min', '5min'],\n    [12, 36, 60]\n) # you can adjust the `stats` and `minutes_periods` as well\n# 1 step = 5 sec, 1 min = 12 steps\n\ncombinations = list(product(feats, stats, minutes_periods))\n\ndef create_features(df, verbose=False):\n    start_time = time()\n    \n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"minute\"] = df[\"timestamp\"].dt.minute\n    df['weekday'] = df['timestamp'].dt.weekday\n    df['is_weekend'] = np.where(df['weekday'] > 5, 1, 0)\n    df['month'] = df['timestamp'].dt.month\n    df['day'] = df['timestamp'].dt.day\n    \n    df[\"anglez_x_enmo\"] = df[\"anglez\"] * df[\"enmo\"]\n    df[\"abs_anglez_x_enmo\"] = abs(df[\"anglez\"]) * df[\"enmo\"]\n    df[\"relu_anglez_x_enmo\"] = np.maximum(df[\"anglez\"], 0.) * df[\"enmo\"]\n    \n    # (anglez, enmo, anglez_x_enmo, abs_anglez_x_enmo, relu_anglez_x_enmo\n    # (rolling_mean_*, diff_1st_*, diff_2nd_*, rolling_diff_1st_*, rolling_diff_2nd_*)\n    # 2 + 6 + 3 + 5 x 5 x 3 = 86 feats\n    feat_names = []\n    \n    for feat, stat, (minutes, periods) in combinations:\n        feat_name = f'{feat}_{stat}_{minutes}'\n        feat_names.append(feat_name)\n\n        st_ = time()\n        if stat == 'rolling_mean':\n            df[feat_name] = df.groupby('series_id')[feat].rolling(periods, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n        if stat == 'diff_1st':\n            df[feat_name] = df.groupby('series_id')[feat].diff(periods=periods).fillna(method='bfill').values\n        if stat == 'diff_2nd':\n            feat_ = f'{feat}_diff_1st_{minutes}'\n            df[feat_name] = df.groupby('series_id')[feat_].diff(periods=periods).fillna(method='bfill').values\n        if stat == 'rolling_mean_diff_1st':\n            feat_ = f'{feat}_diff_1st_{minutes}'\n            df[feat_name] = df.groupby('series_id')[feat_].rolling(periods, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n        if stat == 'rolling_mean_diff_2nd':\n            feat_ = f'{feat}_diff_2nd_{minutes}'\n            df[feat_name] = df.groupby('series_id')[feat_].rolling(periods, center=True).mean().fillna(method='bfill').fillna(method='ffill').values\n        en_ = time()\n        if verbose:\n            eta = int(en_ - st_)\n            print(f'Done creating {feat_name} in {eta}s')\n                    \n    feat_names = [\n        \"anglez\", \"enmo\", \"hour\", \"minute\", 'weekday', 'is_weekend', 'month', 'day',\n        'anglez_x_enmo', 'abs_anglez_x_enmo', 'relu_anglez_x_enmo'\n    ] + feat_names\n    \n    end_time = time()\n    eta = int(end_time - start_time)\n    if verbose:\n        print(f'Done in: {eta}s')\n    \n    return feat_names","metadata":{"execution":{"iopub.status.busy":"2023-09-09T19:07:03.636902Z","iopub.execute_input":"2023-09-09T19:07:03.637449Z","iopub.status.idle":"2023-09-09T19:07:03.661102Z","shell.execute_reply.started":"2023-09-09T19:07:03.637404Z","shell.execute_reply":"2023-09-09T19:07:03.659651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = create_features(test, verbose=False)\nfeatures = create_features(train, verbose=True)\n\nprint(features)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T19:07:33.390363Z","iopub.execute_input":"2023-09-09T19:07:33.390840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}