{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebook description\n\nIn this notebook you will find:\n* Research of features from https://www.kaggle.com/code/smnuruzzaman/feature-engineering-and-ensemble\n* Removing redundant features\n* Creating and testing lag features\n* Creating and testing features obtained by arithmetic operations from the best ones in the previous step (aka bruteforce FE)\n\nIn all steps I will use 5-fold CV with lightgbm to get feature importance.","metadata":{}},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom itertools import groupby\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import f1_score, accuracy_score\nfrom pandas.api.types import is_datetime64_ns_dtype\n\nimport lightgbm as lgb\nfrom joblib import Parallel, delayed\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-10-30T22:42:50.366780Z","iopub.execute_input":"2023-10-30T22:42:50.367215Z","iopub.status.idle":"2023-10-30T22:42:50.375404Z","shell.execute_reply.started":"2023-10-30T22:42:50.367183Z","shell.execute_reply":"2023-10-30T22:42:50.374085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet'","metadata":{"execution":{"iopub.status.busy":"2023-10-30T22:42:50.377181Z","iopub.execute_input":"2023-10-30T22:42:50.377474Z","iopub.status.idle":"2023-10-30T22:42:50.388620Z","shell.execute_reply.started":"2023-10-30T22:42:50.377448Z","shell.execute_reply":"2023-10-30T22:42:50.387594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id  = pd.read_parquet(TRAIN_PATH, columns=['series_id'])\nseries_id = series_id['series_id'].unique()\n\nweird_series = ['31011ade7c0a', 'a596ad0b82aa']\nseries_id = [s for s in series_id if s not in weird_series]","metadata":{"execution":{"iopub.status.busy":"2023-10-30T22:42:52.589525Z","iopub.execute_input":"2023-10-30T22:42:52.589948Z","iopub.status.idle":"2023-10-30T22:42:54.812652Z","shell.execute_reply.started":"2023-10-30T22:42:52.589916Z","shell.execute_reply":"2023-10-30T22:42:54.811429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Selection\n## Basic Features","metadata":{}},{"cell_type":"markdown","source":"Mostly features from https://www.kaggle.com/code/smnuruzzaman/feature-engineering-and-ensemble. \n\nI removed the **median** as they are very close to the means, you can try with them.","metadata":{}},{"cell_type":"code","source":"def basic_fe(df, periods):\n    \n    df['series_id'] = df['series_id'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df['hour'] = df[\"timestamp\"].dt.hour\n    \n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float16)\n    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float16)\n    \n    for col in ['enmo', 'anglez', 'anglezdiffabs']:\n        for p in periods:\n            rol_args = {'window':f'{p+5}s', 'min_periods':10, 'center':True}\n            \n            for agg in ['mean', 'max', 'min']:\n                df[f'{col}_{p}_{agg}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float16).values\n                gc.collect()\n            for agg in ['var']:\n                df[f'{col}_{p}_{agg}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n\n            \n            df[f'{col}_{p}_amplit'] = df[f'{col}_{p}_max'] - df[f'{col}_{p}_min']\n            df[f'{col}_diff_{p}_max'] = df[f'{col}_{p}_max'].diff().abs().rolling(**rol_args).max().astype(np.float16)\n            df[f'{col}_diff_{p}_mean'] = df[f'{col}_{p}_min'].diff().abs().rolling(**rol_args).mean().astype(np.float16)\n            gc.collect()\n \n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n    gc.collect()\n\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2023-10-30T22:42:54.815012Z","iopub.execute_input":"2023-10-30T22:42:54.815369Z","iopub.status.idle":"2023-10-30T22:42:54.831826Z","shell.execute_reply.started":"2023-10-30T22:42:54.815339Z","shell.execute_reply":"2023-10-30T22:42:54.830718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng_by_id(idx, periods):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    df = pd.read_parquet(TRAIN_PATH, filters=[('series_id','=',idx)])\n    df['awake'] = df['awake'].astype(np.int8)\n    df = basic_fe(df, periods)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-30T22:42:54.833154Z","iopub.execute_input":"2023-10-30T22:42:54.833480Z","iopub.status.idle":"2023-10-30T22:42:54.848236Z","shell.execute_reply.started":"2023-10-30T22:42:54.833452Z","shell.execute_reply":"2023-10-30T22:42:54.847171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"periods = [60, 300, 600, 3600] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndf = Parallel(n_jobs=-2)(delayed(feat_eng_by_id)(i, periods) for i in series_id)\ndf = pd.concat(df, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-30T22:42:54.867322Z","iopub.execute_input":"2023-10-30T22:42:54.868268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The Correlation heatmap between all features would take too long to compute, so let's compute it for the feature sets corresponding to each time window.","metadata":{}},{"cell_type":"code","source":"len(df.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df.filter(like=f'_{periods[0]}_').columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [col for col in df.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]\n\n    \ns720 = {'timestamp', 'series_id', 'step', 'anglez', 'enmo', 'awake', 'hour', 'lids', 'anglezdiffabs', 'enmo_median_60', 'enmo_mean_60', 'enmo_max_60', 'enmo_min_60', 'enmo_var_60', 'enmo_amplit_60', 'enmo_amplit_60_min', 'enmo_diff_60_max', 'enmo_diff_60_mean', 'enmo_median_360', 'enmo_mean_360', 'enmo_max_360', 'enmo_min_360', 'enmo_var_360', 'enmo_amplit_360', 'enmo_amplit_360_min', 'enmo_diff_360_max', 'enmo_diff_360_mean', 'enmo_median_720', 'enmo_mean_720', 'enmo_max_720', 'enmo_min_720', 'enmo_var_720', 'enmo_amplit_720', 'enmo_amplit_720_min', 'enmo_diff_720_max', 'enmo_diff_720_mean', 'enmo_median_3600', 'enmo_mean_3600', 'enmo_max_3600', 'enmo_min_3600', 'enmo_var_3600', 'enmo_mad_3600', 'enmo_amplit_3600', 'enmo_amplit_3600_min', 'enmo_diff_3600_max', 'enmo_diff_3600_mean', 'anglez_median_60', 'anglez_mean_60', 'anglez_max_60', 'anglez_min_60', 'anglez_var_60', 'anglez_amplit_60', 'anglez_amplit_60_min', 'anglez_diff_60_max', 'anglez_diff_60_mean', 'anglez_median_360', 'anglez_mean_360', 'anglez_max_360', 'anglez_min_360', 'anglez_var_360', 'anglez_amplit_360', 'anglez_amplit_360_min', 'anglez_diff_360_max', 'anglez_diff_360_mean', 'anglez_median_720', 'anglez_mean_720', 'anglez_max_720', 'anglez_min_720', 'anglez_var_720', 'anglez_amplit_720', 'anglez_amplit_720_min', 'anglez_diff_720_max', 'anglez_diff_720_mean', 'anglez_median_3600', 'anglez_mean_3600', 'anglez_max_3600', 'anglez_min_3600', 'anglez_var_3600', 'anglez_mad_3600', 'anglez_amplit_3600', 'anglez_amplit_3600_min', 'anglez_diff_3600_max', 'anglez_diff_3600_mean', 'anglezdiffabs_median_60', 'anglezdiffabs_mean_60', 'anglezdiffabs_max_60', 'anglezdiffabs_min_60', 'anglezdiffabs_var_60', 'anglezdiffabs_amplit_60', 'anglezdiffabs_amplit_60_min', 'anglezdiffabs_diff_60_max', 'anglezdiffabs_diff_60_mean', 'anglezdiffabs_median_360', 'anglezdiffabs_mean_360', 'anglezdiffabs_max_360', 'anglezdiffabs_min_360', 'anglezdiffabs_var_360', 'anglezdiffabs_amplit_360', 'anglezdiffabs_amplit_360_min', 'anglezdiffabs_diff_360_max', 'anglezdiffabs_diff_360_mean', 'anglezdiffabs_median_720', 'anglezdiffabs_mean_720', 'anglezdiffabs_max_720', 'anglezdiffabs_min_720', 'anglezdiffabs_var_720', 'anglezdiffabs_amplit_720', 'anglezdiffabs_amplit_720_min', 'anglezdiffabs_diff_720_max', 'anglezdiffabs_diff_720_mean', 'anglezdiffabs_median_3600', 'anglezdiffabs_mean_3600', 'anglezdiffabs_max_3600', 'anglezdiffabs_min_3600', 'anglezdiffabs_var_3600', 'anglezdiffabs_mad_3600', 'anglezdiffabs_amplit_3600', 'anglezdiffabs_amplit_3600_min', 'anglezdiffabs_diff_3600_max', 'anglezdiffabs_diff_3600_mean'}\n\ns600 = set(features)\n\ns720.difference(s600)\ns600.difference(s720)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for p in periods:\n    plt.figure(figsize=(12,12))\n    plt.title(f\"Correlation heatmap for features with period {p}[s]\")\n    sns.heatmap(df[features].filter(like=f'_{p}_').corr(),\n    vmin=-1, vmax=1, center=0,\n    cmap=sns.color_palette(\"flare\", as_cmap=True),\n    square=True)\n    plt.show() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Insights:\n1. **enmo\\_{p}\\_max** and **enmo\\_{p}\\_amplit** are highly correlated for each aggregation window, which is logical since enmo is normally fluctuating around zero, with a limited number of sharp spikes. It's worth dropping one of them.\n2. **anglezdiffabs\\_{p}\\_max** and **anglezdiffabs\\_{p}\\_amplit** are highly correlated for each aggregation window.\n3. **anglezdiffabs\\_3600\\_min** and **anglezdiffabs\\_600\\_min** are just columns of zeros (see below), useless feature aggregation for large periods. \n4. **anglezdiffabs\\_diff\\_3600_mean** and **anglezdiffabs\\_diff\\_600\\_mean** - same issue.\n5. **enmo\\_3600\\_min** and **enmo\\_600\\_min** values are also all close to zero and make no sense.","metadata":{}},{"cell_type":"code","source":"features_to_drop = ['anglezdiffabs_3600_min', 'anglezdiffabs_600_min', 'anglezdiffabs_diff_3600_mean', 'anglezdiffabs_diff_600_mean',\n                        'enmo_3600_min', 'enmo_600_min']\nfor p in periods:\n    features_to_drop.append(f'anglezdiffabs_{p}_amplit')\n    features_to_drop.append(f'enmo_{p}_amplit')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.anglezdiffabs_3600_min.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.anglezdiffabs_600_min.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.anglezdiffabs_diff_300_mean.value_counts().head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.anglezdiffabs_diff_3600_mean.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.anglezdiffabs_diff_600_mean.value_counts().head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.enmo_3600_min.value_counts().head(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Top features by correlation with the target:","metadata":{}},{"cell_type":"code","source":"corr_with_target = df[features].corrwith(df['awake'])\ncorr_with_target.abs().sort_values().tail(50)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_for_heatmap = corr_with_target.abs().sort_values().tail(27).index.to_list()\nfeatures_for_heatmap = [f for f in features_for_heatmap if f not in ['anglezdiffabs_3600_min','anglezdiffabs_diff_3600_mean']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14,14))\nplt.title(f\"Correlation heatmap\")\nsns.heatmap(df[features_for_heatmap].corr(),\nvmin=-1, vmax=1, center=0,\ncmap=sns.color_palette(\"flare\", as_cmap=True),\nsquare=True)\nplt.show() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training","metadata":{}},{"cell_type":"markdown","source":"Let's train ensemble to check if our assumptions about features are valid.","metadata":{}},{"cell_type":"code","source":"# To spped up the process\ndf = df[::50]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = {\n'boosting_type':'gbdt',\n'num_leaves':35,\n'max_depth':6,\n'learning_rate':0.03,\n'n_estimators':850,\n'subsample_for_bin':200000,\n'min_child_weight':0.001,\n'min_child_samples':20,\n'subsample':0.9,\n'reg_alpha':0.05,\n'reg_lambda':0.05,\n'verbose': -1\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(df, cv):\n    \n    drop_cols = ['series_id', 'step', 'timestamp']\n    X, y = df.drop(columns=drop_cols+['awake']), df['awake']\n    gc.collect()\n    \n    cv_split = cv.split(X)\n    models = []\n    mean_f1 = 0\n    mean_acc = 0\n    \n    for i, (train_index, test_index) in enumerate(cv_split):\n\n        X_train, y_train = X.iloc[train_index], y.iloc[train_index]\n        X_test, y_test = X.iloc[test_index], y.iloc[test_index]\n        \n        print(f\"Fitting model {i+1} of {cv.n_splits}\")\n        \n        model = lgb.LGBMClassifier(random_state=42, n_jobs=-1, **lgb_params)\n        model.fit(X_train, y_train)\n        preds = model.predict(X_test)\n        f1 = f1_score(y_test, preds)\n        acc = accuracy_score(y_test, preds)\n        print(f\"Fold {i+1} finished training\\n F1:{f1_score(y_test, preds)} | Acc:{accuracy_score(y_test, preds)}\")\n        mean_f1 += f1\n        mean_acc += acc\n        models.append(model)\n        \n    print('-'*50)\n    print(f\"Training finished\\nF1: {mean_f1 / cv.n_splits}, Acc: {mean_acc / cv.n_splits}\")   \n    gc.collect()\n    \n    return models","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprint(f\"Fitting model with all {len(features)} features\")\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df, kf)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's investigate feature importances of models","metadata":{}},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance = fi.mean(axis=0).sort_values()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Least important features\nfeature_importance.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.drop(columns=features_to_drop)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [col for col in df.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprint(f\"Fitting model with {len(features)} features\")\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df, kf)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Same score with less features and less computation time. Nice.","metadata":{}},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])\nfeature_importance = fi.mean(axis=0).sort_values()\nplt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"## Lag Features\nNow let's add some new features and check their impact. I want to try lag features. Note that it's okay to use \"information from the future\" like **shift(-1)**, because the organizer is going to use the model on previously collected data for labeling.","metadata":{}},{"cell_type":"code","source":"for col in df.columns.to_list():\n    if col not in ['hour', 'series_id', 'step', 'timestamp', 'awake']:\n        df[f'{col}_lag_-1'] = df[col].shift(-1).astype(\"float16\")\n        df[f'{col}_lag_1'] = df[col].shift(1).astype(\"float16\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [col for col in df.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprint(f\"Fitting model with {len(features)} features\")\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df, kf)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It works (accuracy increased), but training the model started to take much longer. Let's try to remove some of the generated new features that have the lowest feature importance.","metadata":{}},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])\nfeature_importance = fi.mean(axis=0).sort_values()\nplt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance.filter(like='_lag_1').head(25)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance.filter(like='_lag_-1').head(25)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_to_drop = feature_importance.filter(like='_lag_1').head(25).index.to_list() +\\\n                   feature_importance.filter(like='_lag_-1').head(25).index.to_list()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.drop(columns=features_to_drop)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprint(f\"Fitting model with {len(df.columns)-4} features\")\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df, kf)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Accuracy score with 50 removed features is the same.\n\nNow let's try to combine top performing features with each other using basic arithmetic operations.","metadata":{}},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=model.booster_.feature_name()) for model in models])\nfeature_importance = fi.mean(axis=0).sort_values()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_importance.tail(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candidates = feature_importance.tail(20).index.to_list()\nfor feature1 in candidates:\n    df[f'{feature1}_SQR'] = df[feature1] * df[feature1]\n    for feature2 in candidates:\n        if feature1 != feature2:\n            df[f'{feature1}_MUL_{feature2}'] = df[feature1] * df[feature2]\n            df[f'{feature1}_DIV_{feature2}'] = df[feature1] / df[feature2]        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [col for col in df.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprint(f\"Fitting model with {len(features)} features\")\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df, kf)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Before\n#F1: 0.9880503108878849, Acc: 0.9843715584218957\n#After\n#F1: 0.9900570304275906, Acc: 0.9869845275025252","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is a clear increase in the model's metrics, but most of the features generated in this way are obviously worthless. Let's take out 80% and see what happens.","metadata":{}},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])\nfeature_importance = fi.mean(axis=0).sort_values()\nplt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_to_drop = feature_importance.head(800).index.to_list()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_light = df.drop(columns=features_to_drop)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_light.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [col for col in df_light.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nprint(f\"Fitting model with {len(features)} features\")\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df_light, kf)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"80% of the generated features are removed and the model shows the same high accuracy.","metadata":{}},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])\nfeature_importance = fi.mean(axis=0).sort_values()\nplt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Final set of features:","metadata":{}},{"cell_type":"code","source":"features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A few observations: \n* Among the remaining features, most belong to the 3600 second window. Aggregations over shorter periods are also present, though.\n* The models \"liked\" features normalized by **hour**, maybe you should add them, even if you don't intend to use the whole set derived here. \n* This notebook has many degrees of freedom: you could try to remove lag(-1) or for example add another time window for aggregation at the very beginning. \n\nGood luck with your experiments.","metadata":{}},{"cell_type":"markdown","source":"Please leave an upvote if you found this work useful. It motivates me to post new versions. Feel free to suggest new ideas for experiments, I will try to include the most interesting things in this notebook.","metadata":{}}]}