{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Feature Engineering, Model Training and Prediction\n\nData Preparation and Analysis can be found in https://www.kaggle.com/shenghaonan/sleepyu-data-analysis","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport datetime \nfrom tqdm import tqdm\nimport seaborn as sns\nfrom joblib import Parallel, delayed\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport gc\nfrom metric import score # Import event detection ap score function\n\n# These are variables to be used by the score function\ncolumn_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n}\n\ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}\n\npd.set_option('display.max_columns', None)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T12:56:07.418293Z","iopub.execute_input":"2023-11-04T12:56:07.418993Z","iopub.status.idle":"2023-11-04T12:56:07.433485Z","shell.execute_reply.started":"2023-11-04T12:56:07.418904Z","shell.execute_reply":"2023-11-04T12:56:07.431320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Importing data \n\n# Column transformations\n\ndt_transforms = [\n      pl.col('timestamp').str.to_datetime(), \n    (pl.col('timestamp').str.to_datetime().dt.date()).alias('date'), \n    pl.col('timestamp').str.to_datetime().dt.time().alias('time')\n    ]\n\ntrain_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n                .with_columns(\n    (dt_transforms)\n    )\n                .collect()\n                .to_pandas()\n               )\n\ntrain_events = (pl.scan_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n                .with_columns(\n                    (dt_transforms)\n                    )\n                .collect()\n                .to_pandas()\n               )\n\ntest_series = (pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\n                .with_columns(\n                    (dt_transforms)\n                    )\n                .collect()\n                .to_pandas()\n               )","metadata":{"execution":{"iopub.status.busy":"2023-11-04T12:56:07.435915Z","iopub.execute_input":"2023-11-04T12:56:07.436297Z","iopub.status.idle":"2023-11-04T12:57:21.224122Z","shell.execute_reply.started":"2023-11-04T12:56:07.436269Z","shell.execute_reply":"2023-11-04T12:57:21.223254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def basic_fe(df, periods):\n    \n    df['series_id'] = df['series_id'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n#     df['hour'] = df[\"timestamp\"].dt.hour\n    \n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float16)\n    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float16)\n    \n    for col in ['enmo', 'anglez', 'anglezdiffabs']:\n        for p in periods:\n            rol_args = {'window':f'{p+5}s', 'min_periods':10, 'center':True}\n            \n            for agg in ['mean', 'max', 'min']:\n                df[f'{col}_{p}_{agg}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float16).values\n                gc.collect()\n            # There was an overflow error when trying to cast to float16, so I decided to leave it as float32\n            for agg in ['var']:\n                df[f'{col}_{p}_{agg}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n\n            \n            df[f'{col}_{p}_amplit'] = df[f'{col}_{p}_max'] - df[f'{col}_{p}_min']\n            df[f'{col}_diff_{p}_max'] = df[f'{col}_{p}_max'].diff().abs().rolling(**rol_args).max().astype(np.float16)\n            df[f'{col}_diff_{p}_mean'] = df[f'{col}_{p}_min'].diff().abs().rolling(**rol_args).mean().astype(np.float16)\n            gc.collect()\n \n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n    gc.collect()\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-04T12:57:21.225449Z","iopub.execute_input":"2023-11-04T12:57:21.226152Z","iopub.status.idle":"2023-11-04T12:57:21.240642Z","shell.execute_reply.started":"2023-11-04T12:57:21.226118Z","shell.execute_reply":"2023-11-04T12:57:21.238955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng_by_id(idx, periods):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    df = pd.read_parquet('/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet', filters=[('series_id','=',idx)])\n    df = basic_fe(df, periods)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-04T12:57:21.244647Z","iopub.execute_input":"2023-11-04T12:57:21.244979Z","iopub.status.idle":"2023-11-04T12:57:21.262803Z","shell.execute_reply.started":"2023-11-04T12:57:21.244952Z","shell.execute_reply":"2023-11-04T12:57:21.261096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"periods = [60] # 30 120 300","metadata":{"execution":{"iopub.status.busy":"2023-11-04T12:57:21.265076Z","iopub.execute_input":"2023-11-04T12:57:21.265953Z","iopub.status.idle":"2023-11-04T12:57:21.283282Z","shell.execute_reply.started":"2023-11-04T12:57:21.265910Z","shell.execute_reply":"2023-11-04T12:57:21.281331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id  = pd.read_parquet('/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet', columns=['series_id'])\nseries_id = series_id['series_id'].unique()\n\nweird_series = ['31011ade7c0a', 'a596ad0b82aa']\nseries_id = [s for s in series_id if s not in weird_series]\n\n\ndf = Parallel(n_jobs=-2)(delayed(feat_eng_by_id)(i, periods) for i in series_id)\ndf = pd.concat(df, ignore_index=True)\n\nfeatures = [col for col in df.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]\n\nfor p in periods:\n    plt.figure(figsize=(12,12))\n    plt.title(f\"Correlation heatmap for features with period {p}[s]\")\n    sns.heatmap(df[features].filter(like=f'_{p}_').corr(),\n    vmin=-1, vmax=1, center=0,\n    cmap=sns.color_palette(\"flare\", as_cmap=True),\n    square=True)\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2023-11-04T12:57:21.285272Z","iopub.execute_input":"2023-11-04T12:57:21.285669Z","iopub.status.idle":"2023-11-04T13:02:24.652612Z","shell.execute_reply.started":"2023-11-04T12:57:21.285641Z","shell.execute_reply":"2023-11-04T13:02:24.650759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_to_drop = ['anglezdiffabs_300_min', 'anglezdiffabs_120_min',  ]\nfor p in periods:\n    features_to_drop.append(f'anglezdiffabs_{p}_amplit')\n    features_to_drop.append(f'enmo_{p}_amplit')","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:02:24.654726Z","iopub.execute_input":"2023-11-04T13:02:24.655312Z","iopub.status.idle":"2023-11-04T13:02:24.665624Z","shell.execute_reply.started":"2023-11-04T13:02:24.655259Z","shell.execute_reply":"2023-11-04T13:02:24.663318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_with_target = df[features].corrwith(df['awake'])\ncorr_with_target.abs().sort_values().tail(50)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:02:24.667387Z","iopub.execute_input":"2023-11-04T13:02:24.667760Z","iopub.status.idle":"2023-11-04T13:02:31.509266Z","shell.execute_reply.started":"2023-11-04T13:02:24.667732Z","shell.execute_reply":"2023-11-04T13:02:31.507580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_for_heatmap = corr_with_target.abs().sort_values().tail(27).index.to_list()\nfeatures_for_heatmap = [f for f in features_for_heatmap if f not in ['anglezdiffabs_3600_min','anglezdiffabs_diff_3600_mean']]\nplt.figure(figsize=(14,14))\nplt.title(f\"Correlation heatmap\")\nsns.heatmap(df[features_for_heatmap].corr(),\nvmin=-1, vmax=1, center=0,\ncmap=sns.color_palette(\"flare\", as_cmap=True),\nsquare=True)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:02:31.510717Z","iopub.execute_input":"2023-11-04T13:02:31.511032Z","iopub.status.idle":"2023-11-04T13:02:50.236353Z","shell.execute_reply.started":"2023-11-04T13:02:31.511008Z","shell.execute_reply":"2023-11-04T13:02:50.234624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df[::50]\n\nlgb_params = {\n'boosting_type':'gbdt',\n'num_leaves':35,\n'max_depth':6,\n'learning_rate':0.03,\n'n_estimators':850,\n'subsample_for_bin':200000,\n'min_child_weight':0.001,\n'min_child_samples':20,\n'subsample':0.9,\n'reg_alpha':0.05,\n'reg_lambda':0.05,\n'verbose': -1\n}","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:02:50.240158Z","iopub.execute_input":"2023-11-04T13:02:50.240537Z","iopub.status.idle":"2023-11-04T13:02:50.247609Z","shell.execute_reply.started":"2023-11-04T13:02:50.240511Z","shell.execute_reply":"2023-11-04T13:02:50.246231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(df, cv):\n    \n    drop_cols = ['series_id', 'step', 'timestamp']\n    X, y = df.drop(columns=drop_cols+['awake']), df['awake']\n    gc.collect()\n    \n    cv_split = cv.split(X)\n    models = []\n    mean_f1 = 0\n    mean_acc = 0\n    \n    for i, (train_index, test_index) in enumerate(cv_split):\n\n        X_train, y_train = X.iloc[train_index], y.iloc[train_index]\n        X_test, y_test = X.iloc[test_index], y.iloc[test_index]\n        \n        print(f\"Fitting model {i+1} of {cv.n_splits}\")\n        \n        model = lgb.LGBMClassifier(random_state=42, n_jobs=-1, **lgb_params)\n        model.fit(X_train, y_train)\n        preds = model.predict(X_test)\n        f1 = f1_score(y_test, preds)\n        acc = accuracy_score(y_test, preds)\n        print(f\"Fold {i+1} finished training\\n F1:{f1_score(y_test, preds)} | Acc:{accuracy_score(y_test, preds)}\")\n        mean_f1 += f1\n        mean_acc += acc\n        models.append(model)\n        \n    print('-'*50)\n    print(f\"Training finished\\nF1: {mean_f1 / cv.n_splits}, Acc: {mean_acc / cv.n_splits}\")   \n    gc.collect()\n    \n    return models","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:02:50.249694Z","iopub.execute_input":"2023-11-04T13:02:50.250041Z","iopub.status.idle":"2023-11-04T13:02:50.267858Z","shell.execute_reply.started":"2023-11-04T13:02:50.250013Z","shell.execute_reply":"2023-11-04T13:02:50.266780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Fitting model with all {len(features)} features\")\nfrom sklearn.model_selection import KFold, train_test_split\nimport lightgbm as lgb\nfrom sklearn.metrics import f1_score, accuracy_score\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\nmodels = train_model(df, kf)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:02:50.269459Z","iopub.execute_input":"2023-11-04T13:02:50.269776Z","iopub.status.idle":"2023-11-04T13:05:14.104699Z","shell.execute_reply.started":"2023-11-04T13:02:50.269748Z","shell.execute_reply":"2023-11-04T13:05:14.103539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])\nfeature_importance = fi.mean(axis=0).sort_values()\n\nplt.figure(figsize=(15,8))\nplt.xticks(rotation='vertical')\nsns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)\nplt.title('LightGBM feature importance')","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:14.106430Z","iopub.execute_input":"2023-11-04T13:05:14.106765Z","iopub.status.idle":"2023-11-04T13:05:14.510671Z","shell.execute_reply.started":"2023-11-04T13:05:14.106740Z","shell.execute_reply":"2023-11-04T13:05:14.508931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = df.drop(columns=features_to_drop)\n# features = [col for col in df.columns if col not in ['timestamp', 'series_id', 'step', 'awake']]\n\n# print(f\"Fitting model with {len(features)} features\")\n\n# kf = KFold(n_splits=5, shuffle=True, random_state=42)\n# models = train_model(df, kf)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:14.512766Z","iopub.execute_input":"2023-11-04T13:05:14.513095Z","iopub.status.idle":"2023-11-04T13:05:14.516946Z","shell.execute_reply.started":"2023-11-04T13:05:14.513061Z","shell.execute_reply":"2023-11-04T13:05:14.515945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fi = pd.DataFrame([pd.Series(model.feature_importances_, index=features) for model in models])\n# feature_importance = fi.mean(axis=0).sort_values()\n# plt.figure(figsize=(15,8))\n# plt.xticks(rotation='vertical')\n# sns.barplot(x=feature_importance[-15:].index, y=feature_importance[-15:].values)\n# plt.title('LightGBM feature importance after dropping some features')","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:14.517943Z","iopub.execute_input":"2023-11-04T13:05:14.518275Z","iopub.status.idle":"2023-11-04T13:05:14.539588Z","shell.execute_reply.started":"2023-11-04T13:05:14.518247Z","shell.execute_reply":"2023-11-04T13:05:14.537730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Model","metadata":{}},{"cell_type":"code","source":"# Saving classifier \nimport pickle\nwith open('model.pkl', 'wb') as f:\n    pickle.dump(models, f)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:14.541663Z","iopub.execute_input":"2023-11-04T13:05:14.542133Z","iopub.status.idle":"2023-11-04T13:05:14.863333Z","shell.execute_reply.started":"2023-11-04T13:05:14.542101Z","shell.execute_reply":"2023-11-04T13:05:14.861782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Appling to test data","metadata":{}},{"cell_type":"code","source":"# Recovering memory\ndel features\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:14.864575Z","iopub.execute_input":"2023-11-04T13:05:14.864958Z","iopub.status.idle":"2023-11-04T13:05:15.018778Z","shell.execute_reply.started":"2023-11-04T13:05:14.864929Z","shell.execute_reply":"2023-11-04T13:05:15.017267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import groupby\n\ndef predict_test(idx): # 这里period只能用一个\n    \n    test  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet',\n                            filters=[('series_id','=',idx)])\n    \n    test = basic_fe(test, periods)\n    \n    drop_cols = ['series_id', 'step', 'timestamp']\n    \n#     features_to_drop = ['anglezdiffabs_300_min', 'anglezdiffabs_120_min',  ]\n#     for p in periods:\n#         features_to_drop.append(f'anglezdiffabs_{p}_amplit')\n#         features_to_drop.append(f'enmo_{p}_amplit')\n        \n    X_test = test.drop(columns=drop_cols)\n#     X_test = X_test.drop(columns=features_to_drop)\n    \n    p = mod.predict(X_test)\n    \n    test[\"not_awake\"] = p[0]\n    test[\"awake\"]     = p[1]\n    \n    smoothing_length = 2*230\n\n    test[\"score\"]  = test[\"awake\"].rolling(smoothing_length, center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    test[\"smooth\"] = test[\"not_awake\"].rolling(smoothing_length, center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    # re-binarize\n    test[\"smooth\"] = test[\"smooth\"].round()\n\n    # https://stackoverflow.com/questions/73777727/how-to-mark-start-end-of-a-series-of-non-null-and-non-0-values-in-a-column-of-a\n    def get_event(df):\n        lstCV = zip(df.series_id, df.smooth)\n        lstPOI = []\n        for (c, v), g in groupby(lstCV, lambda cv: \n                                (cv[0], cv[1]!=0 and not pd.isnull(cv[1]))):\n            llg = sum(1 for item in g)\n            if v is False: \n                lstPOI.extend([0]*llg)\n            else: \n                lstPOI.extend(['onset']+(llg-2)*[0]+['wakeup'] if llg > 1 else [0])\n        return lstPOI\n\n    test['event'] = get_event(test)\n    \n    return test.loc[test['event'] != 0][['series_id','step','event','score']]","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:15.020850Z","iopub.execute_input":"2023-11-04T13:05:15.021311Z","iopub.status.idle":"2023-11-04T13:05:15.034916Z","shell.execute_reply.started":"2023-11-04T13:05:15.021273Z","shell.execute_reply":"2023-11-04T13:05:15.033285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id'])\nseries_id = series_id.series_id.unique()\ntests = []\n\nfor idx in series_id:\n    for mod in models:\n        tests.append(predict_test(idx))","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:15.037898Z","iopub.execute_input":"2023-11-04T13:05:15.038877Z","iopub.status.idle":"2023-11-04T13:05:50.425975Z","shell.execute_reply.started":"2023-11-04T13:05:15.038831Z","shell.execute_reply":"2023-11-04T13:05:50.423901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.concat(tests)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:50.427958Z","iopub.execute_input":"2023-11-04T13:05:50.428337Z","iopub.status.idle":"2023-11-04T13:05:50.443252Z","shell.execute_reply.started":"2023-11-04T13:05:50.428310Z","shell.execute_reply":"2023-11-04T13:05:50.441289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:50.445909Z","iopub.execute_input":"2023-11-04T13:05:50.447505Z","iopub.status.idle":"2023-11-04T13:05:50.461365Z","shell.execute_reply.started":"2023-11-04T13:05:50.447459Z","shell.execute_reply":"2023-11-04T13:05:50.460585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = test.copy().reset_index(drop=True).reset_index(names='row_id')\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:50.462387Z","iopub.execute_input":"2023-11-04T13:05:50.463957Z","iopub.status.idle":"2023-11-04T13:05:50.475473Z","shell.execute_reply.started":"2023-11-04T13:05:50.463905Z","shell.execute_reply":"2023-11-04T13:05:50.473648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission","metadata":{"execution":{"iopub.status.busy":"2023-11-04T13:05:50.477162Z","iopub.execute_input":"2023-11-04T13:05:50.477827Z","iopub.status.idle":"2023-11-04T13:05:50.492096Z","shell.execute_reply.started":"2023-11-04T13:05:50.477796Z","shell.execute_reply":"2023-11-04T13:05:50.490854Z"},"trusted":true},"execution_count":null,"outputs":[]}]}