{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n## How to perform CV and optimization for model in in this competition \n\nI have used the work of [How to perform CV - Gradient Boosting](https://www.kaggle.com/code/yeoyunsianggeremie/how-to-perform-cv-gradient-boosting)","metadata":{}},{"cell_type":"code","source":"!pip install -q /kaggle/input/xgboost-2-0-0-whl/xgboost-2.0.0-py3-none-manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:08:34.570728Z","iopub.execute_input":"2023-11-01T01:08:34.571029Z","iopub.status.idle":"2023-11-01T01:09:07.948803Z","shell.execute_reply.started":"2023-11-01T01:08:34.571007Z","shell.execute_reply":"2023-11-01T01:09:07.948071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cp /kaggle/input/zzzs-optimized-models-cmi-competition/* /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:09:07.950012Z","iopub.execute_input":"2023-11-01T01:09:07.950257Z","iopub.status.idle":"2023-11-01T01:09:07.954230Z","shell.execute_reply.started":"2023-11-01T01:09:07.950236Z","shell.execute_reply":"2023-11-01T01:09:07.953417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom itertools import groupby\nfrom sklearn.model_selection import train_test_split\nfrom pandas.api.types import is_datetime64_ns_dtype\nfrom IPython.display import display_html, clear_output, Markdown;\nimport optuna as opt\nfrom sklearn.model_selection import StratifiedGroupKFold, GroupKFold\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom xgboost import XGBRegressor, XGBClassifier\nfrom lightgbm import LGBMClassifier\nfrom lightgbm import log_evaluation, early_stopping, record_evaluation\nfrom catboost import CatBoostRegressor, CatBoostClassifier, Pool\n\n\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom joblib import Parallel, delayed\nimport gc\nimport plotly.express as px\n\nfrom os import system, getpid, walk;\nfrom psutil import Process;\nfrom colorama import Fore, Style, init\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom metric import score # Import event detection ap score function\n\n# These are variables to be used by the score function\ncolumn_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n}\n\ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:09:07.955178Z","iopub.execute_input":"2023-11-01T01:09:07.955477Z","iopub.status.idle":"2023-11-01T01:09:07.967723Z","shell.execute_reply.started":"2023-11-01T01:09:07.955452Z","shell.execute_reply":"2023-11-01T01:09:07.966917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Auxiliar Functions","metadata":{}},{"cell_type":"code","source":"def GetMemUsage():\n    \"\"\"\n    This function defines the memory usage across the kernel. \n    Source-\n    https://stackoverflow.com/questions/61366458/how-to-find-memory-usage-of-kaggle-notebook\n    \"\"\";\n    \n    pid = getpid();\n    py = Process(pid);\n    memory_use = py.memory_info()[0] / 2. ** 30;\n    return f\"RAM memory GB usage = {memory_use :.4}\";\n\ndef reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]) and not 'category':\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int32)  \n            else:\n                df[col] = df[col].astype(np.float16)\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:09:07.969971Z","iopub.execute_input":"2023-11-01T01:09:07.970220Z","iopub.status.idle":"2023-11-01T01:09:07.983078Z","shell.execute_reply.started":"2023-11-01T01:09:07.970199Z","shell.execute_reply":"2023-11-01T01:09:07.982384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering Function","metadata":{}},{"cell_type":"code","source":"def feat_eng(df):\n    \n    df['series_id'] = df['series_id'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df['hour'] = df[\"timestamp\"].dt.hour\n    \n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    # limit outiliers in enmo\n    df['enmo'] = df['enmo'].clip(upper=4)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float32)\n    \n    for col in ['enmo', 'anglez', 'anglezdiffabs']:\n        \n        # periods in seconds        \n        periods = [60, 300, 700, 1500, 3000]\n        \n        for n in periods:\n            \n            rol_args = {'window':f'{n+5}s', 'min_periods':10, 'center':True}\n            \n            for agg in ['median', 'mean', 'max', 'min', 'var']:\n                df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n            \n            if n == max(periods):\n                df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n            \n            df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}']\n            df[f'{col}_amplit_{n}_min'] = df[f'{col}_amplit_{n}'].rolling(**rol_args).min().astype(np.float32).values\n            \n            df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).max().astype(np.float32)\n            df[f'{col}_diff_{n}_mean'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).mean().astype(np.float32)\n\n    \n            gc.collect()\n        \n    df.drop(columns=['anglezdiffabs_min_3000', 'anglezdiffabs_min_700',\n       'anglezdiffabs_min_1500', 'anglezdiffabs_min_300',\n       'anglezdiffabs_amplit_700', 'anglezdiffabs_min_60',\n       'anglezdiffabs_amplit_1500', 'anglezdiffabs_amplit_300',\n       'anglez_diff_60_max', 'anglezdiffabs_diff_60_max', 'anglezdiffabs',\n       'anglezdiffabs_amplit_3000', 'anglezdiffabs_diff_300_mean',\n       'anglez_diff_60_mean', 'anglezdiffabs_diff_300_max',\n       'anglezdiffabs_max_700', 'anglez_diff_300_max', 'enmo_diff_300_mean',\n       'anglezdiffabs_diff_60_mean'], inplace=True)\n    \n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:09:07.984062Z","iopub.execute_input":"2023-11-01T01:09:07.984502Z","iopub.status.idle":"2023-11-01T01:09:07.998172Z","shell.execute_reply.started":"2023-11-01T01:09:07.984476Z","shell.execute_reply":"2023-11-01T01:09:07.997475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet'\n# file = '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet'\n\ndef feat_eng_by_id(idx):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    df  = pd.read_parquet(file, filters=[('series_id','=',idx)])\n    df['awake'] = df['awake'].astype(np.int8)\n    df = feat_eng(df)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:09:07.998968Z","iopub.execute_input":"2023-11-01T01:09:07.999264Z","iopub.status.idle":"2023-11-01T01:09:08.010878Z","shell.execute_reply.started":"2023-11-01T01:09:07.999239Z","shell.execute_reply":"2023-11-01T01:09:08.010192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training and validating ","metadata":{}},{"cell_type":"code","source":"DEV = False\nTRUNCATION_FACTOR = 60\n\nseries_id  = pd.read_parquet(file, columns=['series_id'])\nseries_id = series_id.series_id.unique()\n\nprint(len(series_id))\n\nif DEV:\n    series_id = series_id[::10]","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:09:08.011673Z","iopub.execute_input":"2023-11-01T01:09:08.011876Z","iopub.status.idle":"2023-11-01T01:09:09.456928Z","shell.execute_reply.started":"2023-11-01T01:09:08.011851Z","shell.execute_reply":"2023-11-01T01:09:09.456119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain = Parallel(n_jobs=-1)(delayed(feat_eng_by_id)(i) for i in series_id)\ntrain = pd.concat(train, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:09:09.457870Z","iopub.execute_input":"2023-11-01T01:09:09.458119Z","iopub.status.idle":"2023-11-01T01:11:36.048250Z","shell.execute_reply.started":"2023-11-01T01:09:09.458097Z","shell.execute_reply":"2023-11-01T01:11:36.047468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop_cols = ['series_id', 'step', 'timestamp']\n\nX, y = train.drop(columns=['awake']), train['awake']\ngroups = train[\"series_id\"].values\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:11:36.049213Z","iopub.execute_input":"2023-11-01T01:11:36.049466Z","iopub.status.idle":"2023-11-01T01:11:39.454976Z","shell.execute_reply.started":"2023-11-01T01:11:36.049444Z","shell.execute_reply":"2023-11-01T01:11:39.454151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsembleAvgProba():\n    \n    def __init__(self, classifiers):\n        \n        self.classifiers = classifiers\n    \n    def fit(self,X,y):\n        \n        for classifier in self.classifiers:                \n            classifier.fit(X, y)\n            gc.collect()\n     \n    def predict_proba(self, X):\n        \n        probs = []\n        \n        for m in self.classifiers:\n            probs.append(m.predict_proba(X))\n        \n        probabilities = np.stack(probs)\n        p = np.mean(probabilities, axis=0)\n        \n        return p \n    \n    def predict(self, X):\n        \n        probs = []\n        \n        for m in self.classifiers:\n            probs.append(m.predict(X))\n        \n        probabilities = np.stack(probs)\n        p = np.mean(probabilities, axis=0)\n        \n        return p.round()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:11:39.457280Z","iopub.execute_input":"2023-11-01T01:11:39.457515Z","iopub.status.idle":"2023-11-01T01:11:39.463488Z","shell.execute_reply.started":"2023-11-01T01:11:39.457495Z","shell.execute_reply":"2023-11-01T01:11:39.462917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Events Function","metadata":{}},{"cell_type":"code","source":"def get_events(idx, classifier, file='test_series.parquet') :\n    \n    test  = pd.read_parquet(f'/kaggle/input/child-mind-institute-detect-sleep-states/{file}',\n                    filters=[('series_id','=',idx)])\n    test = feat_eng(test)\n    X_test = test.drop(columns=drop_cols)\n    test = test[drop_cols]\n\n    preds, probs = classifier.predict(X_test), classifier.predict_proba(X_test)[:, 1]\n    \n    test['prediction'] = preds\n    test['prediction'] = test['prediction'].rolling(360+1, center=True).median()\n    test['probability'] = probs\n    \n    test = test[test['prediction']!=2]\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n    \n    test['pred_diff'] = test['prediction'].diff()\n    \n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan})\n    \n    test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n    test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n    \n    df_events = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('step')\n\n    if df_events.shape[0]>0:\n        if df_events.iloc[0]['event'] == 'wakeup':\n            df_events = df_events.iloc[1:]\n\n    return df_events\n\n\ndef get_validation_events(idx, classifier, test) :\n    \n    drop_cols = ['series_id', 'step', 'timestamp']\n    X_test = test.drop(columns=drop_cols)\n    test = test[drop_cols]\n    \n    #display(test)\n\n    preds, probs = classifier.predict(X_test), classifier.predict_proba(X_test)[:, 1]\n    \n    test['prediction'] = preds\n    test['prediction'] = test['prediction'].rolling(300+1, center=True).median()\n    test['probability'] = probs\n    \n    test = test[test['prediction']!=2]\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n\n    \n    test['pred_diff'] = test['prediction'].diff()\n    \n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan})\n    test.dropna(subset='event', inplace=True)\n    \n    test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n    test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n    df_events = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('step')\n    \n    if df_events.shape[0]>0:\n        if df_events.iloc[0]['event'] == 'wakeup':\n            df_events = df_events.iloc[1:]\n\n    return df_events","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:11:39.464337Z","iopub.execute_input":"2023-11-01T01:11:39.464707Z","iopub.status.idle":"2023-11-01T01:11:39.482873Z","shell.execute_reply.started":"2023-11-01T01:11:39.464686Z","shell.execute_reply":"2023-11-01T01:11:39.482255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optimizer","metadata":{}},{"cell_type":"code","source":"train_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\ntrain_events['timestamp'] = pd.to_datetime(train_events['timestamp']).apply(lambda t: t.tz_localize(None))","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:11:39.483629Z","iopub.execute_input":"2023-11-01T01:11:39.484095Z","iopub.status.idle":"2023-11-01T01:11:39.620746Z","shell.execute_reply.started":"2023-11-01T01:11:39.484076Z","shell.execute_reply":"2023-11-01T01:11:39.619914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEV:\n    SPLITS = 2\nelse:\n    SPLITS = 5\n    \ndef cv_score(model, SPLITS=SPLITS):\n    \n    CV_scores = []\n\n    sgkf = StratifiedGroupKFold(n_splits=SPLITS)   \n    cols_sub = ['series_id','step','event','score']\n    CV_scores = []\n\n    for i, (dev_ind, val_ind) in enumerate(sgkf.split(X, y, groups)):\n\n        dev_ind = dev_ind[::TRUNCATION_FACTOR]\n\n        dev_groups, val_groups = groups[dev_ind], groups[val_ind]\n        X_train = X.iloc[dev_ind].drop(columns = ['series_id', 'step', 'timestamp'])\n        y_train = y.iloc[dev_ind]\n        model.fit(X_train, y_train)\n\n        series_id = list(set(val_groups))\n\n        assert len(set(dev_groups) & set(val_groups)) == 0\n\n        val_events = []\n\n        for idx in series_id: \n            test = get_validation_events(idx, model, X[X.series_id == idx])\n            val_events.append(test[cols_sub])\n\n        val = pd.concat(val_events, ignore_index=True).reset_index(names='row_id')\n        val_solution = train_events[train_events['series_id'].isin(series_id)]\n\n        fold_CV = score(val_solution, val, tolerances, **column_names)\n\n        CV_scores.append(fold_CV)\n\n    return CV_scores","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:11:39.621803Z","iopub.execute_input":"2023-11-01T01:11:39.622033Z","iopub.status.idle":"2023-11-01T01:11:39.629556Z","shell.execute_reply.started":"2023-11-01T01:11:39.622012Z","shell.execute_reply":"2023-11-01T01:11:39.628553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nN_TRIALS = 30\nscores = []\n\ndef objective(trial):\n    \n    lgb_params = {\n        'n_estimators': trial.suggest_int('n_estimators', 1000, 1500),\n        'learning_rate': trial.suggest_float('learning_rate', 1e-3, 5e-2, log=True),\n        'max_depth': trial.suggest_int('max_depth', 2, 6),\n        'reg_alpha': trial.suggest_float('subsample', 0.03, 0.06, step=0.01),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 0.7, step=0.01),\n        'min_child_weight': trial.suggest_int('min_child_weight', 200, 500),\n    }\n    \n    model = LGBMClassifier(**lgb_params, random_state=51)\n    \n    score = cv_score(model, SPLITS=SPLITS)\n#     scores.append(score)\n    \n    return np.mean(score)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:11:39.630404Z","iopub.execute_input":"2023-11-01T01:11:39.630601Z","iopub.status.idle":"2023-11-01T01:11:39.645366Z","shell.execute_reply.started":"2023-11-01T01:11:39.630583Z","shell.execute_reply":"2023-11-01T01:11:39.644520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\n\nstudy_name = \"lgbm_optimization\"  \nstorage_name = f\"sqlite:///{study_name}.db\"\n\nstudy = opt.create_study(direction=\"maximize\", study_name=study_name, storage=storage_name, load_if_exists=True)\nstudy.optimize(objective, n_trials=N_TRIALS, n_jobs=-1 )","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:35:31.165539Z","iopub.execute_input":"2023-11-01T01:35:31.166449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = study.trials_dataframe()\nhistory.sort_values(by=\"value\", ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:35:23.577939Z","iopub.status.idle":"2023-11-01T01:35:23.578349Z","shell.execute_reply.started":"2023-11-01T01:35:23.578154Z","shell.execute_reply":"2023-11-01T01:35:23.578173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(study.best_params)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:35:23.579752Z","iopub.status.idle":"2023-11-01T01:35:23.580138Z","shell.execute_reply.started":"2023-11-01T01:35:23.579933Z","shell.execute_reply":"2023-11-01T01:35:23.579950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt.visualization.plot_slice(study, target_name = 'score')","metadata":{"execution":{"iopub.status.busy":"2023-11-01T01:35:23.582504Z","iopub.status.idle":"2023-11-01T01:35:23.583179Z","shell.execute_reply.started":"2023-11-01T01:35:23.582896Z","shell.execute_reply":"2023-11-01T01:35:23.582929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}