{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6507308,"sourceType":"datasetVersion","datasetId":3706667},{"sourceId":6684904,"sourceType":"datasetVersion","datasetId":3856023},{"sourceId":141456876,"sourceType":"kernelVersion"}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n## How to perform CV in this competition - Gradient Boosting Technique\n\nI noticed almost all public notebooks do not have CV embedded in them\n\nTherefore, I release a starter notebook on how to perform CV in this competition. I am using a 5-fold StratifiedGroupKFold by series_id in this notebook. The model being used here is a single XGBoost\n\nThe solution architecture is similar to https://www.kaggle.com/code/renatoreggiani/zzzs-ensemble-model-cmi, using pure Pandas ","metadata":{}},{"cell_type":"code","source":"!pip install -q /kaggle/input/xgboost-2-0-0-whl/xgboost-2.0.0-py3-none-manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:07:27.955466Z","iopub.execute_input":"2023-10-15T14:07:27.955884Z","iopub.status.idle":"2023-10-15T14:08:02.971481Z","shell.execute_reply.started":"2023-10-15T14:07:27.955851Z","shell.execute_reply":"2023-10-15T14:08:02.969952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom itertools import groupby\nfrom sklearn.model_selection import train_test_split\nfrom pandas.api.types import is_datetime64_ns_dtype\nfrom IPython.display import display_html, clear_output, Markdown;\n\nfrom sklearn.model_selection import StratifiedGroupKFold, GroupKFold\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom xgboost import XGBRegressor, XGBClassifier\nfrom lightgbm import LGBMRegressor, LGBMClassifier\nfrom lightgbm import log_evaluation, early_stopping, record_evaluation\nfrom catboost import CatBoostRegressor, CatBoostClassifier, Pool\n\n\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom joblib import Parallel, delayed\nimport gc\nimport plotly.express as px\n\nfrom os import system, getpid, walk;\nfrom psutil import Process;\nfrom colorama import Fore, Style, init\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom metric import score # Import event detection ap score function\n\n# These are variables to be used by the score function\ncolumn_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n}\n\ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}\n\ndef PrintColor(text:str, color = Fore.BLUE, style = Style.BRIGHT):\n    \"Prints color outputs using colorama using a text F-string\";\n    print(style + color + text + Style.RESET_ALL); \n\ndef GetMemUsage():\n    \"\"\"\n    This function defines the memory usage across the kernel. \n    Source-\n    https://stackoverflow.com/questions/61366458/how-to-find-memory-usage-of-kaggle-notebook\n    \"\"\";\n    \n    pid = getpid();\n    py = Process(pid);\n    memory_use = py.memory_info()[0] / 2. ** 30;\n    return f\"RAM memory GB usage = {memory_use :.4}\";","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:02.974028Z","iopub.execute_input":"2023-10-15T14:08:02.974399Z","iopub.status.idle":"2023-10-15T14:08:02.986687Z","shell.execute_reply.started":"2023-10-15T14:08:02.974366Z","shell.execute_reply":"2023-10-15T14:08:02.98575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]) and not 'category':\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int32)  \n            else:\n                df[col] = df[col].astype(np.float16)\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:02.988143Z","iopub.execute_input":"2023-10-15T14:08:02.988463Z","iopub.status.idle":"2023-10-15T14:08:03.006803Z","shell.execute_reply.started":"2023-10-15T14:08:02.988438Z","shell.execute_reply":"2023-10-15T14:08:03.00585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng(df):\n    \n    df['series_id'] = df['series_id'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df['hour'] = df[\"timestamp\"].dt.hour\n    #df[\"dayofweek\"] = df[\"timestamp\"].dt.dayofweek\n    df['month'] = df[\"timestamp\"].dt.month\n    df[\"spring\"] = df[\"month\"].apply(lambda x: 1 if x in [3,4,5] else 0)\n    df[\"summer\"] = df[\"month\"].apply(lambda x: 1 if x in [6,7,8] else 0)\n    df[\"autumn\"] = df[\"month\"].apply(lambda x: 1 if x in [9,10,11] else 0)\n    df[\"winter\"] = df[\"month\"].apply(lambda x: 1 if x in [12,1,2] else 0)\n    df = df.drop(columns = ['month'])\n    \n    \n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float32)\n    \n    for col in ['enmo', 'anglez', 'anglezdiffabs']:\n        \n        # periods in seconds        \n        periods = [60,360,720,3600] \n        \n        for n in periods:\n            \n            rol_args = {'window':f'{n+5}s', 'min_periods':10, 'center':True}\n            \n            for agg in ['median', 'mean', 'max', 'min', 'var']:\n                df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n            \n            if n == max(periods):\n                df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n            \n            df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}']\n            df[f'{col}_amplit_{n}_min'] = df[f'{col}_amplit_{n}'].rolling(**rol_args).min().astype(np.float32).values\n            \n#             if col in ['enmo', 'anglez']:\n            df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).max().astype(np.float32)\n            df[f'{col}_diff_{n}_mean'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).mean().astype(np.float32)\n\n    \n            gc.collect()\n    \n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:03.010006Z","iopub.execute_input":"2023-10-15T14:08:03.010452Z","iopub.status.idle":"2023-10-15T14:08:03.189981Z","shell.execute_reply.started":"2023-10-15T14:08:03.010416Z","shell.execute_reply":"2023-10-15T14:08:03.188914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#file = '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet'\nfile = '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet'\n\ndef feat_eng_by_id(idx):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    df  = pd.read_parquet(file, filters=[('series_id','=',idx)])\n    df['awake'] = df['awake'].astype(np.int8)\n    df = feat_eng(df)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:03.191141Z","iopub.execute_input":"2023-10-15T14:08:03.191488Z","iopub.status.idle":"2023-10-15T14:08:03.209567Z","shell.execute_reply.started":"2023-10-15T14:08:03.191461Z","shell.execute_reply":"2023-10-15T14:08:03.208606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training and validating ","metadata":{}},{"cell_type":"code","source":"DEV = False\nTRUNCATION_FACTOR = 60\n\nseries_id  = pd.read_parquet(file, columns=['series_id'])\nseries_id = series_id.series_id.unique()\n\nprint(len(series_id))\n\nif DEV:\n    series_id = series_id[::10]","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:03.210693Z","iopub.execute_input":"2023-10-15T14:08:03.211118Z","iopub.status.idle":"2023-10-15T14:08:05.281594Z","shell.execute_reply.started":"2023-10-15T14:08:03.211082Z","shell.execute_reply":"2023-10-15T14:08:05.280745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weird_series = ['31011ade7c0a', 'a596ad0b82aa']\n\nseries_id = [s for s in series_id if s not in weird_series]","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:05.282563Z","iopub.execute_input":"2023-10-15T14:08:05.28353Z","iopub.status.idle":"2023-10-15T14:08:05.288621Z","shell.execute_reply.started":"2023-10-15T14:08:05.28349Z","shell.execute_reply":"2023-10-15T14:08:05.28755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain = Parallel(n_jobs=6)(delayed(feat_eng_by_id)(i) for i in series_id)\ntrain = pd.concat(train, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:05.29008Z","iopub.execute_input":"2023-10-15T14:08:05.290513Z","iopub.status.idle":"2023-10-15T14:08:38.717362Z","shell.execute_reply.started":"2023-10-15T14:08:05.290474Z","shell.execute_reply":"2023-10-15T14:08:38.715897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# REDUCE train data by half\n#train = train.iloc[::60].reset_index(drop = True)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:38.719622Z","iopub.execute_input":"2023-10-15T14:08:38.720077Z","iopub.status.idle":"2023-10-15T14:08:38.725485Z","shell.execute_reply.started":"2023-10-15T14:08:38.720036Z","shell.execute_reply":"2023-10-15T14:08:38.724242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:38.729261Z","iopub.execute_input":"2023-10-15T14:08:38.729601Z","iopub.status.idle":"2023-10-15T14:08:39.115062Z","shell.execute_reply.started":"2023-10-15T14:08:38.729571Z","shell.execute_reply":"2023-10-15T14:08:39.11425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#drop_cols = ['series_id', 'step', 'timestamp']\n\nX, y = train.drop(columns=['awake']), train['awake']\ngroups = train[\"series_id\"].values\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:39.116428Z","iopub.execute_input":"2023-10-15T14:08:39.116943Z","iopub.status.idle":"2023-10-15T14:08:39.989519Z","shell.execute_reply.started":"2023-10-15T14:08:39.116912Z","shell.execute_reply":"2023-10-15T14:08:39.988485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not DEV:\n    del train\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:39.991033Z","iopub.execute_input":"2023-10-15T14:08:39.991438Z","iopub.status.idle":"2023-10-15T14:08:39.99561Z","shell.execute_reply.started":"2023-10-15T14:08:39.99141Z","shell.execute_reply":"2023-10-15T14:08:39.994918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsembleAvgProba():\n    \n    def __init__(self, classifiers):\n        \n        self.classifiers = classifiers\n    \n    def fit(self,X,y):\n        \n        for classifier in self.classifiers:                \n            classifier.fit(X, y)\n            gc.collect()\n     \n    def predict_proba(self, X):\n        \n        probs = []\n        \n        for m in self.classifiers:\n            probs.append(m.predict_proba(X))\n        \n        probabilities = np.stack(probs)\n        p = np.mean(probabilities, axis=0)\n        \n        return p \n    \n    def predict(self, X):\n        \n        probs = []\n        \n        for m in self.classifiers:\n            probs.append(m.predict(X))\n        \n        probabilities = np.stack(probs)\n        p = np.mean(probabilities, axis=0)\n        \n        return p.round()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:39.99674Z","iopub.execute_input":"2023-10-15T14:08:39.997256Z","iopub.status.idle":"2023-10-15T14:08:40.012755Z","shell.execute_reply.started":"2023-10-15T14:08:39.99723Z","shell.execute_reply":"2023-10-15T14:08:40.011607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params1 = {    \n    'boosting_type':'gbdt',\n    'num_leaves':31,\n    'max_depth':6,\n    'learning_rate':0.03,\n    'n_estimators':850,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.9,\n#     'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n             }\n\n\n# Training classifier","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:40.014449Z","iopub.execute_input":"2023-10-15T14:08:40.015037Z","iopub.status.idle":"2023-10-15T14:08:40.025291Z","shell.execute_reply.started":"2023-10-15T14:08:40.015006Z","shell.execute_reply":"2023-10-15T14:08:40.02423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5 fold GroupKFold CV","metadata":{}},{"cell_type":"code","source":"def get_events(idx, classifier, file='test_series.parquet') :\n    \n    test  = pd.read_parquet(f'/kaggle/input/child-mind-institute-detect-sleep-states/{file}',\n                    filters=[('series_id','=',idx)])\n    test = feat_eng(test)\n    X_test = test.drop(columns=drop_cols)\n    test = test[drop_cols]\n\n    preds, probs = classifier.predict(X_test), classifier.predict_proba(X_test)[:, 1]\n    \n    test['prediction'] = preds\n    test['prediction'] = test['prediction'].rolling(360+1, center=True).median()\n    test['probability'] = probs\n    \n    test = test[test['prediction']!=2]\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n\n    \n    test['pred_diff'] = test['prediction'].diff()\n    \n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan})\n    \n    test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n    test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n    test = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('timestamp')\n\n    return test\n\n\ndef get_validation_events(idx, classifier, test) :\n    \n    drop_cols = ['series_id', 'step', 'timestamp']\n    X_test = test.drop(columns=drop_cols)\n    test = test[drop_cols]\n    \n    #PrintColor(\"X_test\")\n    #display(X_test)\n    \n    #print()\n    \n    #PrintColor(\"test\")\n    #display(test)\n\n    preds, probs = classifier.predict(X_test), classifier.predict_proba(X_test)[:, 1]\n    \n    test['prediction'] = preds\n    test['prediction'] = test['prediction'].rolling(360+1, center=True).median()\n    test['probability'] = probs\n    \n    test = test[test['prediction']!=2]\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n\n    \n    test['pred_diff'] = test['prediction'].diff()\n    \n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan})\n    \n    test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n    test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n    test = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('timestamp')\n\n    return test","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:40.026779Z","iopub.execute_input":"2023-10-15T14:08:40.02747Z","iopub.status.idle":"2023-10-15T14:08:40.041652Z","shell.execute_reply.started":"2023-10-15T14:08:40.027441Z","shell.execute_reply":"2023-10-15T14:08:40.040676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\ntrain_events['timestamp'] = pd.to_datetime(train_events['timestamp']).apply(lambda t: t.tz_localize(None))","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:40.043086Z","iopub.execute_input":"2023-10-15T14:08:40.044318Z","iopub.status.idle":"2023-10-15T14:08:40.202476Z","shell.execute_reply.started":"2023-10-15T14:08:40.044276Z","shell.execute_reply":"2023-10-15T14:08:40.201361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEV:\n    SPLITS = 2\nelse:\n    SPLITS = 5\n    \nsgkf = StratifiedGroupKFold(n_splits=SPLITS)   \ncols_sub = ['series_id','step','event','score']\nCV_scores = 0\n\nfor i, (dev_ind, val_ind) in enumerate(sgkf.split(X, y, groups)):\n    \n    dev_ind = dev_ind[::TRUNCATION_FACTOR]\n    \n    model = EnsembleAvgProba(classifiers=[\n                    #LGBMClassifier(random_state=42, **lgb_params1),\n                    #GradientBoostingClassifier(n_estimators=100,max_depth=5,min_samples_leaf=300,random_state=42),\n                    #RandomForestClassifier(n_estimators=500, min_samples_leaf=300, random_state=42, n_jobs=-1),\n                    XGBClassifier(n_estimators=520,objective=\"binary:logistic\", learning_rate=0.02, max_depth=7, random_state=42)    ]\n                )\n    \n    PrintColor(f\"---> Fold {i}:\", color=Fore.BLUE)\n    dev_groups, val_groups = groups[dev_ind], groups[val_ind]\n    X_train = X.iloc[dev_ind].drop(columns = ['series_id', 'step', 'timestamp'])\n    y_train = y.iloc[dev_ind]\n    print(f\"Train Indexes: {dev_ind}\")\n    print(f\"Val Indexes: {val_ind}\")\n    print(f\"Shape: {X_train.shape}\")\n    model.fit(X_train, y_train)\n    \n    series_id = list(set(val_groups))\n    print(f\"Train Groups: {set(dev_groups)}\")\n    print(f\"Val Groups: {set(val_groups)}\")\n    \n    assert len(set(dev_groups) & set(val_groups)) == 0\n\n    val_events = []\n\n    for idx in series_id: \n        \n        test = get_validation_events(idx, model, X[X.series_id == idx])\n        val_events.append(test[cols_sub])\n \n    val = pd.concat(val_events, ignore_index=True).reset_index(names='row_id')\n    val_solution = train_events[train_events['series_id'].isin(series_id)].dropna(subset=[\"step\", \"timestamp\"]).reset_index(drop=True)\n    \n    print()\n    PrintColor(\"Your Output\", color=Fore.RED)\n    display(val[val.series_id == series_id[0]])\n    print()\n    PrintColor(\"Correct Output\", color=Fore.RED)\n    display(val_solution[val_solution.series_id == series_id[0]])\n    \n    fold_CV = score(val_solution, val, tolerances, **column_names)\n    \n    PrintColor(f\"Fold {i} CV Score | {fold_CV}\")\n    CV_scores += fold_CV\n    \n    print()\n    print()\n    \nPrintColor(f\"Mean CV Score | {CV_scores / SPLITS}\", color = Fore.GREEN)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T14:08:40.204148Z","iopub.execute_input":"2023-10-15T14:08:40.204526Z","iopub.status.idle":"2023-10-15T14:10:10.450346Z","shell.execute_reply.started":"2023-10-15T14:08:40.204498Z","shell.execute_reply":"2023-10-15T14:10:10.449019Z"},"trusted":true},"execution_count":null,"outputs":[]}]}