{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":6507308,"sourceType":"datasetVersion","datasetId":3706667}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":54.460986,"end_time":"2023-11-29T11:29:23.096715","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-11-29T11:28:28.635729","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom itertools import groupby\nfrom sklearn.model_selection import train_test_split\nfrom joblib import Parallel, delayed\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.ensemble import GradientBoostingClassifier\nimport joblib\n\nimport gc\n\nfrom pandas.api.types import is_datetime64_ns_dtype\nfrom tqdm import tqdm\nimport plotly.express as px\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom warnings import simplefilter\nsimplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2023-11-29T11:28:32.477098Z","iopub.status.busy":"2023-11-29T11:28:32.476719Z","iopub.status.idle":"2023-11-29T11:28:37.409678Z","shell.execute_reply":"2023-11-29T11:28:37.408109Z"},"papermill":{"duration":4.941913,"end_time":"2023-11-29T11:28:37.412693","exception":false,"start_time":"2023-11-29T11:28:32.470780","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all numeric columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n#         if col_type != object and col_type != 'datetime64[ns, UTC]':\n        if col_type != object and not is_datetime64_ns_dtype(df[col]) and not 'category':\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    gc.collect()\n    \n    return df\n\ndef load_one_ts(idx, file_path):\n    df  = pd.read_parquet(file_path,filters=[('series_id','=',idx)])\n    df = df.drop(columns=['series_id'])\n    gc.collect()\n    return df","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:28:37.424663Z","iopub.status.busy":"2023-11-29T11:28:37.424298Z","iopub.status.idle":"2023-11-29T11:28:37.437972Z","shell.execute_reply":"2023-11-29T11:28:37.436394Z"},"papermill":{"duration":0.021739,"end_time":"2023-11-29T11:28:37.441012","exception":false,"start_time":"2023-11-29T11:28:37.419273","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_feat(df):\n    \n    def get_time_flag(dt, st, ed):\n        arr = np.zeros((len(dt), 1))\n        res = pd.DataFrame(data=arr, index=dt, dtype=int)\n        flag = res.between_time(st,ed,inclusive='right').index\n        res.loc[flag, 0] = 1\n        return res.loc[:,0].values\n    \n    df['date'] = df[\"timestamp\"].dt.date\n    df['hour'] = df[\"timestamp\"].dt.hour.astype(np.int8)\n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    gc.collect()\n    return df\n\ndef volalitity_feat(df, col_list, less_imp_cols=[]):\n    warnings.filterwarnings('ignore')\n\n    for col in col_list:\n        df[f\"{col}_diffabs\"] = df[col].diff().abs().astype(np.float32)\n        df[f'{col}_diffabs_log'] = df[col].map(np.log).replace([np.inf, -np.inf], np.nan).astype(np.float32) \n        \n        periods = [60, 180] +  [720*i for i in range(1,25)]  + [720*j for j in range(25,48,2)] #minute 1,6,12,25,40,60,120,360 ; [60, 360, 720, 1500, 2400, 3600, 4800, 7200] \n        for n in periods:\n            if n < 360:\n                # == old feature: statistics\n                rol_args = {'window':f'{n+5}s', 'min_periods':10, 'center':True}\n                if col not in less_imp_cols:\n                    agg_list = ['mean', 'max', 'min'] #'var','median', \n                    for agg in agg_list:\n                        df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                        gc.collect()\n#                     if n == max(periods):\n#                         df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n                    df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}']\n                    df[f'{col}_amplit_{n}_min'] = df[f'{col}_amplit_{n}'].rolling(**rol_args).min().astype(np.float32).values\n\n                    df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).max().astype(np.float32)\n                    df[f'{col}_diff_{n}_mean'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).mean().astype(np.float32)\n\n#                 if col in ['enmodiffabs']:\n#                     for agg in ['median']:\n#                         df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n#                     if n == max(periods):\n#                         df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n\n                gc.collect()\n\n                # == new feature: volatility\n                df[f'{col}_minus_rolling{n}_mean'] = (df[col] - df[col].rolling(**rol_args).mean()).astype(np.float32)\n                df[f'{col}_vola_rolling{n}'] = df[f'{col}_diffabs_log'].diff(1).rolling(**rol_args).std(numeric_only=True).astype(np.float32)\n                gc.collect()\n\n            \n            # == new feature:increase rate\n            df[f'{col}_increase_rate_{n}'] = df[col].rolling(**rol_args).mean()/df[col].shift(n).rolling(**rol_args).mean()\n            gc.collect()\n            \n    return df\n\ndef applyParallel(dfgrouped, func, col, lags):\n    retList = Parallel(n_jobs=-1, backend='loky')(\n        delayed(func)(group, col, lags) for name,group in dfgrouped)\n    return pd.concat(retList)\n\n\ndef df_stat(df, col, lags):\n    warnings.filterwarnings('ignore')\n\n    for dd in range(2, lags+1):\n        cols_dd = [f'{col}_diffabs_prev_hour_{y}' for y in range(1,dd+1)]\n        df[f'diffabs_prev_hour_{dd}_mean'] = np.nanmean(df[cols_dd].values, axis=1)\n    \n    cols = [f'{col}_diffabs_prev_hour_{y}' for y in range(1, lags + 1)]\n    vs = df[cols].values\n    df[f'{col}_diffabs_prev_hour_{lags}_std'] = np.nanstd(vs, axis=1)\n    df[f'{col}_diffabs_prev_hour_{lags}_min'] = np.nanmin(vs, axis=1)\n    df[f'{col}_diffabs_prev_hour_{lags}_max'] = np.nanmax(vs, axis=1)\n    \n    df[f'{col}_diffabs_prev_hour_{lags}_median'] = np.nanmedian(vs, axis=1)\n    df[f'{col}_diffabs_prev_hour_{lags}_p25'] = np.nanpercentile(vs, 25, axis=1)\n    df[f'{col}_diffabs_prev_hour_{lags}_p75'] = np.nanpercentile(vs, 75, axis=1)\n    \n    del vs\n    gc.collect()\n    return df\n    \n        \n\ndef gen_feat(df):\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df.sort_values(['timestamp'], inplace=True)\n    df = time_feat(df)\n    print('time feature finish')\n    \n\n    # limit outiliers in enmo\n    df['enmo'] = df['enmo'].clip(upper=4)    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float32)\n    df[\"enmodiffabs\"] = df[\"enmo\"].diff().abs().astype(np.float32)\n\n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n       \n    #10.29\n    df['lids_sma_fast'] = df['lids'].rolling('2h', center=True, min_periods=1).agg('mean').astype(np.float32)\n    df['lids_sma_macd'] = (df['lids'].rolling('18h', center=True, min_periods=1).agg('mean') - df['lids_sma_fast']\n                                 ).rolling(f'1h', center=True, min_periods=1).agg('mean').astype(np.float32)\n        \n        \n    col_list = ['enmo', 'anglezdiffabs','enmodiffabs']\n    df = volalitity_feat(df, col_list,less_imp_cols=['enmo','enmodiffabs'])\n    print('volalitity feature finish')\n    \n#     df = reduce_mem_usage(df)\n    df.reset_index(inplace=True)\n    no_need_feat = ['enmo', 'enmo_diffabs', 'enmodiffabs'] + [col for col in df.columns if 'anglezdiffabs_min' in col]\n    df.drop(columns=no_need_feat, inplace=True)\n    gc.collect()\n\n    return df","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:28:37.451372Z","iopub.status.busy":"2023-11-29T11:28:37.451008Z","iopub.status.idle":"2023-11-29T11:28:37.480479Z","shell.execute_reply":"2023-11-29T11:28:37.478701Z"},"papermill":{"duration":0.037386,"end_time":"2023-11-29T11:28:37.482950","exception":false,"start_time":"2023-11-29T11:28:37.445564","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # unit test\n# idx = '08db4255286f'\n# file =  '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet'\n# df = pd.read_parquet(file, filters=[('series_id','=',idx)])\n\n# df_tmp = gen_feat(df)\n# pd.set_option(\"display.max_columns\", None)\n# df_tmp.tail(10)\n","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:28:37.492453Z","iopub.status.busy":"2023-11-29T11:28:37.492031Z","iopub.status.idle":"2023-11-29T11:28:37.496285Z","shell.execute_reply":"2023-11-29T11:28:37.495382Z"},"papermill":{"duration":0.011751,"end_time":"2023-11-29T11:28:37.498694","exception":false,"start_time":"2023-11-29T11:28:37.486943","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# single time series generate features\ndef feat_eng_by_id(idx,file):\n        \n    df = load_one_ts(idx, file)\n    df['awake'] = df['awake'].astype(np.int8)\n#     df = feat_eng(df)\n    df = gen_feat(df)\n    return df","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:28:37.507681Z","iopub.status.busy":"2023-11-29T11:28:37.507359Z","iopub.status.idle":"2023-11-29T11:28:37.511608Z","shell.execute_reply":"2023-11-29T11:28:37.510719Z"},"papermill":{"duration":0.010926,"end_time":"2023-11-29T11:28:37.513362","exception":false,"start_time":"2023-11-29T11:28:37.502436","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 开始运行","metadata":{"papermill":{"duration":0.003285,"end_time":"2023-11-29T11:28:37.520612","exception":false,"start_time":"2023-11-29T11:28:37.517327","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# CONFIG\nDEV = 0\n\nTRAIN = 0\nINFERENCE = 1\n\ntrain_file_path =  '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet'\ntest_file_path = '/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet'\nmodel_path = '/kaggle/input/lgb-1128/LGB_EM_Model_newparam1128.pk'","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:28:37.529486Z","iopub.status.busy":"2023-11-29T11:28:37.529069Z","iopub.status.idle":"2023-11-29T11:28:37.534806Z","shell.execute_reply":"2023-11-29T11:28:37.533260Z"},"papermill":{"duration":0.012695,"end_time":"2023-11-29T11:28:37.536889","exception":false,"start_time":"2023-11-29T11:28:37.524194","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsembleAvgProba():\n    \n    def __init__(self, classifiers):\n        \n        self.classifiers = classifiers\n    \n    def fit(self,X,y):\n\n        for classifier in self.classifiers:\n            print(f\"{classifier.__class__} begin\")\n            if isinstance(classifier, RandomForestClassifier) or isinstance(classifier, GradientBoostingClassifier):\n                print('haha')\n                X.fillna(0, inplace=True)\n            classifier.fit(X, y)\n            print(f\"{classifier.__class__} finished\")\n            gc.collect()\n     \n    def predict_proba(self, X):\n        \n        probs = []\n        \n        for m in self.classifiers:\n            probs.append(m.predict_proba(X))\n        \n        probabilities = np.stack(probs)\n        p = np.mean(probabilities, axis=0)\n        \n        return p \n    \n    def predict(self, X):\n        \n        probs = []\n        \n        for m in self.classifiers:\n            probs.append(m.predict(X))\n        \n        probabilities = np.stack(probs)\n        p = np.mean(probabilities, axis=0)\n        \n        return p.round()","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:28:37.545929Z","iopub.status.busy":"2023-11-29T11:28:37.545549Z","iopub.status.idle":"2023-11-29T11:28:37.555411Z","shell.execute_reply":"2023-11-29T11:28:37.553607Z"},"papermill":{"duration":0.017319,"end_time":"2023-11-29T11:28:37.557983","exception":false,"start_time":"2023-11-29T11:28:37.540664","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### MODEL LOAD & PREDICT","metadata":{"papermill":{"duration":0.003413,"end_time":"2023-11-29T11:28:37.565253","exception":false,"start_time":"2023-11-29T11:28:37.561840","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_events(classifier,idx,drop_cols,res_cols,file='test_series.parquet') :\n    \n    test  = load_one_ts(idx, file)\n#     test = feat_eng(test)\n    test = gen_feat(test)\n    \n    if 'index' in test.columns:\n        X_test = test.drop(columns=drop_cols+['index'])\n    else:\n        X_test = test.drop(columns=drop_cols)\n        \n    preds, probs = classifier.predict(X_test), classifier.predict_proba(X_test)[:, 1]\n    \n    test = test[res_cols]\n    test['prediction'] = preds\n    test['prediction'] = test['prediction'].rolling(360+1, center=True).median()\n    test['probability'] = probs\n    \n    test = test[test['prediction']!=2]\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n\n    test['pred_diff'] = test['prediction'].diff()\n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan})\n    \n#     test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n#     test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n#     test = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('timestamp')\n    test = test.sort_values('timestamp')\n    test['series_id'] = idx\n    \n#     test = test.dropna(subset=['event'])\n#     if len(test) < 1:\n#         test = pd.DataFrame({'row_id':[0],'series_id':idx,'step':0,'event':'onset','score':0})\n\n    gc.collect()\n    return test","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:28:37.576035Z","iopub.status.busy":"2023-11-29T11:28:37.575661Z","iopub.status.idle":"2023-11-29T11:28:37.586762Z","shell.execute_reply":"2023-11-29T11:28:37.585012Z"},"papermill":{"duration":0.019399,"end_time":"2023-11-29T11:28:37.589889","exception":false,"start_time":"2023-11-29T11:28:37.570490","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# MODEL PRED\nif INFERENCE:\n    model = joblib.load(model_path)\n\n    print('==>MODEL PRED BEGIN')\n\n    cols_sub = ['series_id','step','event','score']\n\n    series_id  = pd.read_parquet(test_file_path, columns=['series_id'])\n    series_id = series_id.series_id.unique()\n\n    res_cols = ['step', 'timestamp']\n    drop_cols = ['step', 'timestamp','date']\n    tests = Parallel(n_jobs=-1, backend='loky')(delayed(get_events)(model,i,drop_cols,res_cols,test_file_path) for i in tqdm(series_id))\n\n    submission = pd.concat(tests, ignore_index=True)\n    submission = submission[cols_sub].reset_index(names='row_id')\n    submission.to_csv('/kaggle/working/submission.csv', index=False)\n    print('==>MODEL PRED FINISH')\n#     submission\n","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:28:37.599906Z","iopub.status.busy":"2023-11-29T11:28:37.599510Z","iopub.status.idle":"2023-11-29T11:29:22.217040Z","shell.execute_reply":"2023-11-29T11:29:22.215049Z"},"papermill":{"duration":44.625603,"end_time":"2023-11-29T11:29:22.219988","exception":false,"start_time":"2023-11-29T11:28:37.594385","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.execute_input":"2023-11-29T11:29:22.233146Z","iopub.status.busy":"2023-11-29T11:29:22.232697Z","iopub.status.idle":"2023-11-29T11:29:22.255622Z","shell.execute_reply":"2023-11-29T11:29:22.254284Z"},"papermill":{"duration":0.032934,"end_time":"2023-11-29T11:29:22.258388","exception":false,"start_time":"2023-11-29T11:29:22.225454","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.004537,"end_time":"2023-11-29T11:29:22.268168","exception":false,"start_time":"2023-11-29T11:29:22.263631","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}