{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom itertools import groupby\nfrom sklearn.model_selection import train_test_split\nfrom pandas.api.types import is_datetime64_ns_dtype\nfrom imblearn.under_sampling import RandomUnderSampler\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom joblib import Parallel, delayed\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom warnings import simplefilter\nsimplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-19T23:05:20.036221Z","iopub.execute_input":"2023-09-19T23:05:20.036649Z","iopub.status.idle":"2023-09-19T23:05:24.231066Z","shell.execute_reply.started":"2023-09-19T23:05:20.036612Z","shell.execute_reply":"2023-09-19T23:05:24.229852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]) and not 'category':\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int32)  \n            else:\n\n                df[col] = df[col].astype(np.float16)\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-09-19T23:05:24.233403Z","iopub.execute_input":"2023-09-19T23:05:24.234000Z","iopub.status.idle":"2023-09-19T23:05:24.248398Z","shell.execute_reply.started":"2023-09-19T23:05:24.233968Z","shell.execute_reply":"2023-09-19T23:05:24.246914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def conv1d(s):\n#     if len(s)>2:\n#         return mean(np.multiply(s, [1,0,1]))\n#     else:\n#         return np.nan\n\n# [conv1d(s) for s in df['enmo'].rolling(3, center=True)]","metadata":{"execution":{"iopub.status.busy":"2023-09-19T23:05:24.250129Z","iopub.execute_input":"2023-09-19T23:05:24.251860Z","iopub.status.idle":"2023-09-19T23:05:24.273453Z","shell.execute_reply.started":"2023-09-19T23:05:24.251797Z","shell.execute_reply":"2023-09-19T23:05:24.271390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal_awake = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.208 * np.pi) ** 24))\nsignal_onset = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.555 * np.pi) ** 24))\n\ndef feat_eng(df):\n    \n    df['series_id'] = df['series_id'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    \n    df.sort_values(['timestamp'], inplace=True)\n    \n    df['signal_onset'] = (df.timestamp.dt.hour * 60 + df.timestamp.dt.minute).map(signal_onset).astype(np.float32)\n    df['signal_awake'] = (df.timestamp.dt.hour * 60 + df.timestamp.dt.minute).map(signal_awake).astype(np.float32)\n    df[\"anglez_diff\"] = df[\"anglez\"].diff().astype(np.float32)\n    df[\"anglez_diffabs\"] = abs(df[\"anglez_diff\"]).astype(np.float32)\n    df[\"anglezabs\"] = abs(df[\"anglez\"]).astype(np.float32)\n    df['anglez_x_enmo'] = (df['anglez'] * df['enmo']).astype(np.float32)\n    \n    df.set_index('timestamp', inplace=True)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n    \n    \n    for col in ['enmo', 'anglez', 'anglez_x_enmo', 'anglezabs', 'anglez_diff', \"anglez_diffabs\"]:\n        \n        for n in [51]:\n            df[f'{col}_diff_{n}'] = df[col].diff(periods=n).astype(np.float32)\n        \n            rol_args = {'window':f'{n*5}s', 'min_periods':1, 'center':True}\n            \n            for agg in ['median', 'mean', 'max', 'min']:\n                df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n                \n            df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n\n            df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}']\n            df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].rolling(**rol_args).max().astype(np.float32)\n            \n            if col in ['anglez', 'anglez_diffabs']:\n                df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).std().astype(np.float32)\n    \n            gc.collect()\n        \n#         df[f'conv1d_{col}']\n\n    df.drop(columns=['anglez_x_enmo', 'anglez_diffabs', 'anglez_diff', 'anglez'], inplace = True)\n    \n    df.reset_index(inplace=True)\n    df.bfill(inplace=True)\n    df.ffill(inplace=True)\n    df.dropna(inplace=True)\n\n    df = reduce_mem_usage(df)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-09-19T23:05:24.277915Z","iopub.execute_input":"2023-09-19T23:05:24.278455Z","iopub.status.idle":"2023-09-19T23:05:24.308469Z","shell.execute_reply.started":"2023-09-19T23:05:24.278404Z","shell.execute_reply":"2023-09-19T23:05:24.306605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet'\n\ndef feat_eng_by_id(idx):\n    \n    df  = pd.read_parquet(file, filters=[('series_id','=',idx)])\n    df['awake'] = df['awake'].astype(np.int8)\n    df = feat_eng(df)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-09-19T23:05:24.310508Z","iopub.execute_input":"2023-09-19T23:05:24.310974Z","iopub.status.idle":"2023-09-19T23:05:24.321013Z","shell.execute_reply.started":"2023-09-19T23:05:24.310902Z","shell.execute_reply":"2023-09-19T23:05:24.319184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id  = pd.read_parquet(file, columns=['series_id'])\nseries_id = series_id.series_id.unique()","metadata":{"execution":{"iopub.status.busy":"2023-09-19T23:05:24.323085Z","iopub.execute_input":"2023-09-19T23:05:24.323624Z","iopub.status.idle":"2023-09-19T23:05:27.494662Z","shell.execute_reply.started":"2023-09-19T23:05:24.323579Z","shell.execute_reply":"2023-09-19T23:05:27.493363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain = Parallel(n_jobs=6)(delayed(feat_eng_by_id)(i) for i in series_id)\ntrain = pd.concat(train)","metadata":{"execution":{"iopub.status.busy":"2023-09-19T23:05:27.496497Z","iopub.execute_input":"2023-09-19T23:05:27.496964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['series_id', 'step', 'timestamp']\n\n# X = train.drop(columns=drop_cols+['awake'])\n# y = train['awake']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['awake'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampling_strategy = {0: 4500020 , 1: 5000000, 2:3120000  }\nrus = RandomUnderSampler(random_state=42, sampling_strategy=sampling_strategy)\nX, y = rus.fit_resample(train.drop(columns=drop_cols+['awake']), train['awake'])\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = {    \n    'boosting_type':'gbdt',\n    'num_leaves':31,\n    'max_depth':4,\n    'learning_rate':0.01,\n    'n_estimators':100,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.6,\n#     'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n    'random_state':666\n             }\n\n\nm = lgb.LGBMClassifier(**lgb_params)\nm.fit(X,y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp = pd.Series(m.feature_importances_, index=X.columns).sort_values()\nfig = px.bar(x=feat_imp, y=feat_imp.index, orientation='h')\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp[feat_imp==0].index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X, y\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_test(idx):\n    \n    test  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet',\n                            filters=[('series_id','=',idx)])\n    test = feat_eng(test)\n\n    X_test = test.drop(columns=drop_cols)\n    p = m.predict_proba(X_test)\n    \n    test[\"not_awake\"] = p[:,0]\n    test[\"awake\"]     = p[:,1]\n    \n    smoothing_length = 2*230\n\n    test[\"score\"]  = test[\"awake\"].rolling(smoothing_length, center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    test[\"smooth\"] = test[\"not_awake\"].rolling(smoothing_length, center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    # re-binarize\n    test[\"smooth\"] = test[\"smooth\"].round()\n\n    # https://stackoverflow.com/questions/73777727/how-to-mark-start-end-of-a-series-of-non-null-and-non-0-values-in-a-column-of-a\n    def get_event(df):\n        lstCV = zip(df.series_id, df.smooth)\n        lstPOI = []\n        for (c, v), g in groupby(lstCV, lambda cv: \n                                (cv[0], cv[1]!=0 and not pd.isnull(cv[1]))):\n            llg = sum(1 for item in g)\n            if v is False: \n                lstPOI.extend([0]*llg)\n            else: \n                lstPOI.extend(['onset']+(llg-2)*[0]+['wakeup'] if llg > 1 else [0])\n        return lstPOI\n\n    test['event'] = get_event(test)\n    \n    return test.loc[test['event'] != 0][['series_id','step','event','score']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id'])\nseries_id = series_id.series_id.unique()\ntests = []\n\nfor idx in series_id:\n    tests.append(predict_test(idx))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.concat(tests)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = test.copy().reset_index(drop=True).reset_index(names='row_id')\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}