{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom itertools import groupby\nfrom sklearn.model_selection import train_test_split\nfrom pandas.api.types import is_datetime64_ns_dtype\nfrom imblearn.under_sampling import RandomUnderSampler\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom joblib import Parallel, delayed\nimport gc\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-27T10:26:45.015608Z","iopub.execute_input":"2023-09-27T10:26:45.015940Z","iopub.status.idle":"2023-09-27T10:26:49.429858Z","shell.execute_reply.started":"2023-09-27T10:26:45.015913Z","shell.execute_reply":"2023-09-27T10:26:49.428589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]) and not 'category':\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int32)  \n            else:\n\n                df[col] = df[col].astype(np.float16)\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:26:49.432420Z","iopub.execute_input":"2023-09-27T10:26:49.432703Z","iopub.status.idle":"2023-09-27T10:26:49.443998Z","shell.execute_reply.started":"2023-09-27T10:26:49.432678Z","shell.execute_reply":"2023-09-27T10:26:49.442729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"signal_awake = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.208 * np.pi) ** 24))\nsignal_onset = dict(zip(range(1440), np.sin(np.linspace(0, np.pi, 1440) + 0.555 * np.pi) ** 24))\n\ndef feat_eng(df):\n    \n    df['series_id'] = df['series_id'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    \n    df.sort_values(['timestamp'], inplace=True)\n    \n    df['signal_onset'] = (df.timestamp.dt.hour * 60 + df.timestamp.dt.minute).map(signal_onset).astype(np.float32)\n    df['signal_awake'] = (df.timestamp.dt.hour * 60 + df.timestamp.dt.minute).map(signal_awake).astype(np.float32)\n    df[\"anglez_diff\"] = df[\"anglez\"].diff().astype(np.float32)\n    df[\"anglez_diffabs\"] = abs(df[\"anglez_diff\"]).astype(np.float32)\n    df[\"anglezabs\"] = abs(df[\"anglez\"]).astype(np.float32)\n    df['anglez_x_enmo'] = (df['anglez'] * df['enmo']).astype(np.float32)\n    \n    # TODO \n    # df['anglezabs_x_enmo'] = abs(df['anglez']) * df['enmo']\n    \n    df.set_index('timestamp', inplace=True)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n    \n    \n    for col in ['enmo', 'anglez', 'anglez_x_enmo', 'anglezabs', 'anglez_diff', \"anglez_diffabs\"]:\n        \n        for n in [21, 61]:\n        #TODO\n        #for n in [21, 61, 121]:\n            df[f'{col}_diff_{n}'] = df[col].diff(periods=n).astype(np.float32)\n        \n            rol_args = {'window':f'{n*5}s', 'min_periods':1, 'center':True}\n            \n            for agg in ['median', 'mean', 'max', 'min', 'std']:\n                df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n            \n            df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n\n            df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}']\n            df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].rolling(**rol_args).max().astype(np.float32)\n            df[f'{col}_medianxstd_{n}'] = df[f'{col}_median_{n}'] * df[f'{col}_std_{n}']\n    \n            gc.collect()\n        \n#         df[f'conv1d_{col}']\n\n    df.drop(columns=['anglez_x_enmo', 'anglez_diffabs', 'anglez_diff', 'anglez',], inplace = True)\n    \n    df.drop(columns=[\n        'anglez_x_enmo_diff_21', 'anglez_std_21', 'anglezabs_std_21', 'anglezabs_mad_21', \n        'anglezabs_amplit_21', 'anglez_diff_21', 'anglez_x_enmo_std_21', 'anglez_diff_median_21', \n        'anglez_diff_mean_21', 'anglez_diff_diff_21', 'anglez_x_enmo_mean_21', 'anglez_diff_amplit_21',\n        'anglez_diff_medianxstd_21', 'enmo_diff_21', 'anglez_diffabs_diff_21',\n        'anglez_diff_max_21', 'anglez_diffabs_max_21', 'anglez_diffabs_amplit_21'\n    ], inplace = True)\n    \n    \n    \n    df.reset_index(inplace=True)\n    df.bfill(inplace=True)\n    df.ffill(inplace=True)\n    df.dropna(inplace=True)\n\n    df = reduce_mem_usage(df)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:26:52.661207Z","iopub.execute_input":"2023-09-27T10:26:52.661571Z","iopub.status.idle":"2023-09-27T10:26:52.684138Z","shell.execute_reply.started":"2023-09-27T10:26:52.661541Z","shell.execute_reply":"2023-09-27T10:26:52.683230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# filtering series_id with total onset_counts == total wakeups\ntrain_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\nonset_counts = train_events.query('event==\"onset\"').groupby('series_id').count().sort_values('series_id')\nwakeup_counts = train_events.query('event==\"wakeup\"').groupby('series_id').count().sort_values('series_id')\nseries_id = sorted(train_events['series_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:28:08.829530Z","iopub.execute_input":"2023-09-27T10:28:08.830678Z","iopub.status.idle":"2023-09-27T10:28:08.908163Z","shell.execute_reply.started":"2023-09-27T10:28:08.830647Z","shell.execute_reply":"2023-09-27T10:28:08.907259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"comparison_counts = pd.DataFrame({'series_id':series_id, 'wake_up_counts':wakeup_counts['event'], 'onset_counts':onset_counts['event']}).reset_index(drop='series_id')\ncomparison_counts.query('wake_up_counts==onset_counts')","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:28:34.457276Z","iopub.execute_input":"2023-09-27T10:28:34.457630Z","iopub.status.idle":"2023-09-27T10:28:34.480017Z","shell.execute_reply.started":"2023-09-27T10:28:34.457604Z","shell.execute_reply":"2023-09-27T10:28:34.478934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(comparison_counts['series_id'])","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:29:35.961669Z","iopub.execute_input":"2023-09-27T10:29:35.962105Z","iopub.status.idle":"2023-09-27T10:29:35.968222Z","shell.execute_reply.started":"2023-09-27T10:29:35.962073Z","shell.execute_reply":"2023-09-27T10:29:35.967226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# in the end, they are the same\n\nsum(comparison_counts['wake_up_counts'] == comparison_counts['onset_counts']) == len(comparison_counts['series_id'])","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:29:42.743580Z","iopub.execute_input":"2023-09-27T10:29:42.744518Z","iopub.status.idle":"2023-09-27T10:29:42.750420Z","shell.execute_reply.started":"2023-09-27T10:29:42.744485Z","shell.execute_reply":"2023-09-27T10:29:42.749500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"comparison_counts['total_events'] = comparison_counts['wake_up_counts']+comparison_counts['onset_counts']\ncomparison_counts['total_events'].hist()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:30:09.802687Z","iopub.execute_input":"2023-09-27T10:30:09.803129Z","iopub.status.idle":"2023-09-27T10:30:10.165754Z","shell.execute_reply.started":"2023-09-27T10:30:09.803092Z","shell.execute_reply":"2023-09-27T10:30:10.164672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.percentile(comparison_counts['total_events'], q=5), np.percentile(comparison_counts['total_events'], q=95)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_series_id = comparison_counts.query('total_events>=24 & total_events<=80')\nselected_series_id","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:32:09.338517Z","iopub.execute_input":"2023-09-27T10:32:09.338913Z","iopub.status.idle":"2023-09-27T10:32:09.357421Z","shell.execute_reply.started":"2023-09-27T10:32:09.338881Z","shell.execute_reply":"2023-09-27T10:32:09.356124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet'\n\ndef feat_eng_by_id(idx):\n    \n    df  = pd.read_parquet(file, filters=[('series_id','=',idx)])\n    df['awake'] = df['awake'].astype(np.int8)\n    df = feat_eng(df)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:36:35.865419Z","iopub.execute_input":"2023-09-27T10:36:35.865799Z","iopub.status.idle":"2023-09-27T10:36:35.872456Z","shell.execute_reply.started":"2023-09-27T10:36:35.865773Z","shell.execute_reply":"2023-09-27T10:36:35.871186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id  = pd.read_parquet(file, columns=['series_id'])\nseries_id = series_id.series_id.unique()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:36:40.190152Z","iopub.execute_input":"2023-09-27T10:36:40.190509Z","iopub.status.idle":"2023-09-27T10:36:42.623204Z","shell.execute_reply.started":"2023-09-27T10:36:40.190482Z","shell.execute_reply":"2023-09-27T10:36:42.622149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filtered_id = np.intersect1d(series_id, selected_series_id['series_id'])","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:36:42.625271Z","iopub.execute_input":"2023-09-27T10:36:42.625686Z","iopub.status.idle":"2023-09-27T10:36:42.631134Z","shell.execute_reply.started":"2023-09-27T10:36:42.625648Z","shell.execute_reply":"2023-09-27T10:36:42.630429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain = Parallel(n_jobs=6)(delayed(feat_eng_by_id)(i) for i in filtered_id)\ntrain = pd.concat(train)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:36:43.415996Z","iopub.execute_input":"2023-09-27T10:36:43.416704Z","iopub.status.idle":"2023-09-27T10:38:45.840823Z","shell.execute_reply.started":"2023-09-27T10:36:43.416659Z","shell.execute_reply":"2023-09-27T10:38:45.839778Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['series_id', 'step', 'timestamp']","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:38:45.842646Z","iopub.execute_input":"2023-09-27T10:38:45.842983Z","iopub.status.idle":"2023-09-27T10:38:45.848201Z","shell.execute_reply.started":"2023-09-27T10:38:45.842956Z","shell.execute_reply":"2023-09-27T10:38:45.847068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict(train['awake'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:38:45.849888Z","iopub.execute_input":"2023-09-27T10:38:45.850548Z","iopub.status.idle":"2023-09-27T10:38:46.054312Z","shell.execute_reply.started":"2023-09-27T10:38:45.850520Z","shell.execute_reply":"2023-09-27T10:38:46.052140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sampling_strategy = {0: 4500020 , 1: 5000000, 2:3120000  }\nsampling_strategy = dict(train['awake'].value_counts())\nrus = RandomUnderSampler(random_state=42, sampling_strategy=sampling_strategy)\nX, y = rus.fit_resample(train.drop(columns=drop_cols+['awake']), train['awake'])\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:38:46.055521Z","iopub.status.idle":"2023-09-27T10:38:46.056095Z","shell.execute_reply.started":"2023-09-27T10:38:46.055895Z","shell.execute_reply":"2023-09-27T10:38:46.055915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:38:46.058077Z","iopub.status.idle":"2023-09-27T10:38:46.058512Z","shell.execute_reply.started":"2023-09-27T10:38:46.058316Z","shell.execute_reply":"2023-09-27T10:38:46.058336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = {    \n    'boosting_type':'gbdt',\n    'num_leaves':31,\n    'max_depth':4,\n    'learning_rate':0.03,\n    'n_estimators':150,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.6,\n    'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n    'random_state':666\n             }\n\n\nm = lgb.LGBMClassifier(**lgb_params)\nm.fit(X,y)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:38:46.059949Z","iopub.status.idle":"2023-09-27T10:38:46.060612Z","shell.execute_reply.started":"2023-09-27T10:38:46.060405Z","shell.execute_reply":"2023-09-27T10:38:46.060424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp = pd.Series(m.feature_importances_, index=X.columns).sort_values()\nfig = px.bar(x=feat_imp, y=feat_imp.index, orientation='h')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:38:46.062103Z","iopub.status.idle":"2023-09-27T10:38:46.062547Z","shell.execute_reply.started":"2023-09-27T10:38:46.062357Z","shell.execute_reply":"2023-09-27T10:38:46.062375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:38:46.064460Z","iopub.status.idle":"2023-09-27T10:38:46.065067Z","shell.execute_reply.started":"2023-09-27T10:38:46.064865Z","shell.execute_reply":"2023-09-27T10:38:46.064891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_test(idx):\n    \n    test  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet',\n                            filters=[('series_id','=',idx)])\n    test = feat_eng(test)\n\n    X_test = test.drop(columns=drop_cols)\n    p = m.predict_proba(X_test)\n    \n    test[\"not_awake\"] = p[:,0]\n    test[\"awake\"]     = p[:,1]\n    \n    smoothing_length = 2*230\n\n    test[\"score\"]  = test[\"awake\"].rolling(smoothing_length, center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    test[\"smooth\"] = test[\"not_awake\"].rolling(smoothing_length, center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    # re-binarize\n    test[\"smooth\"] = test[\"smooth\"].round()\n\n    # https://stackoverflow.com/questions/73777727/how-to-mark-start-end-of-a-series-of-non-null-and-non-0-values-in-a-column-of-a\n    def get_event(df):\n        lstCV = zip(df.series_id, df.smooth)\n        lstPOI = []\n        for (c, v), g in groupby(lstCV, lambda cv: \n                                (cv[0], cv[1]!=0 and not pd.isnull(cv[1]))):\n            llg = sum(1 for item in g)\n            if v is False: \n                lstPOI.extend([0]*llg)\n            else: \n                lstPOI.extend(['onset']+(llg-2)*[0]+['wakeup'] if llg > 1 else [0])\n        return lstPOI\n\n    test['event'] = get_event(test)\n    \n    return test.loc[test['event'] != 0][['series_id','step','event','score']]","metadata":{"execution":{"iopub.status.busy":"2023-09-27T10:38:46.066379Z","iopub.status.idle":"2023-09-27T10:38:46.066799Z","shell.execute_reply.started":"2023-09-27T10:38:46.066594Z","shell.execute_reply":"2023-09-27T10:38:46.066613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id'])\nseries_id = series_id.series_id.unique()\ntests = []\n\nfor idx in series_id:\n    tests.append(predict_test(idx))","metadata":{"execution":{"iopub.status.busy":"2023-09-26T13:13:02.691360Z","iopub.execute_input":"2023-09-26T13:13:02.691729Z","iopub.status.idle":"2023-09-26T13:14:47.448490Z","shell.execute_reply.started":"2023-09-26T13:13:02.691697Z","shell.execute_reply":"2023-09-26T13:14:47.447343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.concat(tests)","metadata":{"execution":{"iopub.status.busy":"2023-09-26T13:14:47.449984Z","iopub.execute_input":"2023-09-26T13:14:47.450377Z","iopub.status.idle":"2023-09-26T13:14:47.459938Z","shell.execute_reply.started":"2023-09-26T13:14:47.450344Z","shell.execute_reply":"2023-09-26T13:14:47.458753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = test.copy().reset_index(drop=True).reset_index(names='row_id')\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-26T13:14:47.463269Z","iopub.execute_input":"2023-09-26T13:14:47.463639Z","iopub.status.idle":"2023-09-26T13:14:47.482565Z","shell.execute_reply.started":"2023-09-26T13:14:47.463610Z","shell.execute_reply":"2023-09-26T13:14:47.481285Z"},"trusted":true},"execution_count":null,"outputs":[]}]}