{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom itertools import groupby\nfrom sklearn.model_selection import train_test_split\nfrom joblib import Parallel, delayed\nimport lightgbm as lgb\nimport gc\n\nfrom warnings import simplefilter \nsimplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-12T16:07:59.975086Z","iopub.execute_input":"2023-10-12T16:07:59.975732Z","iopub.status.idle":"2023-10-12T16:07:59.980915Z","shell.execute_reply.started":"2023-10-12T16:07:59.975703Z","shell.execute_reply":"2023-10-12T16:07:59.979679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all numeric columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object and col_type != 'datetime64[ns, UTC]':\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    gc.collect()\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:07:59.985617Z","iopub.execute_input":"2023-10-12T16:07:59.985892Z","iopub.status.idle":"2023-10-12T16:07:59.998901Z","shell.execute_reply.started":"2023-10-12T16:07:59.985869Z","shell.execute_reply":"2023-10-12T16:07:59.997971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng(df):\n    \n    df['series_id'] = df['series_id'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df['hour'] = df[\"timestamp\"].dt.hour\n    \n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    # limit outiliers in enmo\n    df['enmo'] = df['enmo'].clip(upper=4)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float32)\n    \n    for col in ['enmo', 'anglezdiffabs']:\n        \n        # periods in seconds        \n        periods = [60, 360, 720, 1500]\n        \n        for n in periods:\n            \n            rol_args = {'window':f'{n+5}s', 'min_periods':10, 'center':True}\n            \n            for agg in ['median', 'mean', 'max', 'min', 'var']:\n                df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n            \n            if n == max(periods):\n                df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n            \n            df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}']\n            df[f'{col}_amplit_{n}_min'] = df[f'{col}_amplit_{n}'].rolling(**rol_args).min().astype(np.float32).values\n            \n            df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).max().astype(np.float32)\n            df[f'{col}_diff_{n}_mean'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).mean().astype(np.float32)\n\n    \n            gc.collect()\n        \n\n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:08:00.000921Z","iopub.execute_input":"2023-10-12T16:08:00.001318Z","iopub.status.idle":"2023-10-12T16:08:00.019007Z","shell.execute_reply.started":"2023-10-12T16:08:00.001282Z","shell.execute_reply":"2023-10-12T16:08:00.018239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:08:00.019936Z","iopub.execute_input":"2023-10-12T16:08:00.020575Z","iopub.status.idle":"2023-10-12T16:08:01.923700Z","shell.execute_reply.started":"2023-10-12T16:08:00.020537Z","shell.execute_reply":"2023-10-12T16:08:01.922915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = '/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet'\n\nseries_id  = pd.read_parquet(file, columns=['series_id'])\nseries_id = series_id.series_id.unique()\n\ndef feat_eng_by_id(idx):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    df  = pd.read_parquet(file, filters=[('series_id','=',idx)])\n    df['awake'] = df['awake'].astype(np.int8)\n    df = feat_eng(df)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:08:29.961606Z","iopub.execute_input":"2023-10-12T16:08:29.961989Z","iopub.status.idle":"2023-10-12T16:08:31.648024Z","shell.execute_reply.started":"2023-10-12T16:08:29.961960Z","shell.execute_reply":"2023-10-12T16:08:31.647174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain = Parallel(n_jobs=6)(delayed(feat_eng_by_id)(i) for i in series_id)\ntrain = pd.concat(train, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:08:31.649245Z","iopub.execute_input":"2023-10-12T16:08:31.650484Z","iopub.status.idle":"2023-10-12T16:11:27.324859Z","shell.execute_reply.started":"2023-10-12T16:08:31.650457Z","shell.execute_reply":"2023-10-12T16:11:27.323988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['series_id', 'step', 'timestamp']\n\nX = train.drop(columns=drop_cols+['awake'])\ny = train['awake']","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:11:39.612013Z","iopub.execute_input":"2023-10-12T16:11:39.614426Z","iopub.status.idle":"2023-10-12T16:11:46.157880Z","shell.execute_reply.started":"2023-10-12T16:11:39.614359Z","shell.execute_reply":"2023-10-12T16:11:46.156649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:12:25.051385Z","iopub.execute_input":"2023-10-12T16:12:25.051806Z","iopub.status.idle":"2023-10-12T16:12:26.258573Z","shell.execute_reply.started":"2023-10-12T16:12:25.051770Z","shell.execute_reply":"2023-10-12T16:12:26.257654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nm = lgb.LGBMClassifier()\nm.fit(X,y)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:12:31.016698Z","iopub.execute_input":"2023-10-12T16:12:31.017111Z","iopub.status.idle":"2023-10-12T16:16:15.849606Z","shell.execute_reply.started":"2023-10-12T16:12:31.017061Z","shell.execute_reply":"2023-10-12T16:16:15.848512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:16:15.851297Z","iopub.execute_input":"2023-10-12T16:16:15.851600Z","iopub.status.idle":"2023-10-12T16:16:16.240831Z","shell.execute_reply.started":"2023-10-12T16:16:15.851573Z","shell.execute_reply":"2023-10-12T16:16:16.239796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_events(idx, classifier, file='test_series.parquet') :\n    \n    test  = pd.read_parquet(f'/kaggle/input/child-mind-institute-detect-sleep-states/{file}',\n                    filters=[('series_id','=',idx)])\n    test = feat_eng(test)\n    X_test = test.drop(columns=drop_cols)\n    test = test[drop_cols]\n\n    preds, probs = classifier.predict(X_test), classifier.predict_proba(X_test)[:, 1]\n    \n    test['prediction'] = preds\n    test['prediction'] = test['prediction'].rolling(360+1, center=True).median()\n    test['probability'] = probs\n    \n    test = test[test['prediction']!=2]\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n\n    \n    test['pred_diff'] = test['prediction'].diff()\n\n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan})\n    \n    test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n    test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n    test = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('timestamp')\n\n    return test","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:16:16.242363Z","iopub.execute_input":"2023-10-12T16:16:16.242674Z","iopub.status.idle":"2023-10-12T16:16:16.255403Z","shell.execute_reply.started":"2023-10-12T16:16:16.242647Z","shell.execute_reply":"2023-10-12T16:16:16.254549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ncols_sub = ['series_id','step','event','score']\n\nseries_id  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id'])\nseries_id = series_id.series_id.unique()\n\ntests = Parallel(n_jobs=-1)(delayed(get_events)(i, m) for i in series_id)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:19:24.607231Z","iopub.execute_input":"2023-10-12T16:19:24.607686Z","iopub.status.idle":"2023-10-12T16:20:17.215222Z","shell.execute_reply.started":"2023-10-12T16:19:24.607653Z","shell.execute_reply":"2023-10-12T16:20:17.214109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat(tests, ignore_index=True)\nsubmission = submission[cols_sub].reset_index(names='row_id')\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-10-12T16:20:17.216745Z","iopub.execute_input":"2023-10-12T16:20:17.217131Z","iopub.status.idle":"2023-10-12T16:20:17.240010Z","shell.execute_reply.started":"2023-10-12T16:20:17.217088Z","shell.execute_reply":"2023-10-12T16:20:17.239308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}