{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7018283,"sourceType":"datasetVersion","datasetId":4035468}],"dockerImageVersionId":30587,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\nimport os \n\nimport torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\nfrom itertools import groupby\nimport gc\n\n\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\n\nnp.random.seed(0)\npd.set_option('display.precision', 2)  \npd.set_option('display.float_format', '{:.2f}'.format) ","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:04:11.473681Z","iopub.execute_input":"2023-11-21T12:04:11.474500Z","iopub.status.idle":"2023-11-21T12:04:11.483521Z","shell.execute_reply.started":"2023-11-21T12:04:11.474448Z","shell.execute_reply":"2023-11-21T12:04:11.481812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"csv_file_path = '/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv'\n\ntrain_events  = pd.read_csv(csv_file_path)\n\nseries_has_nan = train_events.groupby('series_id')['step'].apply(lambda x: x.isnull().any())\ntrain_ids = series_has_nan[~series_has_nan].index.tolist()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:04:17.253806Z","iopub.execute_input":"2023-11-21T12:04:17.254424Z","iopub.status.idle":"2023-11-21T12:04:17.327635Z","shell.execute_reply.started":"2023-11-21T12:04:17.254379Z","shell.execute_reply":"2023-11-21T12:04:17.326332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nparquet_file_path = os.path.join('/kaggle/input/light-weight-data/light_weight_data/Zzzs_train.parquet')\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parquet_file_path ='/kaggle/input/light-weight-data/light_weight_data/Zzzs_train_multi.parquet'\n\ndef get_multi_light_series(series_ids):\n    print(f'fetch {series_ids} \\n')\n    multi_series = pd.read_parquet(parquet_file_path, filters=[('series_id','in',series_ids)])\n    multi_series = multi_series.astype({'series_id':'category','step':'int16','awake': 'int16'})\n    multi_series =  add_features(multi_series)\n    return multi_series\n\n\n\ndef add_features(df):\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"],utc=True)\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"dayofweek\"] = df[\"timestamp\"].dt.dayofweek\n\n    df[\"anglez_times_enmo\"] = abs(df[\"anglez\"]) * df[\"enmo\"]\n\n    periods = 6 # 1/2 minute\n    df[\"anglez_diff\"] = df.groupby('series_id')['anglez'].diff(periods=periods).fillna(method=\"bfill\")\n    df[\"enmo_diff\"]   = df.groupby('series_id')['enmo'].diff(periods=periods).fillna(method=\"bfill\")\n    \n    df[\"anglez_mean\"] = df[\"anglez\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    df[\"enmo_mean\"]   = df[\"enmo\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    \n    df[\"anglez_min\"] = df[\"anglez\"].rolling(periods,center=True).min().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    df[\"enmo_min\"]   = df[\"enmo\"].rolling(periods,center=True).min().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    \n    df[\"anglez_max\"] = df[\"anglez\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    df[\"enmo_max\"]   = df[\"enmo\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    \n    df[\"anglez_std\"] = df[\"anglez\"].rolling(periods,center=True).std().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    df[\"enmo_std\"]   = df[\"enmo\"].rolling(periods,center=True).std().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    \n    df[\"anglez_diff_rolling\"] = df[\"anglez_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    df[\"enmo_diff_rolling\"]   = df[\"enmo_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float32')\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:04:28.851958Z","iopub.execute_input":"2023-11-21T12:04:28.852445Z","iopub.status.idle":"2023-11-21T12:04:28.868646Z","shell.execute_reply.started":"2023-11-21T12:04:28.852408Z","shell.execute_reply":"2023-11-21T12:04:28.867153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_all = get_multi_light_series(train_ids[:8])\nprint(f'memory usage: {train_all.memory_usage().sum() / 1024**2: .2f} MB')","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:04:35.107809Z","iopub.execute_input":"2023-11-21T12:04:35.108238Z","iopub.status.idle":"2023-11-21T12:05:00.074606Z","shell.execute_reply.started":"2023-11-21T12:04:35.108195Z","shell.execute_reply":"2023-11-21T12:05:00.073327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_set, test_set = train_test_split(train_all, test_size=0.2)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:06:48.851993Z","iopub.execute_input":"2023-11-21T12:06:48.852474Z","iopub.status.idle":"2023-11-21T12:06:50.418338Z","shell.execute_reply.started":"2023-11-21T12:06:48.852436Z","shell.execute_reply":"2023-11-21T12:06:50.417130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import make_pipeline\n\n\nfeatures = [\"step\", \"hour\",\"dayofweek\",\"anglez_times_enmo\",\n            \"anglez\",\"anglez_diff\",\"anglez_mean\",\"anglez_min\",\"anglez_max\",\"anglez_std\",\"anglez_diff_rolling\",\n            \"enmo\",\"enmo_diff\",\"enmo_mean\",\"enmo_min\",\"enmo_max\",\"enmo_std\",\"enmo_diff_rolling\"]\ntarget = 'awake'\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:07:35.086001Z","iopub.execute_input":"2023-11-21T12:07:35.086444Z","iopub.status.idle":"2023-11-21T12:07:35.093084Z","shell.execute_reply.started":"2023-11-21T12:07:35.086409Z","shell.execute_reply":"2023-11-21T12:07:35.092038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\n\ndt_pipeline = make_pipeline(\n    StandardScaler(),\n    DecisionTreeClassifier(random_state=42)\n)\n\ndt_pipeline.fit(train_set[features], train_set[target])","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:50:18.710948Z","iopub.execute_input":"2023-11-21T12:50:18.711381Z","iopub.status.idle":"2023-11-21T12:52:43.396200Z","shell.execute_reply.started":"2023-11-21T12:50:18.711346Z","shell.execute_reply":"2023-11-21T12:52:43.394221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import lightgbm as lgbm\n\n# lgbm_pipeline = make_pipeline(\n#     StandardScaler(),\n#     lgbm.LGBMClassifier(n_estimators=100, boosting_type='gbdt', random_state=42, n_jobs=-1)\n# )\n\n# lgbm_pipeline.fit(train_set[features], train_set[target])\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:47:43.530523Z","iopub.execute_input":"2023-11-21T12:47:43.531448Z","iopub.status.idle":"2023-11-21T12:48:20.159964Z","shell.execute_reply.started":"2023-11-21T12:47:43.531381Z","shell.execute_reply":"2023-11-21T12:48:20.158991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dt_predictions = dt_pipeline.predict(test_set[features])\n\n\naccuracy = np.mean(dt_predictions == test_set[target])\nprint(\"Accuracy:\", accuracy)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:12:19.062027Z","iopub.execute_input":"2023-11-21T12:12:19.062485Z","iopub.status.idle":"2023-11-21T12:12:19.361008Z","shell.execute_reply.started":"2023-11-21T12:12:19.062449Z","shell.execute_reply":"2023-11-21T12:12:19.359801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test  = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n\ntest  = add_features(test)\n\nX_test = test[features]\n\ntest[\"not_awake\"] = dt_pipeline.predict_proba(X_test)[:,0]\ntest[\"awake\"]     = dt_pipeline.predict_proba(X_test)[:,1]","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:54:07.269801Z","iopub.execute_input":"2023-11-21T12:54:07.270965Z","iopub.status.idle":"2023-11-21T12:54:07.309875Z","shell.execute_reply.started":"2023-11-21T12:54:07.270914Z","shell.execute_reply":"2023-11-21T12:54:07.308403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# smoothing the predictions\nsmoothing_length = 2*230\ntest[\"score\"]  = test[\"awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\ntest[\"smooth\"] = test[\"not_awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n# re-binarize\ntest[\"smooth\"] = test[\"smooth\"].round()\n\n# https://stackoverflow.com/questions/73777727/how-to-mark-start-end-of-a-series-of-non-null-and-non-0-values-in-a-column-of-a\ndef get_event(df):\n    lstCV = zip(df.series_id, df.smooth)\n    lstPOI = []\n    for (c, v), g in groupby(lstCV, lambda cv: \n                            (cv[0], cv[1]!=0 and not pd.isnull(cv[1]))):\n        llg = sum(1 for item in g)\n        if v is False: \n            lstPOI.extend([0]*llg)\n        else: \n            lstPOI.extend(['onset']+(llg-2)*[0]+['wakeup'] if llg > 1 else [0])\n    return lstPOI\n\ntest[\"event\"] = get_event(test)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:54:15.716914Z","iopub.execute_input":"2023-11-21T12:54:15.717324Z","iopub.status.idle":"2023-11-21T12:54:15.732325Z","shell.execute_reply.started":"2023-11-21T12:54:15.717293Z","shell.execute_reply":"2023-11-21T12:54:15.730927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = test.loc[test[\"event\"] != 0][[\"series_id\",\"step\",\"event\",\"score\"]].copy().reset_index(drop=True).reset_index(names=\"row_id\")\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:54:18.990231Z","iopub.execute_input":"2023-11-21T12:54:18.990629Z","iopub.status.idle":"2023-11-21T12:54:19.002478Z","shell.execute_reply.started":"2023-11-21T12:54:18.990596Z","shell.execute_reply":"2023-11-21T12:54:19.001062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ouput = pd.read_csv('/kaggle/working/submission.csv')\n\nouput","metadata":{"execution":{"iopub.status.busy":"2023-11-21T12:55:13.077453Z","iopub.execute_input":"2023-11-21T12:55:13.077986Z","iopub.status.idle":"2023-11-21T12:55:13.092725Z","shell.execute_reply.started":"2023-11-21T12:55:13.077949Z","shell.execute_reply":"2023-11-21T12:55:13.091457Z"},"trusted":true},"execution_count":null,"outputs":[]}]}