{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom itertools import groupby\nimport gc","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-24T04:34:54.901356Z","iopub.execute_input":"2023-09-24T04:34:54.901680Z","iopub.status.idle":"2023-09-24T04:34:54.906931Z","shell.execute_reply.started":"2023-09-24T04:34:54.901654Z","shell.execute_reply":"2023-09-24T04:34:54.905187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = pd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet\")\ntrain_all = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:34:57.797673Z","iopub.execute_input":"2023-09-24T04:34:57.798107Z","iopub.status.idle":"2023-09-24T04:36:14.795319Z","shell.execute_reply.started":"2023-09-24T04:34:57.798072Z","shell.execute_reply":"2023-09-24T04:36:14.794509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:36:14.824770Z","iopub.execute_input":"2023-09-24T04:36:14.825102Z","iopub.status.idle":"2023-09-24T04:36:14.847389Z","shell.execute_reply.started":"2023-09-24T04:36:14.825078Z","shell.execute_reply":"2023-09-24T04:36:14.845470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insomnia_id = ['05e1944c3818', '0cd1e3d0ed95', '10469f6765bf', '13b4d6a01d27', '16fe2798ed0f',\n               '31011ade7c0a', '3be2f86c3e45', '3df0da2e5966', '5aad18e7ce64', '60d31b0bec3b', \n               '60e51cad2ffb', '703b5efa9bc1', 'a596ad0b82aa', 'fb223ed2278c']","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:36:14.850833Z","iopub.execute_input":"2023-09-24T04:36:14.851171Z","iopub.status.idle":"2023-09-24T04:36:14.859500Z","shell.execute_reply.started":"2023-09-24T04:36:14.851141Z","shell.execute_reply":"2023-09-24T04:36:14.857754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_insomnia = train_all.loc[train_all['series_id'].isin(insomnia_id)]\ntrain_insomnia","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:36:14.861346Z","iopub.execute_input":"2023-09-24T04:36:14.861818Z","iopub.status.idle":"2023-09-24T04:36:18.999800Z","shell.execute_reply.started":"2023-09-24T04:36:14.861778Z","shell.execute_reply":"2023-09-24T04:36:18.998461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_all\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:36:19.001444Z","iopub.execute_input":"2023-09-24T04:36:19.001834Z","iopub.status.idle":"2023-09-24T04:36:20.838637Z","shell.execute_reply.started":"2023-09-24T04:36:19.001804Z","shell.execute_reply":"2023-09-24T04:36:20.837097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_series(series):\n    train_series = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", filters=[('series_id','=',series)])\n    train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\").query('series_id == @series')\n    \n    train_events = train_events.dropna()\n    train_events[\"step\"]  = train_events[\"step\"].astype(\"int\")\n    train_events[\"awake\"] = train_events[\"event\"].replace({\"onset\":1,\"wakeup\":0})\n\n    train = pd.merge(train_series, train_events[['step','awake']], on='step', how='left')\n    train[\"awake\"] = train[\"awake\"].bfill(axis ='rows')\n    # final section:\n    # train_events.groupby('series_id').tail(1)[\"event\"].unique()\n    # Result: the last event is always a \"wakeup\"\n    train['awake'] = train['awake'].fillna(1) # awake\n    train[\"awake\"] = train[\"awake\"].astype(\"int\")\n    return(train)","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:36:20.840642Z","iopub.execute_input":"2023-09-24T04:36:20.841085Z","iopub.status.idle":"2023-09-24T04:36:20.860431Z","shell.execute_reply.started":"2023-09-24T04:36:20.841051Z","shell.execute_reply":"2023-09-24T04:36:20.856355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import Markdown","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:42:39.209838Z","iopub.execute_input":"2023-09-24T04:42:39.210369Z","iopub.status.idle":"2023-09-24T04:42:39.217122Z","shell.execute_reply.started":"2023-09-24T04:42:39.210327Z","shell.execute_reply":"2023-09-24T04:42:39.215522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:46:06.249842Z","iopub.execute_input":"2023-09-24T04:46:06.250306Z","iopub.status.idle":"2023-09-24T04:46:06.902364Z","shell.execute_reply.started":"2023-09-24T04:46:06.250277Z","shell.execute_reply":"2023-09-24T04:46:06.900764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smaller_train_data = []\ninsomnia_train = pd.DataFrame()\n\nfor series_id in insomnia_id:\n    train = get_train_series(series_id)\n    #smaller_train_data.append(train)\n    insomnia_train = pd.concat([insomnia_train, train], axis=0)\n    \n    # vizualize these series\n    display(Markdown('###  anglez for series ' + series_id))\n    fig, ax = plt.subplots(figsize=(20, 3))\n    sns.lineplot(data=train, x=\"step\", y=\"anglez\",hue=\"awake\", linewidth = 0.5)\n    plt.show();\n    display(Markdown('###  enmo for series ' + series_id))\n    fig, ax = plt.subplots(figsize=(20, 3))\n    sns.lineplot(data=train, x=\"step\", y=\"enmo\",hue=\"awake\", linewidth = 0.5)\n    plt.show();\n    del train\n    gc.collect();","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:56:14.653493Z","iopub.execute_input":"2023-09-24T04:56:14.653919Z","iopub.status.idle":"2023-09-24T04:58:33.412786Z","shell.execute_reply.started":"2023-09-24T04:56:14.653889Z","shell.execute_reply":"2023-09-24T04:58:33.409990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Feature engineering","metadata":{}},{"cell_type":"code","source":"insomnia_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:59:18.440796Z","iopub.execute_input":"2023-09-24T04:59:18.441293Z","iopub.status.idle":"2023-09-24T04:59:18.455894Z","shell.execute_reply.started":"2023-09-24T04:59:18.441263Z","shell.execute_reply":"2023-09-24T04:59:18.453531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_features(df):\n    # parse the timestamp and create an \"hour\" feature\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    \n    periods = 20\n    df[\"anglez\"] = abs(df[\"anglez\"])\n    df[\"anglez_diff\"] = df.groupby('series_id')['anglez'].diff(periods=periods).fillna(method=\"bfill\").astype('float16')\n    df[\"enmo_diff\"] = df.groupby('series_id')['enmo'].diff(periods=periods).fillna(method=\"bfill\").astype('float16')\n    df[\"anglez_rolling_mean\"] = df[\"anglez\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_mean\"] = df[\"enmo\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_rolling_max\"] = df[\"anglez\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_max\"] = df[\"enmo\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_rolling_std\"] = df[\"anglez\"].rolling(periods,center=True).std().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_std\"] = df[\"enmo\"].rolling(periods,center=True).std().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_diff_rolling_mean\"] = df[\"anglez_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_diff_rolling_mean\"] = df[\"enmo_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_diff_rolling_max\"] = df[\"anglez_diff\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_diff_rolling_max\"] = df[\"enmo_diff\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    \n    return df\n\nfeatures = [\"hour\",\"anglez\",\"anglez_rolling_mean\",\"anglez_rolling_max\",\"anglez_rolling_std\",\"anglez_diff\",\n            \"anglez_diff_rolling_mean\",\"anglez_diff_rolling_max\",\"enmo\",\"enmo_rolling_mean\",\n            \"enmo_rolling_max\",\"enmo_rolling_std\",\"enmo_diff\",\"enmo_diff_rolling_mean\",\"enmo_diff_rolling_max\",]","metadata":{"execution":{"iopub.status.busy":"2023-09-24T04:59:24.690600Z","iopub.execute_input":"2023-09-24T04:59:24.691052Z","iopub.status.idle":"2023-09-24T04:59:24.711348Z","shell.execute_reply.started":"2023-09-24T04:59:24.691024Z","shell.execute_reply":"2023-09-24T04:59:24.708842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Training","metadata":{}},{"cell_type":"code","source":"train   = make_features(insomnia_train)\n\nX_train = train[features]\ny_train = train[\"awake\"]","metadata":{"execution":{"iopub.status.busy":"2023-09-24T05:00:51.774473Z","iopub.execute_input":"2023-09-24T05:00:51.774766Z","iopub.status.idle":"2023-09-24T05:01:26.791599Z","shell.execute_reply.started":"2023-09-24T05:00:51.774733Z","shell.execute_reply":"2023-09-24T05:01:26.790525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-24T05:01:26.793040Z","iopub.execute_input":"2023-09-24T05:01:26.793334Z","iopub.status.idle":"2023-09-24T05:01:26.814640Z","shell.execute_reply.started":"2023-09-24T05:01:26.793310Z","shell.execute_reply":"2023-09-24T05:01:26.812133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[60:80]","metadata":{"execution":{"iopub.status.busy":"2023-09-24T05:01:26.818550Z","iopub.execute_input":"2023-09-24T05:01:26.819075Z","iopub.status.idle":"2023-09-24T05:01:26.858187Z","shell.execute_reply.started":"2023-09-24T05:01:26.819036Z","shell.execute_reply":"2023-09-24T05:01:26.855872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save some memory\ndel train\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2023-09-24T05:23:17.023073Z","iopub.execute_input":"2023-09-24T05:23:17.023588Z","iopub.status.idle":"2023-09-24T05:23:17.158335Z","shell.execute_reply.started":"2023-09-24T05:23:17.023555Z","shell.execute_reply":"2023-09-24T05:23:17.156380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfrom sklearn.ensemble import GradientBoostingClassifier\n#classifier = GradientBoostingClassifier(min_samples_leaf=100,n_estimators=50)\nclassifier = GradientBoostingClassifier(min_samples_leaf=300, max_depth=3, n_estimators=50)\n\nclassifier.fit(X_train, y_train)\n\n\ndel X_train, y_train\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2023-09-24T05:23:21.283307Z","iopub.execute_input":"2023-09-24T05:23:21.283962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#classifier = GradientBoostingClassifier(n_estimators=50,max_depth=4,min_samples_leaf=300,random_state=42)\n\n#clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0,\n#max_depth=1, random_state=0).fit(X_train, y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Predictions","metadata":{}},{"cell_type":"code","source":"test  = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n\ntest  = make_features(test)\n\nX_test = test[features]\n\ntest[\"not_awake\"] = classifier.predict_proba(X_test)[:,0]\ntest[\"awake\"]     = classifier.predict_proba(X_test)[:,1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Processing","metadata":{}},{"cell_type":"code","source":"# smoothing the predictions\nsmoothing_length = 2*230\ntest[\"score\"]  = test[\"awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\ntest[\"smooth\"] = test[\"not_awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n# re-binarize\ntest[\"smooth\"] = test[\"smooth\"].round()\n\n\ndef get_event(df):\n    lstCV = zip(df.series_id, df.smooth)\n    lstPOI = []\n    for (c, v), g in groupby(lstCV, lambda cv: \n                            (cv[0], cv[1]!=0 and not pd.isnull(cv[1]))):\n        llg = sum(1 for item in g)\n        if v is False: \n            lstPOI.extend([0]*llg)\n        else: \n            lstPOI.extend(['onset']+(llg-2)*[0]+['wakeup'] if llg > 1 else [0])\n    return lstPOI\n\ntest[\"event\"] = get_event(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#smoothing_length = 2*230","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Write in submission file","metadata":{}},{"cell_type":"code","source":"sample_submission = test.loc[test[\"event\"] != 0][[\"series_id\",\"step\",\"event\",\"score\"]].copy().reset_index(drop=True).reset_index(names=\"row_id\")\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}