{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.display import Markdown\nimport gc","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:35:22.518150Z","iopub.execute_input":"2024-05-21T09:35:22.518548Z","iopub.status.idle":"2024-05-21T09:35:23.623104Z","shell.execute_reply.started":"2024-05-21T09:35:22.518514Z","shell.execute_reply":"2024-05-21T09:35:23.621612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:35:27.466666Z","iopub.execute_input":"2024-05-21T09:35:27.467179Z","iopub.status.idle":"2024-05-21T09:35:27.601878Z","shell.execute_reply.started":"2024-05-21T09:35:27.467144Z","shell.execute_reply":"2024-05-21T09:35:27.600662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:35:48.925505Z","iopub.execute_input":"2024-05-21T09:35:48.926537Z","iopub.status.idle":"2024-05-21T09:35:48.948895Z","shell.execute_reply.started":"2024-05-21T09:35:48.926498Z","shell.execute_reply":"2024-05-21T09:35:48.947497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.groupby('series_id').head(1)[\"event\"].unique()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:37:05.569822Z","iopub.execute_input":"2024-05-21T09:37:05.570295Z","iopub.status.idle":"2024-05-21T09:37:05.593167Z","shell.execute_reply.started":"2024-05-21T09:37:05.570261Z","shell.execute_reply":"2024-05-21T09:37:05.591868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.groupby('series_id').tail(1)[\"event\"].unique()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:37:37.839466Z","iopub.execute_input":"2024-05-21T09:37:37.839861Z","iopub.status.idle":"2024-05-21T09:37:37.851526Z","shell.execute_reply.started":"2024-05-21T09:37:37.839830Z","shell.execute_reply":"2024-05-21T09:37:37.850230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_has_NaN = train_events.groupby('series_id')['step'].apply(lambda x: x.isnull().any())\nseries_has_NaN.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:37:52.361066Z","iopub.execute_input":"2024-05-21T09:37:52.361438Z","iopub.status.idle":"2024-05-21T09:37:52.404508Z","shell.execute_reply.started":"2024-05-21T09:37:52.361410Z","shell.execute_reply":"2024-05-21T09:37:52.403454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"37 series have no NaN values ","metadata":{}},{"cell_type":"code","source":"no_NaN_series = series_has_NaN[~series_has_NaN].index.tolist()\nno_NaN_series","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:38:32.276686Z","iopub.execute_input":"2024-05-21T09:38:32.277095Z","iopub.status.idle":"2024-05-21T09:38:32.286065Z","shell.execute_reply.started":"2024-05-21T09:38:32.277060Z","shell.execute_reply":"2024-05-21T09:38:32.284611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"no_NaN_series.remove('31011ade7c0a') # incomplete events data\nno_NaN_series.remove('a596ad0b82aa') # incomplete events data","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:41:22.380837Z","iopub.execute_input":"2024-05-21T09:41:22.381224Z","iopub.status.idle":"2024-05-21T09:41:22.386494Z","shell.execute_reply.started":"2024-05-21T09:41:22.381191Z","shell.execute_reply":"2024-05-21T09:41:22.385268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_series(series):\n    train_series = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", filters=[('series_id','=',series)])\n    train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\").query('series_id == @series')\n    \n    train_events = train_events.dropna()\n    train_events[\"step\"]  = train_events[\"step\"].astype(\"int\")\n    train_events[\"awake\"] = train_events[\"event\"].replace({\"onset\":1,\"wakeup\":0})\n\n    train = pd.merge(train_series, train_events[['step','awake']], on='step', how='left')\n    train[\"awake\"] = train[\"awake\"].bfill(axis ='rows')\n    # final section:\n    # train_events.groupby('series_id').tail(1)[\"event\"].unique()\n    # Result: the last event is always a \"wakeup\"\n    train['awake'] = train['awake'].fillna(1) # awake\n    train[\"awake\"] = train[\"awake\"].astype(\"int\")\n    return(train)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:41:39.060500Z","iopub.execute_input":"2024-05-21T09:41:39.060860Z","iopub.status.idle":"2024-05-21T09:41:39.068512Z","shell.execute_reply.started":"2024-05-21T09:41:39.060825Z","shell.execute_reply":"2024-05-21T09:41:39.067336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smaller_train_data = []\n\nfor series_id in no_NaN_series:\n    train = get_train_series(series_id)\n    smaller_train_data.append(train)\n    \n    # vizualize these series\n    display(Markdown('###  anglez for series ' + series_id))\n    fig, ax = plt.subplots(figsize=(20, 3))\n    sns.lineplot(data=train, x=\"step\", y=\"anglez\",hue=\"awake\", linewidth = 0.5)\n    plt.show();\n    display(Markdown('###  enmo for series ' + series_id))\n    fig, ax = plt.subplots(figsize=(20, 3))\n    sns.lineplot(data=train, x=\"step\", y=\"enmo\",hue=\"awake\", linewidth = 0.5)\n    plt.show();\n    del train\n    gc.collect();\n    ","metadata":{"execution":{"iopub.status.busy":"2024-05-21T09:55:39.926837Z","iopub.execute_input":"2024-05-21T09:55:39.927283Z","iopub.status.idle":"2024-05-21T10:02:03.817421Z","shell.execute_reply.started":"2024-05-21T09:55:39.927249Z","shell.execute_reply":"2024-05-21T10:02:03.816283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Zzzs_train = pd.concat(smaller_train_data).reset_index(drop=True)\nZzzs_train[\"series_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:02:03.819221Z","iopub.execute_input":"2024-05-21T10:02:03.819571Z","iopub.status.idle":"2024-05-21T10:02:07.238169Z","shell.execute_reply.started":"2024-05-21T10:02:03.819543Z","shell.execute_reply":"2024-05-21T10:02:07.237065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Zzzs_train","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:24:31.909856Z","iopub.execute_input":"2024-05-21T10:24:31.910283Z","iopub.status.idle":"2024-05-21T10:24:31.926033Z","shell.execute_reply.started":"2024-05-21T10:24:31.910248Z","shell.execute_reply":"2024-05-21T10:24:31.925038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add on the 8 datasets that have no event data ","metadata":{}},{"cell_type":"code","source":"train_series_names = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", columns=['series_id'])\nseries_id_list = train_series_names['series_id'].unique().tolist()\n\ntrain_events = train_events.dropna()\nevents_series_id_list = train_events['series_id'].unique().tolist()\n\nbase_series = list(set(series_id_list) - set(events_series_id_list))\n\n# list out the series:\nbase_series","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:41:12.137085Z","iopub.execute_input":"2024-05-21T10:41:12.137508Z","iopub.status.idle":"2024-05-21T10:41:27.026266Z","shell.execute_reply.started":"2024-05-21T10:41:12.137462Z","shell.execute_reply":"2024-05-21T10:41:27.025198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_series_base(series):\n    train = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", filters=[('series_id','=',series)])\n    train[\"awake\"] = 2\n    # just to make sure\n    train[\"awake\"] = train[\"awake\"].astype(\"int\")\n    return(train)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:42:07.885109Z","iopub.execute_input":"2024-05-21T10:42:07.885508Z","iopub.status.idle":"2024-05-21T10:42:07.891782Z","shell.execute_reply.started":"2024-05-21T10:42:07.885458Z","shell.execute_reply":"2024-05-21T10:42:07.890404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_train_data = []\n\nfor series_id in base_series:\n    train = get_train_series_base(series_id)\n    base_train_data.append(train)\n    \n    # vizualize these series\n    display(Markdown('###  anglez for series ' + series_id))\n    fig, ax = plt.subplots(figsize=(20, 3))\n    sns.lineplot(data=train, x=\"step\", y=\"anglez\", color=\"purple\", linewidth = 0.5)\n    plt.show();\n    display(Markdown('###  enmo for series ' + series_id))\n    fig, ax = plt.subplots(figsize=(20, 3))\n    sns.lineplot(data=train, x=\"step\", y=\"enmo\", color=\"olive\", linewidth = 0.5)\n    plt.show();\n    del train\n    gc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:42:27.040410Z","iopub.execute_input":"2024-05-21T10:42:27.040936Z","iopub.status.idle":"2024-05-21T10:43:47.527255Z","shell.execute_reply.started":"2024-05-21T10:42:27.040819Z","shell.execute_reply":"2024-05-21T10:43:47.526115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Zzzs_base = pd.concat(base_train_data).reset_index(drop=True)\n# combine the 35 series from before with the 8 series that have no event data\nZzzs_train_multi = pd.concat([Zzzs_train,Zzzs_base]).reset_index(drop=True)\n# check number of series\nZzzs_train_multi[\"series_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:43:47.529181Z","iopub.execute_input":"2024-05-21T10:43:47.529545Z","iopub.status.idle":"2024-05-21T10:43:52.173649Z","shell.execute_reply.started":"2024-05-21T10:43:47.529508Z","shell.execute_reply":"2024-05-21T10:43:52.172336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Zzzs_train_multi.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:45:05.947521Z","iopub.execute_input":"2024-05-21T10:45:05.947908Z","iopub.status.idle":"2024-05-21T10:45:05.963705Z","shell.execute_reply.started":"2024-05-21T10:45:05.947870Z","shell.execute_reply":"2024-05-21T10:45:05.962503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = Zzzs_train_multi","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:48:33.831582Z","iopub.execute_input":"2024-05-21T10:48:33.831960Z","iopub.status.idle":"2024-05-21T10:48:33.837853Z","shell.execute_reply.started":"2024-05-21T10:48:33.831928Z","shell.execute_reply":"2024-05-21T10:48:33.836563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_features(df):\n    # parse the timestamp and create an \"hour\" feature\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    \n    periods = 20\n    df[\"anglez\"] = abs(df[\"anglez\"])\n    df[\"anglez_diff\"] = df.groupby('series_id')['anglez'].diff(periods=periods).fillna(method=\"bfill\").astype('float16')\n    df[\"enmo_diff\"] = df.groupby('series_id')['enmo'].diff(periods=periods).fillna(method=\"bfill\").astype('float16')\n    df[\"anglez_rolling_mean\"] = df[\"anglez\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_mean\"] = df[\"enmo\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_rolling_max\"] = df[\"anglez\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_max\"] = df[\"enmo\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_rolling_std\"] = df[\"anglez\"].rolling(periods,center=True).std().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_rolling_std\"] = df[\"enmo\"].rolling(periods,center=True).std().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_diff_rolling_mean\"] = df[\"anglez_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_diff_rolling_mean\"] = df[\"enmo_diff\"].rolling(periods,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"anglez_diff_rolling_max\"] = df[\"anglez_diff\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    df[\"enmo_diff_rolling_max\"] = df[\"enmo_diff\"].rolling(periods,center=True).max().fillna(method=\"bfill\").fillna(method=\"ffill\").astype('float16')\n    \n    return df\n\nfeatures = [\"hour\",\n            \"anglez\",\n            \"anglez_rolling_mean\",\n            \"anglez_rolling_max\",\n            \"anglez_rolling_std\",\n            \"anglez_diff\",\n            \"anglez_diff_rolling_mean\",\n            \"anglez_diff_rolling_max\",\n            \"enmo\",\n            \"enmo_rolling_mean\",\n            \"enmo_rolling_max\",\n            \"enmo_rolling_std\",\n            \"enmo_diff\",\n            \"enmo_diff_rolling_mean\",\n            \"enmo_diff_rolling_max\",\n           ]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:47:23.744807Z","iopub.execute_input":"2024-05-21T10:47:23.745200Z","iopub.status.idle":"2024-05-21T10:47:23.759764Z","shell.execute_reply.started":"2024-05-21T10:47:23.745171Z","shell.execute_reply":"2024-05-21T10:47:23.758631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"train   = make_features(train)\n\nX_train = train[features]\ny_train = train[\"awake\"]\n\n# save some memory\ndel train\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-05-21T10:48:42.490466Z","iopub.execute_input":"2024-05-21T10:48:42.490912Z","iopub.status.idle":"2024-05-21T11:00:42.515961Z","shell.execute_reply.started":"2024-05-21T10:48:42.490832Z","shell.execute_reply":"2024-05-21T11:00:42.514857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfrom sklearn.ensemble import RandomForestClassifier\nclassifier = RandomForestClassifier(n_estimators=50,\n                                    min_samples_leaf=300,\n                                    random_state=42,n_jobs=-1)\n\nclassifier.fit(X_train, y_train)\n\n# save some memory\ndel X_train, y_train\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-05-21T11:05:54.702817Z","iopub.execute_input":"2024-05-21T11:05:54.703928Z","iopub.status.idle":"2024-05-21T11:35:00.999457Z","shell.execute_reply.started":"2024-05-21T11:05:54.703885Z","shell.execute_reply":"2024-05-21T11:35:00.997996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction","metadata":{}},{"cell_type":"code","source":"test  = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n\ntest  = make_features(test)\n\nX_test = test[features]\n\ntest[\"not_awake\"] = classifier.predict_proba(X_test)[:,0]\ntest[\"awake\"]     = classifier.predict_proba(X_test)[:,1]","metadata":{"execution":{"iopub.status.busy":"2024-05-21T11:59:04.532830Z","iopub.execute_input":"2024-05-21T11:59:04.533314Z","iopub.status.idle":"2024-05-21T11:59:04.668384Z","shell.execute_reply.started":"2024-05-21T11:59:04.533277Z","shell.execute_reply":"2024-05-21T11:59:04.667323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Processing","metadata":{}},{"cell_type":"code","source":"# smoothing the predictions\nfrom itertools import groupby\nsmoothing_length = 2*230\ntest[\"score\"]  = test[\"awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\ntest[\"smooth\"] = test[\"not_awake\"].rolling(smoothing_length,center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n# re-binarize\ntest[\"smooth\"] = test[\"smooth\"].round()\n\n# https://stackoverflow.com/questions/73777727/how-to-mark-start-end-of-a-series-of-non-null-and-non-0-values-in-a-column-of-a\ndef get_event(df):\n    lstCV = zip(df.series_id, df.smooth)\n    lstPOI = []\n    for (c, v), g in groupby(lstCV, lambda cv: \n                            (cv[0], cv[1]!=0 and not pd.isnull(cv[1]))):\n        llg = sum(1 for item in g)\n        if v is False: \n            lstPOI.extend([0]*llg)\n        else: \n            lstPOI.extend(['onset']+(llg-2)*[0]+['wakeup'] if llg > 1 else [0])\n    return lstPOI\n\ntest[\"event\"] = get_event(test)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:01:12.119881Z","iopub.execute_input":"2024-05-21T12:01:12.120721Z","iopub.status.idle":"2024-05-21T12:01:12.135108Z","shell.execute_reply.started":"2024-05-21T12:01:12.120686Z","shell.execute_reply":"2024-05-21T12:01:12.133985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submissionn = test.loc[test[\"event\"] != 0][[\"series_id\",\"step\",\"event\",\"score\"]].copy().reset_index(drop=True).reset_index(names=\"row_id\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:01:46.142720Z","iopub.execute_input":"2024-05-21T12:01:46.143160Z","iopub.status.idle":"2024-05-21T12:01:46.154901Z","shell.execute_reply.started":"2024-05-21T12:01:46.143120Z","shell.execute_reply":"2024-05-21T12:01:46.153652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submissionn.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T12:03:32.534531Z","iopub.execute_input":"2024-05-21T12:03:32.534979Z","iopub.status.idle":"2024-05-21T12:03:32.546257Z","shell.execute_reply.started":"2024-05-21T12:03:32.534944Z","shell.execute_reply":"2024-05-21T12:03:32.545111Z"},"trusted":true},"execution_count":null,"outputs":[]}]}