{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6507308,"sourceType":"datasetVersion","datasetId":3706667},{"sourceId":6890006,"sourceType":"datasetVersion","datasetId":3958051}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Sleep Classification Algorithm Other Reference paper : Link \n\n#### [Sleep classification from wrist-worn accelerometer data using random forests](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7794504/)\n\n#### [Three-level Sleep Stage Classification Based on Wrist-worn Accelerometry Data Alone](https://www.biorxiv.org/content/10.1101/2021.08.10.455812v1.full)\n\n#### [A Novel, Open Access Method to Assess Sleep Duration Using a Wrist-Worn Accelerometer](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0142533)\n\n#### [Accelerometer data processing with GGIR](https://cran.r-project.org/web/packages/GGIR/vignettes/GGIR.html)\n\n#### [Self-supervised learning of accelerometer data provides new 2 insights for sleep and its association with mortality](https://www.medrxiv.org/content/medrxiv/early/2023/07/08/2023.07.07.23292251.full.pdf)\n\n\n### Terminology \n\n- ENMO : The Euclidean Norm Minus One (ENMO) with negative values rounded to zero in g has been shown to correlate with the magnitude of acceleration and human energy expenditure\n- Z-angle: Z-angle, computed using Eq. 3, corresponds to the angle between the accelerometer axis perpendicular to the skin surface and the horizontal plane. As described in “vanHees approach”, any change (or lack of change) in the z-angle over successive time intervals may be an indicator of posture change.\n- LIDS: Locomotor Inactivity During Sleep (LIDS) involves a non-linear conversion of locomotor activity and has shown to be sensitive to ultradian sleep cycles.","metadata":{}},{"cell_type":"markdown","source":"## Use Lightwight training dataset\n### [ZZs-lghtweight Training dataset for reduce memory usage](https://www.kaggle.com/datasets/carlmcbrideellis/zzzs-lightweight-training-dataset-target)\n\n## Reference: Load dataset efienient (handles memory usage issues for large files such as train_series overflow Memory usage (over 30GB memory require)\n### [Efficient loading+Memory Usage+Visualizations⚡#CMI](https://www.kaggle.com/code/henriupton/efficient-loading-memory-usage-visualizations-cmi)","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport os\nfrom os import system, getpid, walk\nfrom psutil import Process;\nimport joblib\nfrom joblib import Parallel, delayed\nimport random\nimport math\nfrom tqdm import tqdm\n\n#plot graph library\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly\nimport plotly.express as px\nimport plotly.graph_objs as go\nfrom IPython.display import Markdown\n\n#pyarrow\nimport pyarrow as pq\nimport pyarrow.csv as csv\nfrom pyarrow.parquet import ParquetFile  # PyArrow Parquet file handling ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-27T13:53:34.508769Z","iopub.execute_input":"2023-11-27T13:53:34.509063Z","iopub.status.idle":"2023-11-27T13:53:37.167674Z","shell.execute_reply.started":"2023-11-27T13:53:34.509035Z","shell.execute_reply":"2023-11-27T13:53:37.166707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import ML differnt model\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\nfrom catboost import CatBoostClassifier\nfrom sklearn.ensemble import RandomForestClassifier\n# from cuml.ensemble import RandomForestClassifier # gpu version ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:53:37.169448Z","iopub.execute_input":"2023-11-27T13:53:37.170487Z","iopub.status.idle":"2023-11-27T13:53:40.903946Z","shell.execute_reply.started":"2023-11-27T13:53:37.170454Z","shell.execute_reply":"2023-11-27T13:53:40.902917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import metric \nfrom sklearn.model_selection import train_test_split, KFold, GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import roc_curve, roc_auc_score, classification_report, accuracy_score\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nfrom sklearn.utils import class_weight","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:53:43.880666Z","iopub.execute_input":"2023-11-27T13:53:43.881469Z","iopub.status.idle":"2023-11-27T13:53:43.886409Z","shell.execute_reply.started":"2023-11-27T13:53:43.881435Z","shell.execute_reply":"2023-11-27T13:53:43.885391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"dev = False # set development mode","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:53:53.635611Z","iopub.execute_input":"2023-11-27T13:53:53.635983Z","iopub.status.idle":"2023-11-27T13:53:53.640520Z","shell.execute_reply.started":"2023-11-27T13:53:53.635953Z","shell.execute_reply":"2023-11-27T13:53:53.639542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Tolerances","metadata":{}},{"cell_type":"code","source":"#define tolerance interval \ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], \n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n}\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:53:57.321207Z","iopub.execute_input":"2023-11-27T13:53:57.321559Z","iopub.status.idle":"2023-11-27T13:53:57.326924Z","shell.execute_reply.started":"2023-11-27T13:53:57.321529Z","shell.execute_reply":"2023-11-27T13:53:57.325978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define DataPath","metadata":{}},{"cell_type":"code","source":"local = False  # true for local ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:53:59.660476Z","iopub.execute_input":"2023-11-27T13:53:59.661089Z","iopub.status.idle":"2023-11-27T13:53:59.665126Z","shell.execute_reply.started":"2023-11-27T13:53:59.661058Z","shell.execute_reply":"2023-11-27T13:53:59.664100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if local == True:\n    mainDir = \"./data/\"\n    rootDir = \"./data\"\nelse:\n    mainDir = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    rootDir = \"/kaggle/input\"\n\n#define all file path\nsubMitSampleDir = mainDir + \"sample_submission.csv\"\ntrainDir  = mainDir + \"train_series.parquet\"\ntrainEventDir = mainDir + \"train_events.csv\"\ntestDir  = mainDir  + \"test_series.parquet\"    \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:01.638653Z","iopub.execute_input":"2023-11-27T13:54:01.639007Z","iopub.status.idle":"2023-11-27T13:54:01.644309Z","shell.execute_reply.started":"2023-11-27T13:54:01.638979Z","shell.execute_reply":"2023-11-27T13:54:01.643358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## load Dataset","metadata":{}},{"cell_type":"code","source":"# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk(rootDir):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:04.225728Z","iopub.execute_input":"2023-11-27T13:54:04.226403Z","iopub.status.idle":"2023-11-27T13:54:04.250067Z","shell.execute_reply.started":"2023-11-27T13:54:04.226352Z","shell.execute_reply":"2023-11-27T13:54:04.247334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Memory Reduce","metadata":{}},{"cell_type":"code","source":"def cleanMemory():\n    gc.collect()\n    ctypes.CDLL(\"libc.so.6\").malloc_trim(0)\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:05.864343Z","iopub.execute_input":"2023-11-27T13:54:05.865219Z","iopub.status.idle":"2023-11-27T13:54:05.869603Z","shell.execute_reply.started":"2023-11-27T13:54:05.865186Z","shell.execute_reply":"2023-11-27T13:54:05.868620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduceMemoryUse(df):\n    \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB (before Memory Reduce)'.format(start_mem))\n    #loop for each column\n    for col in df.columns:\n        colType = df[col].dtype #get data type of column\n        print(\"Column Type: \", colType)\n        if colType != object and colType != 'category' and colType !=\"datetime64[ns]\": #check object type\n            colMin = df[col].min() # get min value of column\n            colMax = df[col].max() # get max value of column\n            if str(colType) == \"int\":   # for int \n                if colMin > np.iinfo(np.int8).min and colMax < np.iinfo(np.int8).max: # within int8\n                    df[col] = df[col].astype(np.int8) # convert to int8\n                elif colMin > np.iinfo(np.int16).min and colMax < np.iinfo(np.int16).max: # within int16\n                    df[col] = df[col].astype(np.int16) # convert to int16\n                elif colMin > np.iinfo(np.int32).min and colMax < np.iinfo(np.int32).max: # within int32\n                    df[col] = df[col].astype(np.int32) # convert to int32 \n                elif colMin > np.iinfo(np.int64).min and colMax < np.iinfo(np.int64).max: # within int64\n                    df[col] = df[col].astype(np.int64)\n                    \n            \n            else: # for float point\n                if colMin > np.finfo(np.float16).min and colMax < np.finfo(np.float16).max: # within float16 range\n                    df[col] = df[col].astype(np.float16) # convert to float 16\n                elif colMin > np.finfo(np.float32).min and colMax < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32) # convert to float 32\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n        else:\n            df[col] = df[col].astype('category')\n        \n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    return df\n                    \n                \ndef checkMemory():\n    \"\"\"\n    This function defines the memory usage across the kernel. \n    Source-\n    https://stackoverflow.com/questions/61366458/how-to-find-memory-usage-of-kaggle-notebook\n    \"\"\";\n    pid = getpid();\n    py = Process(pid);\n    memory_use = py.memory_info()[0] / 2. ** 30;\n    return f\"RAM memory GB usage = {memory_use :.4}\";\n    \n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:08.456656Z","iopub.execute_input":"2023-11-27T13:54:08.457011Z","iopub.status.idle":"2023-11-27T13:54:08.470864Z","shell.execute_reply.started":"2023-11-27T13:54:08.456981Z","shell.execute_reply":"2023-11-27T13:54:08.469959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkMemory()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:09.135224Z","iopub.execute_input":"2023-11-27T13:54:09.135573Z","iopub.status.idle":"2023-11-27T13:54:09.142767Z","shell.execute_reply.started":"2023-11-27T13:54:09.135544Z","shell.execute_reply":"2023-11-27T13:54:09.141879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cleaning Data","metadata":{}},{"cell_type":"markdown","source":"### Remove no event sensor id","metadata":{}},{"cell_type":"code","source":"trainEvent = pd.read_csv(trainEventDir)\n# subSample = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:10.375747Z","iopub.execute_input":"2023-11-27T13:54:10.376460Z","iopub.status.idle":"2023-11-27T13:54:10.414169Z","shell.execute_reply.started":"2023-11-27T13:54:10.376424Z","shell.execute_reply":"2023-11-27T13:54:10.413180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainEvent.event.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:12.009592Z","iopub.execute_input":"2023-11-27T13:54:12.010191Z","iopub.status.idle":"2023-11-27T13:54:12.025440Z","shell.execute_reply.started":"2023-11-27T13:54:12.010161Z","shell.execute_reply":"2023-11-27T13:54:12.024459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainEvent.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:12.976406Z","iopub.execute_input":"2023-11-27T13:54:12.976793Z","iopub.status.idle":"2023-11-27T13:54:12.988858Z","shell.execute_reply.started":"2023-11-27T13:54:12.976764Z","shell.execute_reply":"2023-11-27T13:54:12.987851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check that all of the series start off in the `awake` state (*i.e.* the very first `event` is the `onset` of sleep):","metadata":{}},{"cell_type":"code","source":"trainEvent.groupby('series_id').head(1)[\"event\"].unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:15.065185Z","iopub.execute_input":"2023-11-27T13:54:15.065531Z","iopub.status.idle":"2023-11-27T13:54:15.079685Z","shell.execute_reply.started":"2023-11-27T13:54:15.065503Z","shell.execute_reply":"2023-11-27T13:54:15.078679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check that all of the series end in the `awake` state (*i.e.* the very last `event` is a `wakeup`):","metadata":{}},{"cell_type":"code","source":"trainEvent.groupby('series_id').tail(1)[\"event\"].unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:16.101482Z","iopub.execute_input":"2023-11-27T13:54:16.101815Z","iopub.status.idle":"2023-11-27T13:54:16.112385Z","shell.execute_reply.started":"2023-11-27T13:54:16.101789Z","shell.execute_reply":"2023-11-27T13:54:16.111398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### How many series have `NaN` values?","metadata":{}},{"cell_type":"code","source":"## Check NaN step \nnanSeries =trainEvent.groupby(\"series_id\")[\"step\"].apply(lambda x: x.isna().any()) # return true/false\nnanSeries.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:18.244186Z","iopub.execute_input":"2023-11-27T13:54:18.244533Z","iopub.status.idle":"2023-11-27T13:54:18.293604Z","shell.execute_reply.started":"2023-11-27T13:54:18.244504Z","shell.execute_reply":"2023-11-27T13:54:18.292610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 37 series have no NaN values whatsoever:","metadata":{}},{"cell_type":"code","source":"noNaNseries = nanSeries[~nanSeries].index.to_list() #filter 37 series noNaN series ID \nnoNaNseries","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:54:20.078891Z","iopub.execute_input":"2023-11-27T13:54:20.079288Z","iopub.status.idle":"2023-11-27T13:54:20.086965Z","shell.execute_reply.started":"2023-11-27T13:54:20.079259Z","shell.execute_reply":"2023-11-27T13:54:20.085795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check \ntrainEvent[trainEvent[\"series_id\"] == '08db4255286f'] , trainEvent[trainEvent[\"series_id\"] == '08db4255286f'].count()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:55:33.047709Z","iopub.execute_input":"2023-11-27T13:55:33.048081Z","iopub.status.idle":"2023-11-27T13:55:33.069890Z","shell.execute_reply.started":"2023-11-27T13:55:33.048051Z","shell.execute_reply":"2023-11-27T13:55:33.068844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get train data by series ID","metadata":{}},{"cell_type":"code","source":"def getTrainSeries(series):\n    trainSeries = pd.read_parquet(trainDir, filters=[('series_id','=',series)])\n    trainEvent = pd.read_csv(trainEventDir).query('series_id == @series')\n    \n    trainEvent = trainEvent.dropna()\n    trainEvent[\"step\"]  = trainEvent[\"step\"].astype(\"int\") # conver step int\n    trainEvent[\"awake\"] = trainEvent[\"event\"].replace({\"onset\":1,\"wakeup\":0}) #replace onset value to 1, wakeup value to 0 \n\n    train = pd.merge(trainSeries, trainEvent[['step','awake']], on='step', how='left')\n    train[\"awake\"] = train[\"awake\"].bfill(axis ='rows')  #back fill missing vlaue \n    # final section:\n    # train_events.groupby('series_id').tail(1)[\"event\"].unique()\n    # Result: the last event is always a \"wakeup\"\n    train['awake'] = train['awake'].fillna(1) # awake\n    train[\"awake\"] = train[\"awake\"].astype(\"int\")\n    return(train)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:55:59.215541Z","iopub.execute_input":"2023-11-27T13:55:59.216371Z","iopub.status.idle":"2023-11-27T13:55:59.223320Z","shell.execute_reply.started":"2023-11-27T13:55:59.216336Z","shell.execute_reply":"2023-11-27T13:55:59.222319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load train / Test dataset","metadata":{}},{"cell_type":"markdown","source":"## filter the valid sensor and visualize the event","metadata":{}},{"cell_type":"code","source":"def visualAllValidData(nonNaNSeries):\n    #get training set by valid sensor\n    for seriesID in nonNaNSeries:\n        train = getTrainSeries(seriesID)\n        # vizualize these series\n        display(Markdown('###  anglez for series id : ' + seriesID))\n        fig, ax = plt.subplots(figsize=(20, 3))\n        sns.lineplot(data=train, x=\"step\", y=\"anglez\",hue=\"awake\", linewidth = 0.5)\n        plt.show();\n        #display enmo \n        display(Markdown('###  enmo for series id : ' + seriesID))\n        fig, ax = plt.subplots(figsize=(20, 3))\n        sns.lineplot(data=train, x=\"step\", y=\"enmo\",hue=\"awake\", linewidth = 0.5)\n        plt.show();\n        del train\n        gc.collect();","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:56:08.835453Z","iopub.execute_input":"2023-11-27T13:56:08.835818Z","iopub.status.idle":"2023-11-27T13:56:08.842891Z","shell.execute_reply.started":"2023-11-27T13:56:08.835788Z","shell.execute_reply":"2023-11-27T13:56:08.841956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA analysis","metadata":{}},{"cell_type":"code","source":"if dev == True:\n    visualAllValidData(noNaNseries)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:56:12.624618Z","iopub.execute_input":"2023-11-27T13:56:12.625449Z","iopub.status.idle":"2023-11-27T13:56:12.629495Z","shell.execute_reply.started":"2023-11-27T13:56:12.625415Z","shell.execute_reply":"2023-11-27T13:56:12.628535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### find incomplete event data  (e.g. 31011ade7c0a,  a596ad0b82aa) , remove from valid sensor list","metadata":{}},{"cell_type":"code","source":"# find e\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:56:15.526831Z","iopub.execute_input":"2023-11-27T13:56:15.527244Z","iopub.status.idle":"2023-11-27T13:56:15.661134Z","shell.execute_reply.started":"2023-11-27T13:56:15.527212Z","shell.execute_reply":"2023-11-27T13:56:15.660170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# also drop these two \"truncated\" events series seen in EDA:\nnoNaNseries.remove('31011ade7c0a') # incomplete events data\nnoNaNseries.remove('a596ad0b82aa') # incomplete events data","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:56:17.942847Z","iopub.execute_input":"2023-11-27T13:56:17.943688Z","iopub.status.idle":"2023-11-27T13:56:17.947860Z","shell.execute_reply.started":"2023-11-27T13:56:17.943658Z","shell.execute_reply":"2023-11-27T13:56:17.946943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkMemory()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:56:18.661648Z","iopub.execute_input":"2023-11-27T13:56:18.662396Z","iopub.status.idle":"2023-11-27T13:56:18.668649Z","shell.execute_reply.started":"2023-11-27T13:56:18.662360Z","shell.execute_reply":"2023-11-27T13:56:18.667665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"### Get unqies series IDs","metadata":{}},{"cell_type":"code","source":"def extractUniqueSeriesID(df):\n    return df[\"series_id\"].unique()\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:56:23.308459Z","iopub.execute_input":"2023-11-27T13:56:23.309267Z","iopub.status.idle":"2023-11-27T13:56:23.313218Z","shell.execute_reply.started":"2023-11-27T13:56:23.309235Z","shell.execute_reply":"2023-11-27T13:56:23.312342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def isoTimeConvertDateTime(df, timeCol=\"timestamp\", setTimeIndex=True):\n    df[timeCol]= pd.to_datetime(df[timeCol]) # convert isotime to datetime\n    df[\"hour\"]= df[timeCol].dt.hour # extract hour \n    df[\"month\"]= df[timeCol].dt.month # extract month\n    #extract 4 season for month\n    df[\"spring\"] = df[\"month\"].apply(lambda x: 1 if x in [3,4,5] else 0 ) # one hot encoding for spring\n    df[\"summer\"] = df[\"month\"].apply(lambda x: 1 if x in  [6,7,8] else 0 ) # one hot encoding for summer\n    df[\"autumn\"] = df[\"month\"].apply(lambda x: 1 if x in [9,10,11] else 0 ) # one hot encoding for autumn\n    df[\"winter\"] = df[\"month\"].apply(lambda x: 1 if x in [1,2,12] else 0 ) # one hot encoding for winter\n    #extract day of week \n    df[\"day_of_week_index\"]= df[timeCol].dt.weekday\n    \n    if setTimeIndex == True:\n        df.sort_values([timeCol], inplace=True) # sort time column value\n        df.set_index(timeCol, inplace=True) # set pandas dataframe DateTime index\n\n    return df\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:56:28.720565Z","iopub.execute_input":"2023-11-27T13:56:28.721273Z","iopub.status.idle":"2023-11-27T13:56:28.730102Z","shell.execute_reply.started":"2023-11-27T13:56:28.721239Z","shell.execute_reply":"2023-11-27T13:56:28.729047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def featureEng(df, mainCol=\"enmo\"):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    #First convert time\n    df[\"timestamp\"]= pd.to_datetime(df[\"timestamp\"]).apply(lambda t: t.tz_localize(None)) # convert isotime to datetime\n    df[\"hour\"]= df[\"timestamp\"].dt.hour # extract hour \n    df[\"month\"]= df[\"timestamp\"].dt.month # extract month\n    #extract 4 season for month\n    df[\"spring\"] = df[\"month\"].apply(lambda x: 1 if x in [3,4,5] else 0 ) # one hot encoding for spring\n    df[\"summer\"] = df[\"month\"].apply(lambda x: 1 if x in  [6,7,8] else 0 ) # one hot encoding for summer\n    df[\"autumn\"] = df[\"month\"].apply(lambda x: 1 if x in [9,10,11] else 0 ) # one hot encoding for autumn\n    df[\"winter\"] = df[\"month\"].apply(lambda x: 1 if x in [1,2,12] else 0 ) # one hot encoding for winter\n    #extract day of week \n    df[\"day_of_week_index\"]= df[\"timestamp\"].dt.weekday\n    df.sort_values([\"timestamp\"], inplace=True) # sort time column value\n    \n#     if setTimeIndex == True:\n    df.set_index(\"timestamp\", inplace=True) # set pandas dataframe DateTime index\n    #add timediffer\n    df[\"timeDiffSeconds\"] =df.index.to_series().diff().dt.total_seconds()\n\n    \n    # LIDS: Locomotor Inactivity During Sleep (LIDS) feature extract 5sec per sample\n    # LIDS equation: 100/activity count +1 \n    # LIDS 30 minutes Window \n    df[\"lids\"] = np.maximum(0., df[mainCol] - 0.02) # find maximum value \n    df[\"lids\"] = df[\"lids\"].rolling(window=120, center=True, min_periods=1).agg(\"sum\")#  10 mintues window sum 60*10/5 =120 window step\n    df[\"lids\"] = 100 /(df[\"lids\"] + 1) # LIDS equation calculate\n    df[\"lids\"] = df[\"lids\"].rolling(window=360, center=True, min_periods=1).agg(\"mean\").astype(np.float16) # 30 minture 30*10/5 = 360\n    \n    df[mainCol]= (df[mainCol] *1000).astype(np.int16) # convert to int16\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float16) \n    \n#     periods = [60, 300, 600, 3600]\n    periods = [60, 360, 720, 3600] \n    #extract feature min, max, median with different period, for column \n    for col in [\"enmo\", \"anglez\", \"anglezdiffabs\"]:\n            for p in periods:\n                rolArgs = {\"window\": p+1, 'min_periods':10, 'center':True}\n                # calculate each aggrage  \n                for agg in ['median', 'mean', 'max', 'min']:\n                    df[f'{col}_{p}_{agg}'] = df[col].rolling(**rolArgs).agg(agg).astype(np.float16).values\n                    gc.collect()\n                    \n                for agg in ['var']:\n                    df[f'{col}_{p}_{agg}'] = df[col].rolling(**rolArgs).agg(agg).astype(np.float32).values\n                    gc.collect()\n                #find median fo\n                if p == max(periods):\n                    df[f'{col}_{p}_mad'] = (df[col] - df[f'{col}_{p}_median']).abs().rolling(**rolArgs).median().astype(np.float32)\n                    \n                df[f'{col}_{p}_amplit'] = df[f'{col}_{p}_max'] - df[f'{col}_{p}_min'] #find amplitude\n                df[f'{col}_diff_{p}_max'] = df[f'{col}_{p}_max'].diff().abs().rolling(**rolArgs).max().astype(np.float16) # find max diff\n                df[f'{col}_diff_{p}_mean'] = df[f'{col}_{p}_min'].diff().abs().rolling(**rolArgs).mean().astype(np.float16) # find mean diff\n                gc.collect()\n                    \n                \n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n    \n    return df\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:08.141533Z","iopub.execute_input":"2023-11-27T13:57:08.142257Z","iopub.status.idle":"2023-11-27T13:57:08.160745Z","shell.execute_reply.started":"2023-11-27T13:57:08.142224Z","shell.execute_reply":"2023-11-27T13:57:08.159739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Get Sensor Data without Feature ENgineering","metadata":{}},{"cell_type":"code","source":"# get All valid data without Freature Endgineering \ndef getAllValidData(nonNaNSeries, visual=False):\n    reduceTrainData =[]\n    \n    #get training set by valid sensor\n    for seriesID in nonNaNSeries:\n        train = getTrainSeries(seriesID)\n        reduceTrainData.append(train)\n        if visual ==True:\n             # vizualize these series\n            display(Markdown('###  anglez for series ' + seriesID))\n            fig, ax = plt.subplots(figsize=(20, 3))\n            sns.lineplot(data=train, x=\"step\", y=\"anglez\",hue=\"awake\", linewidth = 0.5)\n            plt.show();\n            #display enmo \n            display(Markdown('###  enmo for series ' + seriesID))\n            fig, ax = plt.subplots(figsize=(20, 3))\n            sns.lineplot(data=train, x=\"step\", y=\"enmo\",hue=\"awake\", linewidth = 0.5)\n            plt.show();\n        del train\n        gc.collect();\n    return reduceTrainData","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:09.708393Z","iopub.execute_input":"2023-11-27T13:57:09.708724Z","iopub.status.idle":"2023-11-27T13:57:09.716516Z","shell.execute_reply.started":"2023-11-27T13:57:09.708697Z","shell.execute_reply":"2023-11-27T13:57:09.715561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get Sensor Data With Feature Engineering","metadata":{}},{"cell_type":"code","source":"# get All valid data with Feature Engineering without Parallel \ndef getAllValidDataFeatureEng(nonNaNSeries, visual=False):\n    reduceTrainData =[] \n    #get training set by valid sensor\n    for seriesID in nonNaNSeries:\n        train = getTrainSeries(seriesID)\n#         print(train.head(3))\n        train['awake'] = train['awake'].astype(np.int8)\n        train = featureEng(train)\n        reduceTrainData.append(train)\n        if visual ==True:\n             # vizualize these series\n            display(Markdown('###  anglez for series ' + seriesID))\n            fig, ax = plt.subplots(figsize=(20, 3))\n            sns.lineplot(data=train, x=\"step\", y=\"anglez\",hue=\"awake\", linewidth = 0.5)\n            plt.show();\n            #display enmo \n            display(Markdown('###  enmo for series ' + seriesID))\n            fig, ax = plt.subplots(figsize=(20, 3))\n            sns.lineplot(data=train, x=\"step\", y=\"enmo\",hue=\"awake\", linewidth = 0.5)\n            plt.show();\n        del train\n        gc.collect();\n    return reduceTrainData","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:18.968710Z","iopub.execute_input":"2023-11-27T13:57:18.969547Z","iopub.status.idle":"2023-11-27T13:57:18.977612Z","shell.execute_reply.started":"2023-11-27T13:57:18.969511Z","shell.execute_reply":"2023-11-27T13:57:18.976572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Get Senor Data with Feature Engineering By id","metadata":{}},{"cell_type":"code","source":"# get Data with Feature Enginer by id \ndef getValidDataFeatureEngById(ids, visual=False):\n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    train = getTrainSeries(ids)\n#         print(train.head(3))\n    train['awake'] = train['awake'].astype(np.int8)\n    train = featureEng(train)\n    if visual ==True:\n        # vizualize these series\n        display(Markdown('###  anglez for series ' + ids))\n        fig, ax = plt.subplots(figsize=(20, 3))\n        sns.lineplot(data=train, x=\"step\", y=\"anglez\",hue=\"awake\", linewidth = 0.5)\n        plt.show();\n        #display enmo \n        display(Markdown('###  enmo for series ' + ids))\n        fig, ax = plt.subplots(figsize=(20, 3))\n        sns.lineplot(data=train, x=\"step\", y=\"enmo\",hue=\"awake\", linewidth = 0.5)\n        plt.show();\n    gc.collect()\n    return train","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:24.118490Z","iopub.execute_input":"2023-11-27T13:57:24.119174Z","iopub.status.idle":"2023-11-27T13:57:24.126847Z","shell.execute_reply.started":"2023-11-27T13:57:24.119140Z","shell.execute_reply":"2023-11-27T13:57:24.125942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Without Feature Engineering\n# smallerTrainData =getAllValidData(noNaNseries, visual=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:27.201001Z","iopub.execute_input":"2023-11-27T13:57:27.201390Z","iopub.status.idle":"2023-11-27T13:57:27.205431Z","shell.execute_reply.started":"2023-11-27T13:57:27.201358Z","shell.execute_reply":"2023-11-27T13:57:27.204456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(smallerTrainData)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:27.601263Z","iopub.execute_input":"2023-11-27T13:57:27.601637Z","iopub.status.idle":"2023-11-27T13:57:27.605575Z","shell.execute_reply.started":"2023-11-27T13:57:27.601588Z","shell.execute_reply":"2023-11-27T13:57:27.604650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# smallerTrainData =getAllValidDataFeatureEng(noNaNseries[0:1], visual=False)\n# len(smallerTrainData)\n# ZzzsTrain = pd.concat(smallerTrainData).reset_index(drop=True) \n# ZzzsTrain[\"series_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:27.938218Z","iopub.execute_input":"2023-11-27T13:57:27.938546Z","iopub.status.idle":"2023-11-27T13:57:27.942700Z","shell.execute_reply.started":"2023-11-27T13:57:27.938520Z","shell.execute_reply":"2023-11-27T13:57:27.941666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# smallerTrainData[0].head()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:28.492880Z","iopub.execute_input":"2023-11-27T13:57:28.493254Z","iopub.status.idle":"2023-11-27T13:57:28.497370Z","shell.execute_reply.started":"2023-11-27T13:57:28.493224Z","shell.execute_reply":"2023-11-27T13:57:28.496290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# smallData = getAllValidData(noNaNseries[0:1], visual=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:28.793557Z","iopub.execute_input":"2023-11-27T13:57:28.793949Z","iopub.status.idle":"2023-11-27T13:57:28.798131Z","shell.execute_reply.started":"2023-11-27T13:57:28.793895Z","shell.execute_reply":"2023-11-27T13:57:28.797099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# smallData[0].head()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:29.119558Z","iopub.execute_input":"2023-11-27T13:57:29.119941Z","iopub.status.idle":"2023-11-27T13:57:29.123939Z","shell.execute_reply.started":"2023-11-27T13:57:29.119887Z","shell.execute_reply":"2023-11-27T13:57:29.123000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Parallel Job to get All valid sensor data","metadata":{}},{"cell_type":"code","source":"%%time\nZzzsTrain = Parallel(n_jobs=-1)(delayed(getValidDataFeatureEngById)(i) for i in noNaNseries)\nZzzsTrain = pd.concat(ZzzsTrain, ignore_index=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T13:57:33.309995Z","iopub.execute_input":"2023-11-27T13:57:33.310359Z","iopub.status.idle":"2023-11-27T14:05:16.560426Z","shell.execute_reply.started":"2023-11-27T13:57:33.310329Z","shell.execute_reply":"2023-11-27T14:05:16.559195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain[\"series_id\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:16.562567Z","iopub.execute_input":"2023-11-27T14:05:16.562888Z","iopub.status.idle":"2023-11-27T14:05:17.544103Z","shell.execute_reply.started":"2023-11-27T14:05:16.562860Z","shell.execute_reply":"2023-11-27T14:05:17.543193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:17.545117Z","iopub.execute_input":"2023-11-27T14:05:17.545398Z","iopub.status.idle":"2023-11-27T14:05:17.574881Z","shell.execute_reply.started":"2023-11-27T14:05:17.545374Z","shell.execute_reply":"2023-11-27T14:05:17.574084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## check no incomplete events data sensor ","metadata":{}},{"cell_type":"code","source":"ZzzsTrain[ZzzsTrain[\"series_id\"] == '31011ade7c0a'].count()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:17.576916Z","iopub.execute_input":"2023-11-27T14:05:17.577184Z","iopub.status.idle":"2023-11-27T14:05:19.614795Z","shell.execute_reply.started":"2023-11-27T14:05:17.577161Z","shell.execute_reply":"2023-11-27T14:05:19.613950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain[ZzzsTrain[\"series_id\"] == 'a596ad0b82aa'].count()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:19.615876Z","iopub.execute_input":"2023-11-27T14:05:19.616195Z","iopub.status.idle":"2023-11-27T14:05:21.653858Z","shell.execute_reply.started":"2023-11-27T14:05:19.616169Z","shell.execute_reply":"2023-11-27T14:05:21.652934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:21.655321Z","iopub.execute_input":"2023-11-27T14:05:21.655596Z","iopub.status.idle":"2023-11-27T14:05:21.827662Z","shell.execute_reply.started":"2023-11-27T14:05:21.655571Z","shell.execute_reply":"2023-11-27T14:05:21.826485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkMemory()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:21.829268Z","iopub.execute_input":"2023-11-27T14:05:21.829640Z","iopub.status.idle":"2023-11-27T14:05:21.837347Z","shell.execute_reply.started":"2023-11-27T14:05:21.829609Z","shell.execute_reply":"2023-11-27T14:05:21.836354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:21.838959Z","iopub.execute_input":"2023-11-27T14:05:21.839279Z","iopub.status.idle":"2023-11-27T14:05:25.708545Z","shell.execute_reply.started":"2023-11-27T14:05:21.839248Z","shell.execute_reply":"2023-11-27T14:05:25.707528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkMemory()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:25.709768Z","iopub.execute_input":"2023-11-27T14:05:25.710067Z","iopub.status.idle":"2023-11-27T14:05:25.716754Z","shell.execute_reply.started":"2023-11-27T14:05:25.710041Z","shell.execute_reply":"2023-11-27T14:05:25.715836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:25.720130Z","iopub.execute_input":"2023-11-27T14:05:25.720391Z","iopub.status.idle":"2023-11-27T14:05:25.738320Z","shell.execute_reply.started":"2023-11-27T14:05:25.720369Z","shell.execute_reply":"2023-11-27T14:05:25.737332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain[\"timestamp\"]","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:25.739450Z","iopub.execute_input":"2023-11-27T14:05:25.739741Z","iopub.status.idle":"2023-11-27T14:05:25.749214Z","shell.execute_reply.started":"2023-11-27T14:05:25.739707Z","shell.execute_reply":"2023-11-27T14:05:25.748217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = extractUniqueSeriesID(ZzzsTrain)\nids","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:25.751690Z","iopub.execute_input":"2023-11-27T14:05:25.752215Z","iopub.status.idle":"2023-11-27T14:05:26.700253Z","shell.execute_reply.started":"2023-11-27T14:05:25.752189Z","shell.execute_reply":"2023-11-27T14:05:26.699368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:26.701213Z","iopub.execute_input":"2023-11-27T14:05:26.701478Z","iopub.status.idle":"2023-11-27T14:05:26.707378Z","shell.execute_reply.started":"2023-11-27T14:05:26.701455Z","shell.execute_reply":"2023-11-27T14:05:26.706531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reduce Memory Usage","metadata":{}},{"cell_type":"code","source":"ZzzsTrain= reduceMemoryUse(ZzzsTrain)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:05:26.708549Z","iopub.execute_input":"2023-11-27T14:05:26.708967Z","iopub.status.idle":"2023-11-27T14:06:02.488998Z","shell.execute_reply.started":"2023-11-27T14:05:26.708935Z","shell.execute_reply":"2023-11-27T14:06:02.488070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainEvent = reduceMemoryUse(trainEvent)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:06:02.490357Z","iopub.execute_input":"2023-11-27T14:06:02.490995Z","iopub.status.idle":"2023-11-27T14:06:02.519697Z","shell.execute_reply.started":"2023-11-27T14:06:02.490961Z","shell.execute_reply":"2023-11-27T14:06:02.518827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test = reduceMemoryUse(test)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:06:02.520990Z","iopub.execute_input":"2023-11-27T14:06:02.521340Z","iopub.status.idle":"2023-11-27T14:06:02.525347Z","shell.execute_reply.started":"2023-11-27T14:06:02.521309Z","shell.execute_reply":"2023-11-27T14:06:02.524517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainEvent.isna().sum() # counting number of NaN row ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:06:02.526655Z","iopub.execute_input":"2023-11-27T14:06:02.527025Z","iopub.status.idle":"2023-11-27T14:06:02.538541Z","shell.execute_reply.started":"2023-11-27T14:06:02.526993Z","shell.execute_reply":"2023-11-27T14:06:02.537696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain.isna().sum()# counting number of NaN row ","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:06:02.539716Z","iopub.execute_input":"2023-11-27T14:06:02.540011Z","iopub.status.idle":"2023-11-27T14:06:06.628090Z","shell.execute_reply.started":"2023-11-27T14:06:02.539984Z","shell.execute_reply":"2023-11-27T14:06:06.627125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:17.687270Z","iopub.execute_input":"2023-11-27T14:09:17.688034Z","iopub.status.idle":"2023-11-27T14:09:17.826853Z","shell.execute_reply.started":"2023-11-27T14:09:17.687999Z","shell.execute_reply":"2023-11-27T14:09:17.825825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:17.828820Z","iopub.execute_input":"2023-11-27T14:09:17.829209Z","iopub.status.idle":"2023-11-27T14:09:18.011721Z","shell.execute_reply.started":"2023-11-27T14:09:17.829175Z","shell.execute_reply":"2023-11-27T14:09:18.010652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain.memory_usage(deep = True)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:18.567949Z","iopub.execute_input":"2023-11-27T14:09:18.568327Z","iopub.status.idle":"2023-11-27T14:09:18.579349Z","shell.execute_reply.started":"2023-11-27T14:09:18.568297Z","shell.execute_reply":"2023-11-27T14:09:18.578343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain[\"awake\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:19.178341Z","iopub.execute_input":"2023-11-27T14:09:19.178933Z","iopub.status.idle":"2023-11-27T14:09:19.391536Z","shell.execute_reply.started":"2023-11-27T14:09:19.178890Z","shell.execute_reply":"2023-11-27T14:09:19.390547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain['awake'] = ZzzsTrain['awake'].astype(np.int8)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:19.441136Z","iopub.execute_input":"2023-11-27T14:09:19.441761Z","iopub.status.idle":"2023-11-27T14:09:19.538765Z","shell.execute_reply.started":"2023-11-27T14:09:19.441729Z","shell.execute_reply":"2023-11-27T14:09:19.537933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:19.718020Z","iopub.execute_input":"2023-11-27T14:09:19.718362Z","iopub.status.idle":"2023-11-27T14:09:19.732946Z","shell.execute_reply.started":"2023-11-27T14:09:19.718337Z","shell.execute_reply":"2023-11-27T14:09:19.731958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create Training set","metadata":{}},{"cell_type":"code","source":"# REDUCE train data by half\nZzzsTrain = ZzzsTrain.iloc[::60] # reduce 60 times\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:20.552925Z","iopub.execute_input":"2023-11-27T14:09:20.553278Z","iopub.status.idle":"2023-11-27T14:09:20.558410Z","shell.execute_reply.started":"2023-11-27T14:09:20.553252Z","shell.execute_reply":"2023-11-27T14:09:20.557391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ZzzsTrain.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:21.075256Z","iopub.execute_input":"2023-11-27T14:09:21.075640Z","iopub.status.idle":"2023-11-27T14:09:21.090025Z","shell.execute_reply.started":"2023-11-27T14:09:21.075606Z","shell.execute_reply":"2023-11-27T14:09:21.089001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dropCols = [\"series_id\", \"step\", \"timestamp\"]\n\nxFeature, yLabel = ZzzsTrain.drop(columns= dropCols + [\"awake\"]) , ZzzsTrain[\"awake\"]\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:21.806751Z","iopub.execute_input":"2023-11-27T14:09:21.807567Z","iopub.status.idle":"2023-11-27T14:09:22.161172Z","shell.execute_reply.started":"2023-11-27T14:09:21.807533Z","shell.execute_reply":"2023-11-27T14:09:22.160182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xFeature.columns","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:22.370231Z","iopub.execute_input":"2023-11-27T14:09:22.370580Z","iopub.status.idle":"2023-11-27T14:09:22.376825Z","shell.execute_reply.started":"2023-11-27T14:09:22.370550Z","shell.execute_reply":"2023-11-27T14:09:22.375878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"yLabel","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:22.808646Z","iopub.execute_input":"2023-11-27T14:09:22.809507Z","iopub.status.idle":"2023-11-27T14:09:22.816942Z","shell.execute_reply.started":"2023-11-27T14:09:22.809472Z","shell.execute_reply":"2023-11-27T14:09:22.815919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x=yLabel.value_counts().index, y=yLabel.value_counts()).set(title=\"Awake Counting\")","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:24.055949Z","iopub.execute_input":"2023-11-27T14:09:24.056301Z","iopub.status.idle":"2023-11-27T14:09:24.392660Z","shell.execute_reply.started":"2023-11-27T14:09:24.056271Z","shell.execute_reply":"2023-11-27T14:09:24.390849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if dev == False:\n    del ZzzsTrain\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:24.942047Z","iopub.execute_input":"2023-11-27T14:09:24.942442Z","iopub.status.idle":"2023-11-27T14:09:25.080089Z","shell.execute_reply.started":"2023-11-27T14:09:24.942413Z","shell.execute_reply":"2023-11-27T14:09:25.078926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkMemory()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:25.859805Z","iopub.execute_input":"2023-11-27T14:09:25.860672Z","iopub.status.idle":"2023-11-27T14:09:25.866951Z","shell.execute_reply.started":"2023-11-27T14:09:25.860638Z","shell.execute_reply":"2023-11-27T14:09:25.865952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xFeature.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:26.368327Z","iopub.execute_input":"2023-11-27T14:09:26.368714Z","iopub.status.idle":"2023-11-27T14:09:29.136580Z","shell.execute_reply.started":"2023-11-27T14:09:26.368684Z","shell.execute_reply":"2023-11-27T14:09:29.135642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create Train Model","metadata":{}},{"cell_type":"markdown","source":"### Create Stratified K-Fold Split Cross validation , for imbalance dataset","metadata":{}},{"cell_type":"code","source":"nFold = 6#5 #10 #    data lenght  is 13165525(fullset), 219426(partial) / nFold must no reminder  \nskfold = StratifiedKFold(n_splits=nFold, shuffle=True, random_state=1)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:51.369523Z","iopub.execute_input":"2023-11-27T14:09:51.369855Z","iopub.status.idle":"2023-11-27T14:09:51.374675Z","shell.execute_reply.started":"2023-11-27T14:09:51.369828Z","shell.execute_reply":"2023-11-27T14:09:51.373769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Forest","metadata":{}},{"cell_type":"code","source":"nEstimate= 850#1000#500\nrounds=10\nmaxDepth = 7#10","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:53.061989Z","iopub.execute_input":"2023-11-27T14:09:53.062433Z","iopub.status.idle":"2023-11-27T14:09:53.067298Z","shell.execute_reply.started":"2023-11-27T14:09:53.062403Z","shell.execute_reply":"2023-11-27T14:09:53.066249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfc = RandomForestClassifier(\n                            min_samples_leaf=300,\n                            random_state=1, \n                             n_estimators=nEstimate, \n                             class_weight='balanced',\n                             verbose=1,\n                             n_jobs=-1)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:53.819007Z","iopub.execute_input":"2023-11-27T14:09:53.819358Z","iopub.status.idle":"2023-11-27T14:09:53.824876Z","shell.execute_reply.started":"2023-11-27T14:09:53.819331Z","shell.execute_reply":"2023-11-27T14:09:53.823872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## XGBoost","metadata":{}},{"cell_type":"code","source":"\nxgbModel = XGBClassifier(\n#                         n_jobs= -1, #for CPU\n                        random_state=1,\n                         eta=0.001,#0.0125, #learrning\n                         tree_method='gpu_hist', #support gpu\n                         n_estimators=nEstimate,\n                         max_depth= maxDepth,\n                         objective='binary:logistic')","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:55.762099Z","iopub.execute_input":"2023-11-27T14:09:55.762521Z","iopub.status.idle":"2023-11-27T14:09:55.767969Z","shell.execute_reply.started":"2023-11-27T14:09:55.762489Z","shell.execute_reply":"2023-11-27T14:09:55.766852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LightGM","metadata":{}},{"cell_type":"code","source":"lgbmModel = LGBMClassifier(\n            boosting_type=\"gbdt\",\n            device=\"gpu\",\n            gpu_platform_id= 0,\n            gpu_device_id= 0,\n#             n_jobs= -1, # CPU\n            random_state=1,\n            learning_rate= 0.001,#0.04125, #learrning rate\n            subsample=0.9,\n            num_leaves =50,\n            reg_alpha=0.05,\n            reg_lambda=0.05,\n            n_estimators=nEstimate,\n            max_depth= maxDepth,\n            class_weight='balanced' \n            )","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:56.803520Z","iopub.execute_input":"2023-11-27T14:09:56.803848Z","iopub.status.idle":"2023-11-27T14:09:56.810248Z","shell.execute_reply.started":"2023-11-27T14:09:56.803824Z","shell.execute_reply":"2023-11-27T14:09:56.809039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CatBoost","metadata":{}},{"cell_type":"code","source":"catBoostModel = CatBoostClassifier(\n                thread_count=-1,\n#                 task_type=\"GPU\", not support gpu\n#                 devices='0:1',\n                random_state=1,\n#                 loss_function=\"CrossEntropy\",\n                loss_function=\"Logloss\",\n                learning_rate= 0.001,#0.04125,#0.04125, #learrning rate\n                n_estimators=nEstimate,\n                max_depth= maxDepth,\n               )","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:57.617339Z","iopub.execute_input":"2023-11-27T14:09:57.617810Z","iopub.status.idle":"2023-11-27T14:09:57.626135Z","shell.execute_reply.started":"2023-11-27T14:09:57.617769Z","shell.execute_reply":"2023-11-27T14:09:57.625171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Start Training Loop for Ensemble learning, Use Stratified K-fold train all model","metadata":{}},{"cell_type":"code","source":"%%time\nstart = time.time()\nscoreList = [] # record all model performance\naucRfcList = [] # rfc acu List\naucXgbList = [] # rfc acu List\naucLgbmList = [] # rfc acu List\naucCatList = [] # rfc acu List\nfold = 1\nfor trainIdx,testIdx  in tqdm(skfold.split(xFeature, yLabel),  total=skfold.get_n_splits(), desc=\"k-fold\"):\n    #split into train/test set\n    foldTime = time.time()\n#    print(\"Train Index: \", type(trainIdx), \" data: \", trainIdx)#, \" Test Index: \", testIdx)\n    X_train_fold= xFeature.iloc[trainIdx, :]\n    X_test_fold=  xFeature.iloc[testIdx, :]\n    y_train_fold = yLabel.iloc[trainIdx]\n    y_test_fold =  yLabel.iloc[testIdx]\n    \n    #train all model \n    print(\"Start Training random forest\")\n    rfc.fit(X_train_fold, y_train_fold)\n    print(\"Start Training XGBoost\")\n    xgbModel.fit(X_train_fold, y_train_fold, eval_set=[(X_test_fold, y_test_fold)],verbose=False)\n    print(\"Start Training LightGBM\")\n    lgbmModel.fit(X_train_fold, y_train_fold, eval_set=[(X_test_fold, y_test_fold)],verbose=False)\n    print(\"Start Training CatBoost\")\n    catBoostModel.fit(X_train_fold, y_train_fold, eval_set=[(X_test_fold, y_test_fold)], verbose=False)\n    \n    \n    \n    #test set \n    #store predict result for vailvation set \n    rfc_predictProb = rfc.predict_proba(X_test_fold)[:, 1]\n    rfc_predict = rfc.predict(X_test_fold)\n    xgbPredictProb = xgbModel.predict_proba(X_test_fold)[:, 1]\n    xgbPredict = xgbModel.predict(X_test_fold)\n    lgbmPredictProb = lgbmModel.predict_proba(X_test_fold)[:, 1]\n    lgbmPredict = lgbmModel.predict(X_test_fold)\n    catBoostPredictProb = catBoostModel.predict_proba(X_test_fold)[:, 1]\n    catBoostPredict = catBoostModel.predict(X_test_fold)\n    \n    #Performance for model evaluation\n    fig, axs = plt.subplots(4, 2, figsize=(12,12))\n    \n    #plot classification report\n    print(\"Accurary Score for Random Forest: \", accuracy_score(y_test_fold, rfc_predict))\n    rfc_cm = confusion_matrix(y_test_fold, rfc_predict)\n#     print('\\n\\rConfusion matrix for Random Forest:\\n\\r', rfc_cm)\n    print(\"\\n\\rClassification Report For  Random Forest :\\n\\r\", classification_report(y_test_fold, rfc_predict))\n    ConfusionMatrixDisplay(rfc_cm, display_labels=[False, True]).plot(ax=axs[0, 0])  \n    axs[0, 0].title.set_text(\"Confusion Matrix Random Forest\")\n    \n    print(\"Accurary Score for XGBoost: \", accuracy_score(y_test_fold, xgbPredict))\n    xgb_cm = confusion_matrix(y_test_fold, xgbPredict)\n#     print('\\n\\rConfusion matrix for XGBoost:\\n\\r', xgb_cm)\n    print(\"\\n\\rClassification Report For XGBoost :\\n\\r\", classification_report(y_test_fold, xgbPredict))\n    ConfusionMatrixDisplay(xgb_cm, display_labels=[False, True]).plot(ax=axs[0, 1])  \n    axs[0, 1].title.set_text(\"Confusion Matrix XGBoost\")\n    \n    print(\"Accurary Score for LightGBM: \", accuracy_score(y_test_fold, xgbPredict))\n    lgbm_cm = confusion_matrix(y_test_fold, lgbmPredict)\n#     print('\\n\\rConfusion matrix for LightGBM:\\n\\r', lgbm_cm)\n    print(\"\\n\\rClassification Report For LightGBM :\\n\\r\", classification_report(y_test_fold, lgbmPredict))\n    ConfusionMatrixDisplay(lgbm_cm, display_labels=[False, True]).plot(ax=axs[1, 0])  \n    axs[1, 0].title.set_text(\"Confusion Matrix LightGBM\")\n    \n    \n    print(\"Accurary Score for CatBoost: \", accuracy_score(y_test_fold, catBoostPredict))\n    cat_cm = confusion_matrix(y_test_fold, catBoostPredict)\n#     print('\\n\\rConfusion matrix for CatBoost:\\n\\r', cat_cm)\n    print(\"\\n\\rClassification Report For CatBoost :\\n\\r\", classification_report(y_test_fold, catBoostPredict))\n    ConfusionMatrixDisplay(cat_cm, display_labels=[False, True]).plot(ax=axs[1, 1])  \n    axs[1, 1].title.set_text(\"Confusion Matrix CatBoost\")\n    \n    \n    \n    #Calculate AUC for the fold and add it to the auc_list for evaluation     \n    rfc_auc = roc_auc_score(y_test_fold, rfc_predictProb)\n    aucRfcList.append(rfc_auc) #append roc\n    rfc_fpr, rfc_tpr, rfc_th= roc_curve(y_test_fold, rfc_predictProb)\n    print(\"\\n\\r ROC AUC Score for Random Forest: \", rfc_auc) # predict 1\n    #plot curve\n    axs[2, 0].plot(rfc_fpr, rfc_tpr, label=\"AUC \"+ str(rfc_auc))\n    axs[2, 0].title.set_text(\"ROC AUC Curve Random Forest\")\n    plt.setp(axs[2, 0], ylabel=\"True Positive Rate\")\n    plt.setp(axs[2, 0], xlabel=\"False Positive Rate\")\n    \n    xgb_auc = roc_auc_score(y_test_fold, xgbPredictProb)\n    aucXgbList.append(xgb_auc) #append roc\n    xgb_fpr, xgb_tpr, xgb_th= roc_curve(y_test_fold, xgbPredictProb)\n    print(\"\\n\\r ROC AUC Score for XGBoost: \", xgb_auc) # predict 1\n    #plot curve\n    axs[2, 1].plot(xgb_fpr, xgb_tpr, label=\"AUC \"+ str(xgb_auc))\n    axs[2, 1].title.set_text(\"ROC AUC Curve XGBoost\")\n    plt.setp(axs[1,1], ylabel=\"True Positive Rate\")\n    plt.setp(axs[1,1], xlabel=\"False Positive Rate\")\n    \n    \n    lgbm_auc = roc_auc_score(y_test_fold, lgbmPredictProb)\n    aucLgbmList.append(lgbm_auc) #append roc\n    lgbm_fpr, lgbm_tpr, lgbm_th= roc_curve(y_test_fold, lgbmPredictProb)\n    print(\"\\n\\r ROC AUC Score for LightGBM: \", lgbm_auc) # predict 1\n    #plot curve\n    axs[3, 0].plot(lgbm_fpr, lgbm_tpr, label=\"AUC \"+ str(lgbm_auc))\n    axs[3, 0].title.set_text(\"ROC AUC Curve LightGBM\")\n    plt.setp(axs[3, 0], ylabel=\"True Positive Rate\")\n    plt.setp(axs[3, 0], xlabel=\"False Positive Rate\")\n\n    cat_auc = roc_auc_score(y_test_fold, catBoostPredictProb)\n    aucCatList.append(cat_auc) #append roc\n    cat_fpr, cat_tpr, cat_th= roc_curve(y_test_fold, catBoostPredictProb)\n    print(\"\\n\\r ROC AUC Score for CatBoost: \", cat_auc) # predict 1\n    #plot curve\n    axs[3, 1].plot(cat_fpr, cat_tpr, label=\"AUC \"+ str(cat_auc))\n    axs[3, 1].title.set_text(\"ROC AUC Curve CatBoost\")\n    plt.setp(axs[3, 1], ylabel=\"True Positive Rate\")\n    plt.setp(axs[3, 1], xlabel=\"False Positive Rate\")\n    plt.show()\n    \n    # store all model performance\n    scoreList.append({\n        'Fold':fold,\n        'rfc': rfc_auc,\n        'xgb': xgb_auc,\n        'catboost': cat_auc,\n        'lgbm': lgbm_auc,\n        \n    })\n    fold += 1\n    print(\"Fold end\")\n    print(\"Fold time: \", round(time.time() - foldTime, 3), \"seconds\")\n\nrfcAUC = np.array(aucRfcList).mean() # calulate mean auc \nxgbAUC = np.array(aucXgbList).mean() # calulate mean auc \nlgbmAUC = np.array(aucLgbmList).mean() # calulate mean auc \ncatAUC = np.array(aucCatList).mean() # calulate mean auc \n\nprint(\"Total training time: \", round(time.time() - start, 3), \"seconds\")\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:09:58.482549Z","iopub.execute_input":"2023-11-27T14:09:58.482972Z","iopub.status.idle":"2023-11-27T14:55:14.598777Z","shell.execute_reply.started":"2023-11-27T14:09:58.482937Z","shell.execute_reply":"2023-11-27T14:55:14.597809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Avg AUC for Random Forest: \", rfcAUC)\nprint(\"Avg AUC for XGBoost: \", xgbAUC)\nprint(\"Avg AUC for LightGBM: \", lgbmAUC)\nprint(\"Avg AUC for CatBoost: \", catAUC)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:14.600764Z","iopub.execute_input":"2023-11-27T14:55:14.601094Z","iopub.status.idle":"2023-11-27T14:55:14.606632Z","shell.execute_reply.started":"2023-11-27T14:55:14.601050Z","shell.execute_reply":"2023-11-27T14:55:14.605623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfoldResult = pd.DataFrame(scoreList) # Create K-Fold performance Dataframe\nkfoldResult","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:14.608013Z","iopub.execute_input":"2023-11-27T14:55:14.608352Z","iopub.status.idle":"2023-11-27T14:55:14.623759Z","shell.execute_reply.started":"2023-11-27T14:55:14.608326Z","shell.execute_reply":"2023-11-27T14:55:14.622733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feats = []\n\n# feat_imp = rfc.classifiers[0].booster_.feature_importance(importance_type='gain')\n# feat_imp = pd.Series(model.classifiers[0].feature_importances_, index=X.columns).sort_values()\n# feats.append(feat_imp)\n\n# for m in model.classifiers[1:]:\n#     feat_imp = pd.Series(m.feature_importances_, index=X.columns).sort_values()\n#     feats.append(feat_imp)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:14.627150Z","iopub.execute_input":"2023-11-27T14:55:14.627495Z","iopub.status.idle":"2023-11-27T14:55:14.631631Z","shell.execute_reply.started":"2023-11-27T14:55:14.627468Z","shell.execute_reply":"2023-11-27T14:55:14.630710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Top 50 feature \nfeature_impRFC = pd.Series(rfc.feature_importances_, index=xFeature.columns).sort_values(ascending=False)\ntop50RFC = feature_impRFC[0:50]\nplt.figure(figsize=(10,6))\nsns.barplot(x=top50RFC, y=top50RFC.index)\nplt.xlabel('Feature Importance Score')\nplt.ylabel('Features')\nplt.tight_layout()\nplt.title(\"Top 50 importenace feature for Random Forest\");","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:14.632899Z","iopub.execute_input":"2023-11-27T14:55:14.633290Z","iopub.status.idle":"2023-11-27T14:55:15.852849Z","shell.execute_reply.started":"2023-11-27T14:55:14.633262Z","shell.execute_reply":"2023-11-27T14:55:15.851920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_impXGB = pd.Series(xgbModel.feature_importances_, index=xFeature.columns).sort_values(ascending=False)\ntop50XGB = feature_impXGB[0:50]\nplt.figure(figsize=(10,6))\nsns.barplot(x=top50XGB, y=top50XGB.index)\nplt.xlabel('Feature Importance Score')\nplt.ylabel('Features')\nplt.tight_layout()\nplt.title(\"Top 50 importenace feature for XGBoost\");","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:15.853984Z","iopub.execute_input":"2023-11-27T14:55:15.854280Z","iopub.status.idle":"2023-11-27T14:55:16.904108Z","shell.execute_reply.started":"2023-11-27T14:55:15.854254Z","shell.execute_reply":"2023-11-27T14:55:16.903167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_impLGBM = pd.Series(lgbmModel.feature_importances_, index=xFeature.columns).sort_values(ascending=False)\ntop50LGBM = feature_impLGBM[0:50]\nplt.figure(figsize=(10,6))\nsns.barplot(x=top50LGBM, y=top50LGBM.index)\nplt.xlabel('Feature Importance Score')\nplt.ylabel('Features')\nplt.tight_layout()\nplt.title(\"Top 50 importenace feature for LightGBM\");","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:16.905495Z","iopub.execute_input":"2023-11-27T14:55:16.906087Z","iopub.status.idle":"2023-11-27T14:55:17.986526Z","shell.execute_reply.started":"2023-11-27T14:55:16.906051Z","shell.execute_reply":"2023-11-27T14:55:17.985600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_impCat = pd.Series(catBoostModel.feature_importances_, index=xFeature.columns).sort_values(ascending=False)\ntop50Cat = feature_impCat[0:50]\nplt.figure(figsize=(10,6))\nsns.barplot(x=top50Cat, y=top50Cat.index)\nplt.xlabel('Feature Importance Score')\nplt.ylabel('Features')\nplt.tight_layout()\nplt.title(\"Top 20 importenace feature for CatBoost\");","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:17.987723Z","iopub.execute_input":"2023-11-27T14:55:17.988030Z","iopub.status.idle":"2023-11-27T14:55:18.950687Z","shell.execute_reply.started":"2023-11-27T14:55:17.988004Z","shell.execute_reply":"2023-11-27T14:55:18.949759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predict Test data","metadata":{}},{"cell_type":"code","source":"# predict Test dataset with Feature Enginer by id \ndef predictDataFeatureEngById(ids):\n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    print(\"IDS: \", ids)\n    test  = pd.read_parquet(testDir, filters=[('series_id','=',ids)])\n    test = featureEng(test)\n    xTest = test.drop(columns=dropCols) #drop \n    test = test[dropCols]\n#     print(xTest.info()) # print input predict info\n#     print(test.info()) # print output test info (initial)\n    #predict data probability\n    rfc_predictProb = rfc.predict_proba(xTest)[:, 1]\n    rfc_predict = rfc.predict(xTest)\n    xgbPredictProb = xgbModel.predict_proba(xTest)[:, 1]\n    xgbPredict = xgbModel.predict(xTest)\n    lgbmPredictProb = lgbmModel.predict_proba(xTest)[:, 1]\n    lgbmPredict = lgbmModel.predict(xTest)\n    catBoostPredictProb = catBoostModel.predict_proba(xTest)[:, 1]\n    catBoostPredict = catBoostModel.predict(xTest)\n    \n    print(\"RFC predict: \", rfc_predict)\n    \n    # Ensemble learning weight Average approach\n    finalPred = (0.3*rfc_predict) + (0.25*xgbPredict) + (0.2*lgbmPredictProb) + (0.25*catBoostPredictProb)\n    finalPredPrd =(0.3*rfc_predictProb) + (0.25*xgbPredictProb) + (0.2*lgbmPredictProb) + (0.25*catBoostPredictProb)\n    roundedfialPred  = np.where(finalPred > 0.5, 1, 0)  #[ int(round(x)) for x in finalPred] # convert Predict \n#     print(\"\\n\\rFinal PredPrd :\" , finalPredPrd)\n#     print(\"\\n\\rFinal Predict :\" , roundedfialPred)\n    \n    test['prediction'] = roundedfialPred\n#     test['prediction'] = test['prediction'].rolling(60+1, center=True).median()#test['prediction'].rolling(360+1, center=True).median()\n    test['probability'] = finalPredPrd\n    test = test[test['prediction']!=2]\n#     print(test.describe())\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n\n    \n    test['pred_diff'] = test['prediction'].diff() # find prediction different \n    \n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan}) # \n    test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n    test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n    test = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('timestamp')\n    print(test.describe())\n    gc.collect()\n    return test ,xTest","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:18.952206Z","iopub.execute_input":"2023-11-27T14:55:18.952574Z","iopub.status.idle":"2023-11-27T14:55:18.968236Z","shell.execute_reply.started":"2023-11-27T14:55:18.952539Z","shell.execute_reply":"2023-11-27T14:55:18.967328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get Test sensor ID \ntestID=  pd.read_parquet(testDir, columns=['series_id'])\ntestID = testID.series_id.unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:18.971142Z","iopub.execute_input":"2023-11-27T14:55:18.971448Z","iopub.status.idle":"2023-11-27T14:55:19.009348Z","shell.execute_reply.started":"2023-11-27T14:55:18.971414Z","shell.execute_reply":"2023-11-27T14:55:19.008368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(testID)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:19.010712Z","iopub.execute_input":"2023-11-27T14:55:19.011116Z","iopub.status.idle":"2023-11-27T14:55:19.017685Z","shell.execute_reply.started":"2023-11-27T14:55:19.011081Z","shell.execute_reply":"2023-11-27T14:55:19.016645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predict test loop","metadata":{}},{"cell_type":"code","source":"%%time\ncols_sub = ['series_id','step','event','score']\ntests = []\nxTestList = []\nfor idx in testID: \n    test, xTest =  predictDataFeatureEngById(idx) # predict test data\n    xTestList.append(xTest)\n    print(\"Test: \", test)\n    tests.append(test[cols_sub])","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:19.019272Z","iopub.execute_input":"2023-11-27T14:55:19.019726Z","iopub.status.idle":"2023-11-27T14:55:53.958367Z","shell.execute_reply.started":"2023-11-27T14:55:19.019689Z","shell.execute_reply":"2023-11-27T14:55:53.957301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(tests)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:53.959948Z","iopub.execute_input":"2023-11-27T14:55:53.960242Z","iopub.status.idle":"2023-11-27T14:55:53.965884Z","shell.execute_reply.started":"2023-11-27T14:55:53.960216Z","shell.execute_reply":"2023-11-27T14:55:53.965042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tests","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:53.967255Z","iopub.execute_input":"2023-11-27T14:55:53.967871Z","iopub.status.idle":"2023-11-27T14:55:53.977999Z","shell.execute_reply.started":"2023-11-27T14:55:53.967837Z","shell.execute_reply":"2023-11-27T14:55:53.976924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xTestList[0]","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:53.979261Z","iopub.execute_input":"2023-11-27T14:55:53.979662Z","iopub.status.idle":"2023-11-27T14:55:54.018319Z","shell.execute_reply.started":"2023-11-27T14:55:53.979627Z","shell.execute_reply":"2023-11-27T14:55:54.017218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat(tests, ignore_index=True).reset_index(names='row_id')\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:55:54.019677Z","iopub.execute_input":"2023-11-27T14:55:54.020053Z","iopub.status.idle":"2023-11-27T14:55:54.033689Z","shell.execute_reply.started":"2023-11-27T14:55:54.020026Z","shell.execute_reply":"2023-11-27T14:55:54.032684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")\n# train = pd.read_parquet(trainDir)\n# test = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n# test = pd.read_parquet(testDir)\n# trainEvent = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\n\n# subSample = pd.read_csv(subMitSampleDir)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T15:33:57.203259Z","iopub.execute_input":"2023-11-16T15:33:57.203549Z","iopub.status.idle":"2023-11-16T15:33:57.216532Z","shell.execute_reply.started":"2023-11-16T15:33:57.203518Z","shell.execute_reply":"2023-11-16T15:33:57.215785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trainEvent[\"event\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T10:48:02.317734Z","iopub.execute_input":"2023-11-16T10:48:02.317988Z","iopub.status.idle":"2023-11-16T10:48:02.326732Z","shell.execute_reply.started":"2023-11-16T10:48:02.317967Z","shell.execute_reply":"2023-11-16T10:48:02.325884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.info(memory_usage = \"deep\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T10:48:02.327923Z","iopub.execute_input":"2023-11-16T10:48:02.328193Z","iopub.status.idle":"2023-11-16T10:48:02.337178Z","shell.execute_reply.started":"2023-11-16T10:48:02.328171Z","shell.execute_reply":"2023-11-16T10:48:02.336226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T10:48:02.338346Z","iopub.execute_input":"2023-11-16T10:48:02.338608Z","iopub.status.idle":"2023-11-16T10:48:02.480944Z","shell.execute_reply.started":"2023-11-16T10:48:02.338586Z","shell.execute_reply":"2023-11-16T10:48:02.480003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T10:48:02.482297Z","iopub.execute_input":"2023-11-16T10:48:02.482623Z","iopub.status.idle":"2023-11-16T10:48:02.489283Z","shell.execute_reply.started":"2023-11-16T10:48:02.482593Z","shell.execute_reply":"2023-11-16T10:48:02.488359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# subSample.info(memory_usage =\"deep\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T10:48:02.490425Z","iopub.execute_input":"2023-11-16T10:48:02.490673Z","iopub.status.idle":"2023-11-16T10:48:02.498229Z","shell.execute_reply.started":"2023-11-16T10:48:02.490651Z","shell.execute_reply":"2023-11-16T10:48:02.497339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.columns","metadata":{"execution":{"iopub.status.busy":"2023-11-16T10:48:02.499544Z","iopub.execute_input":"2023-11-16T10:48:02.500056Z","iopub.status.idle":"2023-11-16T10:48:02.507560Z","shell.execute_reply.started":"2023-11-16T10:48:02.500024Z","shell.execute_reply":"2023-11-16T10:48:02.506669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train[\"step\"].plot()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T10:48:02.508655Z","iopub.execute_input":"2023-11-16T10:48:02.508967Z","iopub.status.idle":"2023-11-16T10:48:02.516591Z","shell.execute_reply.started":"2023-11-16T10:48:02.508938Z","shell.execute_reply":"2023-11-16T10:48:02.515675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train[\"series_id\"].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}