{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nplt.style.use(\"ggplot\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-09T17:11:33.707698Z","iopub.execute_input":"2023-11-09T17:11:33.708200Z","iopub.status.idle":"2023-11-09T17:11:34.164392Z","shell.execute_reply.started":"2023-11-09T17:11:33.708161Z","shell.execute_reply":"2023-11-09T17:11:34.162831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    SUBMISSION_PATH = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS_PATH = MAIN_DIR + \"train_events.csv\"\n    TRAIN_SERIES_PATH = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES_PATH = MAIN_DIR + \"test_series.parquet\"","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:11:34.167615Z","iopub.execute_input":"2023-11-09T17:11:34.168427Z","iopub.status.idle":"2023-11-09T17:11:34.175114Z","shell.execute_reply.started":"2023-11-09T17:11:34.168346Z","shell.execute_reply":"2023-11-09T17:11:34.174063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" def data_cleaning(data):\n        before_cleaning = len(data)\n        print(\"Missing timestamps count : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        print(\"Steps removed : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:11:34.176629Z","iopub.execute_input":"2023-11-09T17:11:34.177730Z","iopub.status.idle":"2023-11-09T17:11:34.189149Z","shell.execute_reply.started":"2023-11-09T17:11:34.177689Z","shell.execute_reply":"2023-11-09T17:11:34.188089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_from_source(datapath,is_parquet):\n    if(is_parquet):\n        data = pd.read_parquet(datapath)\n    else:\n        data = pd.read_csv(datapath)\n    gc.collect()\n    return data ","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:11:34.190565Z","iopub.execute_input":"2023-11-09T17:11:34.191131Z","iopub.status.idle":"2023-11-09T17:11:34.203484Z","shell.execute_reply.started":"2023-11-09T17:11:34.191098Z","shell.execute_reply":"2023-11-09T17:11:34.202171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series = load_from_source(PATHS.TRAIN_SERIES_PATH,True)\nevents = load_from_source(PATHS.TRAIN_EVENTS_PATH,False)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:11:34.207386Z","iopub.execute_input":"2023-11-09T17:11:34.207772Z","iopub.status.idle":"2023-11-09T17:12:40.577213Z","shell.execute_reply.started":"2023-11-09T17:11:34.207741Z","shell.execute_reply":"2023-11-09T17:12:40.575289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.to_csv('original_event.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:12:40.579360Z","iopub.execute_input":"2023-11-09T17:12:40.579786Z","iopub.status.idle":"2023-11-09T17:12:40.716357Z","shell.execute_reply.started":"2023-11-09T17:12:40.579752Z","shell.execute_reply":"2023-11-09T17:12:40.715245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:12:40.717574Z","iopub.execute_input":"2023-11-09T17:12:40.718164Z","iopub.status.idle":"2023-11-09T17:12:40.736594Z","shell.execute_reply.started":"2023-11-09T17:12:40.718128Z","shell.execute_reply":"2023-11-09T17:12:40.734821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:12:40.738880Z","iopub.execute_input":"2023-11-09T17:12:40.739507Z","iopub.status.idle":"2023-11-09T17:12:40.757931Z","shell.execute_reply.started":"2023-11-09T17:12:40.739451Z","shell.execute_reply":"2023-11-09T17:12:40.756302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:12:40.759881Z","iopub.execute_input":"2023-11-09T17:12:40.760445Z","iopub.status.idle":"2023-11-09T17:12:52.955976Z","shell.execute_reply.started":"2023-11-09T17:12:40.760359Z","shell.execute_reply":"2023-11-09T17:12:52.954705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.describe()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:12:52.957503Z","iopub.execute_input":"2023-11-09T17:12:52.958503Z","iopub.status.idle":"2023-11-09T17:12:52.981459Z","shell.execute_reply.started":"2023-11-09T17:12:52.958462Z","shell.execute_reply":"2023-11-09T17:12:52.980104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_counts_series = series.isna().sum()\nprint(null_counts_series)\n\ntotal_null_count_series = null_counts_series.sum()\nprint(\"Total missing values in the Series DataFrame: \", total_null_count_series)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:12:52.984484Z","iopub.execute_input":"2023-11-09T17:12:52.985682Z","iopub.status.idle":"2023-11-09T17:13:20.182714Z","shell.execute_reply.started":"2023-11-09T17:12:52.985627Z","shell.execute_reply":"2023-11-09T17:13:20.181335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_counts_events = events.isna().sum()\nprint(null_counts_events)\n\ntotal_null_count_events = null_counts_events.sum()\nprint(\"Total missing values in the Events DataFrame: \", total_null_count_events)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:13:20.184414Z","iopub.execute_input":"2023-11-09T17:13:20.184788Z","iopub.status.idle":"2023-11-09T17:13:20.199892Z","shell.execute_reply.started":"2023-11-09T17:13:20.184754Z","shell.execute_reply":"2023-11-09T17:13:20.198455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Further checking rows which have missing values to see both steps and timestamps feature bare missing values in the same row\nrows_with_missing_values = events[events.isna().any(axis=1)]\nrows_count_with_missing_values = len(rows_with_missing_values)\n\nprint(\"Number of rows with missing values in the Series DataFrame: \", rows_count_with_missing_values)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:13:20.201678Z","iopub.execute_input":"2023-11-09T17:13:20.202066Z","iopub.status.idle":"2023-11-09T17:13:20.216968Z","shell.execute_reply.started":"2023-11-09T17:13:20.202002Z","shell.execute_reply":"2023-11-09T17:13:20.215408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seriesId = events[\"series_id\"].unique()\nprint(f\"Number of series_id in events : {len(seriesId)}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:13:20.222510Z","iopub.execute_input":"2023-11-09T17:13:20.222972Z","iopub.status.idle":"2023-11-09T17:13:20.233598Z","shell.execute_reply.started":"2023-11-09T17:13:20.222937Z","shell.execute_reply":"2023-11-09T17:13:20.232130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seriesId = series[\"series_id\"].unique()\nprint(f\"Number of series_id in series : {len(seriesId)}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:13:20.235256Z","iopub.execute_input":"2023-11-09T17:13:20.235648Z","iopub.status.idle":"2023-11-09T17:13:32.137730Z","shell.execute_reply.started":"2023-11-09T17:13:20.235608Z","shell.execute_reply":"2023-11-09T17:13:32.136423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.to_csv('event_before_clean.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:13:32.139319Z","iopub.execute_input":"2023-11-09T17:13:32.139810Z","iopub.status.idle":"2023-11-09T17:13:32.232906Z","shell.execute_reply.started":"2023-11-09T17:13:32.139766Z","shell.execute_reply":"2023-11-09T17:13:32.231451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_cleaned  = data_cleaning(series)\nevents_cleaned =  data_cleaning(events)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:13:32.235074Z","iopub.execute_input":"2023-11-09T17:13:32.235466Z","iopub.status.idle":"2023-11-09T17:14:07.694220Z","shell.execute_reply.started":"2023-11-09T17:13:32.235424Z","shell.execute_reply":"2023-11-09T17:14:07.692732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_cleaned.to_csv('event_after_clean.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:07.695743Z","iopub.execute_input":"2023-11-09T17:14:07.696140Z","iopub.status.idle":"2023-11-09T17:14:07.766991Z","shell.execute_reply.started":"2023-11-09T17:14:07.696105Z","shell.execute_reply":"2023-11-09T17:14:07.765761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_cleaned.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:07.768806Z","iopub.execute_input":"2023-11-09T17:14:07.769657Z","iopub.status.idle":"2023-11-09T17:14:07.785980Z","shell.execute_reply.started":"2023-11-09T17:14:07.769608Z","shell.execute_reply":"2023-11-09T17:14:07.784479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_cleaned.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:07.787960Z","iopub.execute_input":"2023-11-09T17:14:07.788405Z","iopub.status.idle":"2023-11-09T17:14:07.806983Z","shell.execute_reply.started":"2023-11-09T17:14:07.788347Z","shell.execute_reply":"2023-11-09T17:14:07.805883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = events_cleaned\nseries = series_cleaned","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:07.808863Z","iopub.execute_input":"2023-11-09T17:14:07.809271Z","iopub.status.idle":"2023-11-09T17:14:07.817587Z","shell.execute_reply.started":"2023-11-09T17:14:07.809239Z","shell.execute_reply":"2023-11-09T17:14:07.816668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_counts_events = events.isna().sum()\nprint(null_counts_events)\n\ntotla_null_counts_events = null_counts_events.sum()\nprint(\"Missing values in the Series DataFrame: \", totla_null_counts_events)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:07.819183Z","iopub.execute_input":"2023-11-09T17:14:07.819578Z","iopub.status.idle":"2023-11-09T17:14:07.836494Z","shell.execute_reply.started":"2023-11-09T17:14:07.819542Z","shell.execute_reply":"2023-11-09T17:14:07.835162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events['timestamp'] = pd.to_datetime(events['timestamp'], format='%Y-%m-%dT%H:%M:%S%z', errors='coerce')\nevents['timezone_offset'] = events['timestamp'].apply(lambda x: x.strftime('%z') if not pd.isnull(x) else pd.NaT)\n\n#print(events)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:07.838098Z","iopub.execute_input":"2023-11-09T17:14:07.838522Z","iopub.status.idle":"2023-11-09T17:14:08.008535Z","shell.execute_reply.started":"2023-11-09T17:14:07.838487Z","shell.execute_reply":"2023-11-09T17:14:08.007216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#series['timestamp'] = pd.to_datetime(series['timestamp'], format='%Y-%m-%dT%H:%M:%S%z', errors='coerce')\n#series['timezone_offset'] = series['timestamp'].apply(lambda x: x.strftime('%z') if not pd.isnull(x) else pd.NaT)\n\n#print(series)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:08.010175Z","iopub.execute_input":"2023-11-09T17:14:08.010567Z","iopub.status.idle":"2023-11-09T17:14:08.017245Z","shell.execute_reply.started":"2023-11-09T17:14:08.010534Z","shell.execute_reply":"2023-11-09T17:14:08.015356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events.to_csv('event_after_timezone.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:08.018562Z","iopub.execute_input":"2023-11-09T17:14:08.019109Z","iopub.status.idle":"2023-11-09T17:14:08.153010Z","shell.execute_reply.started":"2023-11-09T17:14:08.019072Z","shell.execute_reply":"2023-11-09T17:14:08.152014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"null_counts_events = events.isna().sum()\nprint(null_counts_events)\n\ntotla_null_counts_events = null_counts_events.sum()\nprint(\"Missing values in the events DataFrame: \", totla_null_counts_events)","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:08.154394Z","iopub.execute_input":"2023-11-09T17:14:08.155586Z","iopub.status.idle":"2023-11-09T17:14:08.170004Z","shell.execute_reply.started":"2023-11-09T17:14:08.155546Z","shell.execute_reply":"2023-11-09T17:14:08.167393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ts_vs_night_lineplot(df, xlabel, ylabel, title=None):\n    _onset = df.loc[df['event'] == 'onset', 'timestamp'].to_numpy()\n    _wakeup = df.loc[df['event'] == 'wakeup', 'timestamp'].to_numpy()\n\n    plt.figure(figsize=(20, 5))\n    plt.plot(df[xlabel], df[ylabel], marker='x', linestyle='-', color='g')\n    \n    for onset in _onset:\n        plt.axvline(x=onset, color='r', linestyle='--', label='onset')\n    for wakeup in _wakeup:\n        plt.axvline(x=wakeup, color='b', linestyle='--', label='wakeup')\n\n    plt.title(title)\n    plt.xlabel(xlabel)\n    plt.ylabel(ylabel)\n    plt.grid(True)\n    plt.show()\n\nfor ids in range(5):\n    temp = events[events['series_id'] == seriesId[ids]]\n    ts_vs_night_lineplot(temp, \"timestamp\", \"night\", \"timestamp vs night for each series id\")\n","metadata":{"execution":{"iopub.status.busy":"2023-11-09T17:14:08.172046Z","iopub.execute_input":"2023-11-09T17:14:08.173147Z","iopub.status.idle":"2023-11-09T17:14:10.636052Z","shell.execute_reply.started":"2023-11-09T17:14:08.173084Z","shell.execute_reply":"2023-11-09T17:14:10.634462Z"},"trusted":true},"execution_count":null,"outputs":[]}]}