{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objs as go\nimport plotly\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport cufflinks as cf\ncf.set_config_file(offline=True)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom tqdm import tqdm\nimport math","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-03T11:18:50.277266Z","iopub.execute_input":"2023-10-03T11:18:50.277664Z","iopub.status.idle":"2023-10-03T11:18:53.949949Z","shell.execute_reply.started":"2023-10-03T11:18:50.277629Z","shell.execute_reply":"2023-10-03T11:18:53.948852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hello everyone!\n\nUntil today, I used [this dataset](http://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/437345), consisting of 35 series without NaN values.\n\nTo increase the amount of training data, I decided to use more series by truncating them to the first NaN value in the event column.\n\nIn addition, I dropped the series consisting of less than 7 nights so that the model could take into account the influence of the weekend.\n\nAs a result, the dataset has grown by about 3 times.\n\nI split the dataset into 3 sets (lists, actually) of series containing approximately an equal number of nights. I use these sets of series A B C to train models with 3 folds cross validation.\n\nIn the OUTPUT section, you can take files containing the desired train series and their corresponding events.\n\nAlso at the end I put a visualization of the series with events marked. The visualization code was taken from [this excellent notebook](http://www.kaggle.com/code/patrick0302/viz-of-sleeping-time-series).","metadata":{}},{"cell_type":"markdown","source":"##### READING ORIGINAL EVENTS FILE","metadata":{}},{"cell_type":"code","source":"events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\nseries_list = events.series_id.unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:53.951722Z","iopub.execute_input":"2023-10-03T11:18:53.952337Z","iopub.status.idle":"2023-10-03T11:18:54.014682Z","shell.execute_reply.started":"2023-10-03T11:18:53.952304Z","shell.execute_reply":"2023-10-03T11:18:54.013441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### TRUNCATE EVENTS TO FIRST NAN","metadata":{}},{"cell_type":"code","source":"events_shortened = []\n\nfor series in tqdm(series_list):\n    \n    tmp_df = events[events['series_id'] == series]\n        \n    for index, row in tmp_df.iterrows():\n        if math.isnan(row.step) == False:\n            events_shortened.append(row.ravel())\n        else:\n            break\n\nevents_shortened = pd.DataFrame(events_shortened)\nevents_shortened.rename(columns={0:'series_id', 1:'night', 2:'event', 3:'step', 4:'timestamp'}, inplace = True)\nevents_shortened['date'] = pd.to_datetime(events_shortened['timestamp'].str[:10])\nevents_shortened.to_csv('train_events_shortened.csv', index  = False)\nevents_shortened","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:54.016044Z","iopub.execute_input":"2023-10-03T11:18:54.016321Z","iopub.status.idle":"2023-10-03T11:18:54.928097Z","shell.execute_reply.started":"2023-10-03T11:18:54.016298Z","shell.execute_reply":"2023-10-03T11:18:54.927309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### CHECK NANS","metadata":{}},{"cell_type":"code","source":"events_shortened.isna().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:54.930067Z","iopub.execute_input":"2023-10-03T11:18:54.930628Z","iopub.status.idle":"2023-10-03T11:18:54.939186Z","shell.execute_reply.started":"2023-10-03T11:18:54.930599Z","shell.execute_reply":"2023-10-03T11:18:54.937987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### DF WITH LAST STEP IN SERIES W/O NANS","metadata":{}},{"cell_type":"code","source":"last_step_df = events_shortened[['series_id', 'step']].groupby('series_id').max().reset_index()\nlast_step_df['step'] = last_step_df['step'].astype('Int32')\nlast_step_df","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:54.940518Z","iopub.execute_input":"2023-10-03T11:18:54.940935Z","iopub.status.idle":"2023-10-03T11:18:54.967740Z","shell.execute_reply.started":"2023-10-03T11:18:54.940907Z","shell.execute_reply":"2023-10-03T11:18:54.966460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### DF WITH NUMBER OF NIGHTS IN SERIES LEFT","metadata":{}},{"cell_type":"code","source":"max_night = events_shortened[['series_id', 'night']].groupby('series_id').max().reset_index()\nmax_night.rename(columns={'night':'max_night'}, inplace = True)\nmax_night","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:54.969000Z","iopub.execute_input":"2023-10-03T11:18:54.969425Z","iopub.status.idle":"2023-10-03T11:18:54.992777Z","shell.execute_reply.started":"2023-10-03T11:18:54.969398Z","shell.execute_reply":"2023-10-03T11:18:54.991666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_night.max_night.sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:54.994201Z","iopub.execute_input":"2023-10-03T11:18:54.994908Z","iopub.status.idle":"2023-10-03T11:18:55.007486Z","shell.execute_reply.started":"2023-10-03T11:18:54.994870Z","shell.execute_reply":"2023-10-03T11:18:55.006508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### TAKING ONLY SERIES NOT SHORTER THAN A WEEK TO SEE HOW WEEKENDS AFFECT MODEL","metadata":{}},{"cell_type":"code","source":"week_and_longer = max_night[max_night['max_night'] > 6]\nweek_and_longer.sort_values('max_night', ascending = False)","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.009114Z","iopub.execute_input":"2023-10-03T11:18:55.009806Z","iopub.status.idle":"2023-10-03T11:18:55.027933Z","shell.execute_reply.started":"2023-10-03T11:18:55.009737Z","shell.execute_reply":"2023-10-03T11:18:55.026817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"week_and_longer.max_night.sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.029279Z","iopub.execute_input":"2023-10-03T11:18:55.029714Z","iopub.status.idle":"2023-10-03T11:18:55.048058Z","shell.execute_reply.started":"2023-10-03T11:18:55.029688Z","shell.execute_reply":"2023-10-03T11:18:55.046889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"week_and_longer.max_night.sum() / max_night.max_night.sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.052243Z","iopub.execute_input":"2023-10-03T11:18:55.052971Z","iopub.status.idle":"2023-10-03T11:18:55.065160Z","shell.execute_reply.started":"2023-10-03T11:18:55.052928Z","shell.execute_reply":"2023-10-03T11:18:55.064149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"week_and_longer_series = week_and_longer.series_id.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.066657Z","iopub.execute_input":"2023-10-03T11:18:55.067947Z","iopub.status.idle":"2023-10-03T11:18:55.076230Z","shell.execute_reply.started":"2023-10-03T11:18:55.067908Z","shell.execute_reply":"2023-10-03T11:18:55.075219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### FORM 3 LISTS OF SERIES WITH ALMOST EQUAL NUMBER OF NIGHTS","metadata":{}},{"cell_type":"code","source":"listA = []\nlistB = []\nlistC = []\nsumA = 0\nsumB = 0\nlim = 730\n\nfor index, row in week_and_longer.iterrows():\n#     print(row)\n    if sumA < lim:\n        sumA += row.max_night\n        listA.append(row)\n#         print('sum1 = ', sum1)\n    elif sumB < lim:\n        sumB += row.max_night\n        listB.append(row)\n#         print('sum2 = ', sum2)\n    else:\n        listC.append(row)\n\ndfA = pd.DataFrame(listA)\ndfB = pd.DataFrame(listB)\ndfC = pd.DataFrame(listC)","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.077467Z","iopub.execute_input":"2023-10-03T11:18:55.078590Z","iopub.status.idle":"2023-10-03T11:18:55.105955Z","shell.execute_reply.started":"2023-10-03T11:18:55.078545Z","shell.execute_reply":"2023-10-03T11:18:55.105119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfA.max_night.sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.107257Z","iopub.execute_input":"2023-10-03T11:18:55.107728Z","iopub.status.idle":"2023-10-03T11:18:55.114653Z","shell.execute_reply.started":"2023-10-03T11:18:55.107700Z","shell.execute_reply":"2023-10-03T11:18:55.113443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfB.max_night.sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.116309Z","iopub.execute_input":"2023-10-03T11:18:55.116704Z","iopub.status.idle":"2023-10-03T11:18:55.129441Z","shell.execute_reply.started":"2023-10-03T11:18:55.116667Z","shell.execute_reply":"2023-10-03T11:18:55.128281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfC.max_night.sum()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.130874Z","iopub.execute_input":"2023-10-03T11:18:55.132025Z","iopub.status.idle":"2023-10-03T11:18:55.144053Z","shell.execute_reply.started":"2023-10-03T11:18:55.131986Z","shell.execute_reply":"2023-10-03T11:18:55.142893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfA.series_id.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.145862Z","iopub.execute_input":"2023-10-03T11:18:55.146655Z","iopub.status.idle":"2023-10-03T11:18:55.157244Z","shell.execute_reply.started":"2023-10-03T11:18:55.146611Z","shell.execute_reply":"2023-10-03T11:18:55.156412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfB.series_id.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.158417Z","iopub.execute_input":"2023-10-03T11:18:55.159464Z","iopub.status.idle":"2023-10-03T11:18:55.173315Z","shell.execute_reply.started":"2023-10-03T11:18:55.159426Z","shell.execute_reply":"2023-10-03T11:18:55.172436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfC.series_id.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.174941Z","iopub.execute_input":"2023-10-03T11:18:55.175280Z","iopub.status.idle":"2023-10-03T11:18:55.189045Z","shell.execute_reply.started":"2023-10-03T11:18:55.175212Z","shell.execute_reply":"2023-10-03T11:18:55.187664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### FORM AND SAVE TO DISK NEW TRAIN DF","metadata":{}},{"cell_type":"code","source":"new_df = []\nres_len = 0\nfor series in tqdm(week_and_longer_series):\n    \n    last_step = last_step_df[last_step_df['series_id'] == series]['step'].iloc[0]\n    temp_df = pl.scan_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet').filter(pl.col('series_id') == series)\n    temp_df = temp_df.collect()\n#     temp_df = temp_df.with_columns(pl.col('timestamp').str.to_datetime().dt.date().alias('date'))\n    temp_df = temp_df.filter(pl.col('step') <= last_step + 1440)\n    if len(temp_df) > 0:\n        res_len += len(temp_df)\n        new_df.append(temp_df)\nnew_df = pl.concat(new_df)\n\nnew_df","metadata":{"execution":{"iopub.status.busy":"2023-10-03T11:18:55.190729Z","iopub.execute_input":"2023-10-03T11:18:55.191199Z","iopub.status.idle":"2023-10-03T11:36:16.722832Z","shell.execute_reply.started":"2023-10-03T11:18:55.191171Z","shell.execute_reply":"2023-10-03T11:36:16.721681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df.write_parquet('train_week_and_longer.parquet', compression = 'gzip')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### VISUALISATION OF NEW TRAIN SERIES AND NEW TRAIN EVENTS","metadata":{}},{"cell_type":"code","source":"def plot_series_with_events(series_id, train_series, train_events):\n    train_series = train_series.to_pandas().copy()\n    # Filter the DataFrame based on the series_id\n    sample_serie = train_series[train_series['series_id'] == series_id]\n    \n    # Convert timestamp to Pandas datetime object\n    sample_serie['date'] = sample_serie['timestamp'].str.split('T', expand=True)[0]\n    sample_serie['time'] = sample_serie['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    sample_serie['timestamp'] = pd.to_datetime(sample_serie['date'] + ' ' + sample_serie['time'])\n    \n    # Filter event data based on the series_id\n    sample_events = train_events[train_events['series_id'] == series_id]\n    sample_events['date'] = sample_events['timestamp'].str.split('T', expand=True)[0]\n    sample_events['time'] = sample_events['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    sample_events['timestamp'] = pd.to_datetime(sample_events['date']+' '+sample_events['time'])\n    sample_events['hour'] = sample_events['timestamp'].dt.hour\n    sample_onset = sample_events.loc[sample_events['event'] == 'onset', 'timestamp'].dropna()\n    sample_wakeup = sample_events.loc[sample_events['event'] == 'wakeup', 'timestamp'].dropna()\n    \n    def plot_data_and_events(data, ylabel):\n        plt.figure(figsize=(50, 5))\n        plt.plot(sample_serie['timestamp'], sample_serie[data], label=data, linewidth=1)\n        \n        for onset in sample_onset:\n            plt.axvline(x=onset, color='r', linestyle='--', label='onset')\n            \n        for wakeup in sample_wakeup:\n            plt.axvline(x=wakeup, color='g', linestyle='--', label='wakeup')\n            \n        handles, labels = plt.gca().get_legend_handles_labels()\n        new_labels, new_handles = [], []\n    \n        for handle, label in zip(handles, labels):\n            if label not in new_labels:\n                new_handles.append(handle)\n                new_labels.append(label)\n        \n        plt.legend(new_handles, new_labels)\n        plt.xlabel('Timestamp')\n        plt.ylabel(ylabel)\n        plt.title(f'{ylabel} over Time with Event Flags - '+series_id)\n        plt.show()\n    \n    # Helper function to plot data and events\n    \n    # Plot enmo and anglez\n    plot_data_and_events('enmo', 'ENMO Value')\n    plot_data_and_events('anglez', 'anglez Value')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_list = dfA.series_id.tolist() + dfB.series_id.tolist() + dfC.series_id.tolist()\n\ncount = 0\ntot = len(total_list)\n\nfor series_id in total_list:\n    print(f'series #{count+1} of {tot} ({series_id})')\n    plot_series_with_events(series_id, new_df, events_shortened)\n    count += 1","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]}]}