{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-10-08T09:34:29.040443Z","iopub.execute_input":"2023-10-08T09:34:29.040911Z","iopub.status.idle":"2023-10-08T09:34:30.133463Z","shell.execute_reply.started":"2023-10-08T09:34:29.040874Z","shell.execute_reply":"2023-10-08T09:34:30.132338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_1 = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\ndf_train_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\ndf_test_1 = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-10-08T09:34:30.135184Z","iopub.execute_input":"2023-10-08T09:34:30.135708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = 'all' #shows output for all the commands in a single cell","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Processing_functions","metadata":{}},{"cell_type":"code","source":"#training functions\ndef single_user_process(dfu,dfuevents):\n    #print('-Interpolating steps')\n    dfuevents['step'].interpolate(method='linear', inplace=True)\n    \n    #print('-Merging events with watch data')\n    dfu11 = pd.merge(dfu,dfuevents, how = 'left')\n    \n    #print('-Adding Sleep and Awake State')\n    dfu12 = dfu11.copy()\n    element1 = 'onset'\n    element2 = 'wakeup'\n    replacement_element = 'asleep'\n\n    # Create a new list to store the updated values\n    new_values = []\n\n    # Iterate through the DataFrame to replace values between element1 and element2\n    between_elements = False  # A flag to indicate when to start replacing\n\n    for value in dfu12['event']:\n        if value == element1:\n            between_elements = True\n            new_values.append(element1)\n        elif value == element2:\n            between_elements = False\n            new_values.append(element2)\n        elif between_elements:\n            new_values.append(replacement_element)\n        else:\n            new_values.append('awake')\n\n    # Update the 'Category' column with the new values\n    dfu12['event'] = new_values\n    dfu12['event'] = dfu12['event'].fillna('awake')\n    \n#     print('-Adding Night Count')\n#     dfu13 = dfu12.copy()\n#     dfu13['night'] = dfu13['night'].fillna(method = 'bfill')\n#     dfu13['night'] = dfu13['night'].fillna(method = 'ffill')\n    \n    #print('-Adding Date Time Columns')\n    dfu14 = dfu12.copy()\n    #print('    -breaking timestamp')\n    dfu14['timestamp'] = dfu14['timestamp'].apply(lambda x: x.split('T')) # split n T\n    #print('    -adding date')\n    dfu14['date'] = dfu14['timestamp'].str.get(0)\n    #print('    -adding time')\n    dfu14['time_of_day'] = dfu14['timestamp'].str.get(1)\n    #print('    -adding year')\n    dfu14['year'] = dfu14['date'].apply(lambda x: x[:4])\n    #print('    -adding hour')\n    dfu14['hour'] = dfu14['time_of_day'].apply(lambda x: x[:2])\n    #print('    -adding minute')\n    dfu14['minute'] = dfu14['time_of_day'].apply(lambda x: x[3:5])\n    #print('    -adding GMT')\n    dfu14['GMT'] = dfu14['time_of_day'].apply(lambda x: x[8:13])\n    #dfu14['second'] = dfu14['time_of_day'].apply(lambda x: x[6:8])\n    dfu14 = dfu14.drop(['timestamp','time_of_day'], axis = 'columns')\n    \n#     #a function that inputs df and counts number of days\n#     def add_day_column(df):\n#         # Convert the \"date\" column to datetime objects\n#         df['date'] = pd.to_datetime(df['date'])\n\n#         # Create a new column \"day\" based on the rank of unique dates\n#         df['night'] = df['date'].rank(method='dense').astype(int)\n#         return df\n\n    dfu15 = dfu14.copy()\n#    dfu15 = add_day_column(dfu15)\n    \n    def numerical_targets(x):\n        if x == 'awake':\n            return 0\n        if x == 'onset':\n            return 1\n        if x == 'asleep':\n            return 1\n        if x == 'wakeup':\n            return 0\n    \n    #print('-Making Targets Numeric')\n    df = dfu15.copy()\n    df['event'] = df['event'].apply(numerical_targets)\n    \n    df = df.drop(['series_id','date','night'], axis = 'columns')\n    \n    return df\n\ndef process_by_user(df_train,df_events,series_id):\n    dfu = df_train[df_train['series_id'] == series_id]\n    dfuevents = df_events[df_events['series_id'] == series_id]\n    \n    prepared_data = single_user_process(dfu,dfuevents)\n    \n    #print('User: ', series_id, ' data process complete')\n    return prepared_data\n\n#test function:\ndef single_user_process_t(dfu):\n    #dfuevents = dfuevents.dropna()\n    \n    #dfu11 = pd.merge(dfu,dfuevents, how = 'left')\n    \n    #dfu12 = dfu11.copy()\n    #dfu12['event'] = dfu12['event'].fillna('awake')\n    \n    #dfu13 = dfu12.copy()\n    #dfu13['night'] = dfu13['night'].fillna(method = 'bfill')\n    #dfu13['night'] = dfu13['night'].fillna(method = 'ffill')\n    \n    #print('-Adding Date Time Columns')\n    dfu14 = dfu.copy()\n    #print('    -breaking timestamp')\n    dfu14['timestamp'] = dfu14['timestamp'].apply(lambda x: x.split('T')) # split n T\n    #print('    -adding date')\n    dfu14['date'] = dfu14['timestamp'].str.get(0)\n    #print('    -adding time')\n    dfu14['time_of_day'] = dfu14['timestamp'].str.get(1)\n    #print('    -adding year')\n    dfu14['year'] = dfu14['date'].apply(lambda x: x[:4])\n    #print('    -adding hour')\n    dfu14['hour'] = dfu14['time_of_day'].apply(lambda x: x[:2])\n    #print('    -adding minute')\n    dfu14['minute'] = dfu14['time_of_day'].apply(lambda x: x[3:5])\n    #print('    -adding GMT')\n    dfu14['GMT'] = dfu14['time_of_day'].apply(lambda x: x[8:13])\n    #dfu14['second'] = dfu14['time_of_day'].apply(lambda x: x[6:8])\n    dfu14 = dfu14.drop(['timestamp','time_of_day'], axis = 'columns')\n    \n    #a function that inputs df and counts number of days\n#     def add_day_column(df):\n#         # Convert the \"date\" column to datetime objects\n#         df['date'] = pd.to_datetime(df['date'])\n\n#         # Create a new column \"day\" based on the rank of unique dates\n#         df['night'] = df['date'].rank(method='dense').astype(int)\n#         return df\n\n    dfu15 = dfu14.copy()\n#     dfu15 = add_day_column(dfu15)\n    \n    #def numerical_targets(x):\n        #if x == 'awake':\n            #return 0\n        #if x == 'onset':\n            #return 1\n        #if x == 'wakeup':\n            #return 2\n    \n    df = dfu15.copy()\n    #df['event'] = df['event'].apply(numerical_targets)\n    \n    df = df.drop(['date'], axis = 'columns')\n    \n    return df\n\ndef process_by_user_t(df, series_id):\n    dfu = df[df['series_id'] == series_id]\n    #dfuevents = df_events[df_events['series_id'] == series_id]\n    \n    prepared_data = single_user_process_t(dfu)\n    \n    \n    \n    return prepared_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Filtering Functions ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\ndef categorize_events(df):\n    # Initialize a new column to store the categories\n    df['category'] = ''\n\n    # Initialize variables to track the current event and its index\n    current_event = None\n    current_event_index = None\n\n    # Variable to track whether the first 'onset' event has been encountered\n    onset_encountered = False\n\n    # Iterate through the DataFrame\n    for index, row in df.iterrows():\n        if not onset_encountered:\n            if row['event'] == 1:\n                # First 'onset' event encountered, start categorization\n                onset_encountered = True\n                current_event = 1\n                current_event_index = index\n                df.at[index, 'category'] = 'onset'\n        else:\n            if row['event'] != current_event:\n                # Change in event, update current_event and index\n                current_event = row['event']\n                current_event_index = index\n                df.at[index, 'category'] = 'wake up' if current_event == 0 else 'onset'\n            elif current_event_index is not None:\n                # Still the same event, categorize as 'ignore'\n                df.at[index, 'category'] = 'ignore'\n\n    # Filter the DataFrame to keep only 'wake up' and 'onset' categories\n    filtered_df = df[df['category'].isin(['wake up', 'onset'])]\n\n    # Drop the 'category' column if it's not needed\n    filtered_df = filtered_df.drop(columns=['category'])\n\n    return filtered_df\n\n\ndef extract_clusters(df):\n    # Initialize variables to track clusters and cluster start indices\n    clusters = []\n    current_cluster = []\n    cluster_start_index = None\n\n    # Iterate through the DataFrame\n    for index, row in df.iterrows():\n        if cluster_start_index is None:\n            # Start a new cluster\n            cluster_start_index = index\n            current_cluster.append(index)\n        elif row['step'] - df.at[current_cluster[-1], 'step'] < 320:\n            # Add to the current cluster\n            current_cluster.append(index)\n        else:\n            # End the current cluster and start a new one\n            clusters.append(current_cluster)\n            cluster_start_index = index\n            current_cluster = [index]\n\n    # Ensure the last cluster is added\n    if current_cluster:\n        clusters.append(current_cluster)\n\n    # Extract first 1 event row and last 0 event row from each cluster\n    output_rows = []\n    for cluster in clusters:\n        for index in cluster:\n            if df.at[index, 'event'] == 1:\n                output_rows.append(index)\n                break\n        for index in reversed(cluster):\n            if df.at[index, 'event'] == 0:\n                output_rows.append(index)\n                break\n\n    # Create a new DataFrame with the selected rows\n    output_df = df.loc[output_rows]\n\n    return output_df\n\n# Example usage:\n# Call the function with your DataFrame 'df' as input\n# result_df = extract_clusters(df)\n\ndef event(x):\n    if x == 1:\n        return 'onset'\n    else:\n        return 'wakeup'\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"users = df_train_1.groupby('series_id')['series_id'].count().index.values\nusers","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Your single array\nsingle_array = users\n\n# Calculate the length of the single array\narray_length = len(single_array)\n\n# Calculate the size of each equal subarray\nsubarray_size = array_length // 5\n\n# Split the single array into three equal arrays\ntrain_1 = single_array[:subarray_size]\ntrain_2 = single_array[subarray_size:2*subarray_size]\ntrain_3 = single_array[2*subarray_size:]\n\n# Print the three equal arrays\nprint(\"Array 1:\", train_1)\nprint(\"Array 2:\", train_2)\nprint(\"Array 3:\", train_3)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_processing(df_train_1,df_train_events,users):\n    processed_data = pd.DataFrame()\n    for series_id in tqdm(users):\n        processed_data = pd.concat([processed_data, process_by_user(df_train_1,df_train_events, series_id)], axis=0, ignore_index=True)\n        \n    return processed_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_processing_t(df_test_1):\n    processed_data = pd.DataFrame()\n    for series_id in tqdm(df_test_1.groupby('series_id')['series_id'].count().index.values):\n        processed_data = pd.concat([processed_data, process_by_user_t(df_test_1, series_id)], axis=0, ignore_index=True)\n        \n    return processed_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_1.iloc[::120]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_data_1 = data_processing(df_train_1,df_train_events,train_1)\nprocessed_data_1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# processed_train = data_processing(df_train_1.iloc[::24],df_train_events, users)\n# processed_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# processed_test = data_processing_t(df_test_1.iloc[::12])\n# processed_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_data_1.iloc[::36]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#processed_data_1.to_csv('/kaggle/working/processed_data_1.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a href='/kaggle/working/processed_data_1.csv'> Download File </a>","metadata":{}},{"cell_type":"code","source":"# processed_data_2 = data_processing(df_train_1,df_train_events,train_2)\n# processed_data_2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# processed_data_3 = data_processing(df_train_1,df_train_events,train_3)\n# processed_data_3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# processed_data = pd.concat([processed_data_1, processed_data_2, processed_data_3], axis=0, ignore_index=True)\n# processed_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_user(df_train_1, df_test_1, df_train_events):\n    print('Processing Train Data')\n    df = df_train_1\n    print('Train Data Processing Done')\n    \n    print('Processing Test Data')\n    test1 = single_user_process_t(df_test_1)\n    dft = test1.drop(['series_id'],axis = 'columns')\n    print('Test Data Processing Done')\n    \n    #print(test1)\n    \n    X = df.drop(['event'],axis = 'columns')\n    y = df['event']\n    \n    print('Training Model')\n    cbc = RandomForestClassifier(verbose=True)\n    cbc.fit(X.drop(['step'], axis = 1),y)\n    print('Model Training Complete')\n    \n    y_pred = cbc.predict(dft.drop(['step'], axis = 1))\n    y_pred_prob = cbc.predict_proba(dft.drop(['step'], axis = 1)).round(2)\n    associated_prob = y_pred_prob[np.arange(len(dft.index.values)), y_pred]\n    dff = pd.DataFrame({'event':y_pred,'prob':associated_prob})\n    dff = dff.set_index(test1.index.values)\n    dff[test1.columns] = test1\n    \n    #print(dff)\n    \n    #finding onset and wakeup\n    wakeon = categorize_events(dff)\n    wakeon2 = extract_clusters(wakeon)\n    \n    print('Categorization Done')\n    \n    if(len(wakeon2)==0):\n        p = pd.DataFrame()\n        p[0] = dff.iloc[0,:]\n        p[1] = dff.iloc[-1,:] \n        p = p.T\n\n        p.iloc[0,:]['event'] = 1\n        p.iloc[0,:]['prob'] = 1 - p.iloc[1,:]['prob']\n\n        wakeon2 = p\n    \n    #plotting sleep cycle\n    plt.figure(figsize=(16, 6))\n    plt.plot(dff['step'], dff['event'], color='C9', label='predicted_sleep')\n    plt.plot(wakeon2['step'], wakeon2['event'], color='g', label='onset_wakeup_events')\n    #plt.plot(test1['step'], test1['event'], color='r', label='origianal_sleep_cycle')\n\n    # Naming the x-axis, y-axis and the whole graph\n    plt.xlabel(\"step\")\n    plt.ylabel(\"event\")\n    plt.title(\"Predicted Sleep Event and wakeup onset Selection\")\n\n    # Adding legend, which helps us recognize the curve according to it's color\n    plt.legend()\n\n    # To load the display window\n    plt.show()\n    \n    final = pd.DataFrame()\n    final['step'] = wakeon2['step']\n    final['event'] = wakeon2['event']\n    final['score'] = wakeon2['prob']\n    final['series_id'] = wakeon2['series_id']\n\n    final['event'] = final['event'].apply(event)\n    \n    return final","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dfu1events = df_train_events[df_train_events['series_id'] == '038441c925bb']\n# dfu1 = df_train_1[df_train_1['series_id'] == '038441c925bb']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train, X_test = train_test_split(dfu1,test_size=0.2,shuffle = False)\n#X_train, X_test = train_test_split(processed_data_1,test_size=0.2,shuffle = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"user_prediction = train_user(processed_data_1.iloc[::36], df_test_1, df_train_events)\nuser_prediction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dfu11 = pd.merge(X_test,dfu1events, how = 'left')\n\n# print('-Adding Sleep and Awake State')\n# dfu12 = dfu11.copy()\n# element1 = 'onset'\n# element2 = 'wakeup'\n# replacement_element = 'asleep'\n\n# # Create a new list to store the updated values\n# new_values = []\n\n# # Iterate through the DataFrame to replace values between element1 and element2\n# between_elements = False  # A flag to indicate when to start replacing\n\n# for value in dfu12['event']:\n#     if value == element1:\n#         between_elements = True\n#         new_values.append(element1)\n#     elif value == element2:\n#         between_elements = False\n#         new_values.append(element2)\n#     elif between_elements:\n#         new_values.append(replacement_element)\n#     else:\n#         new_values.append('awake')\n\n# # Update the 'Category' column with the new values\n# dfu12['event'] = new_values\n# dfu12['event'] = dfu12['event'].fillna('awake')\n\n# plt.figure(figsize=(16, 6))\n# plt.plot(dfu12['step'], dfu12['event'], color='r', label='origianal_sleep_cycle')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in df_test_1.groupby('series_id')['series_id'].count().index.values:\n#     print(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def predict(df_train_1, df_test_1, df_train_events):\n#     submission = pd.DataFrame()\n#     for series_id in df_test_1.groupby('series_id')['series_id'].count().index.values:\n#         submission = pd.concat([submission, train_user(df_train_1, df_test_1, df_train_events, series_id)], axis=0, ignore_index=True)\n        \n#     return submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# final = predict(df_train_1, df_test_1, df_train_events)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final = user_prediction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = 0\nrow_id = [p]\nwhile p<len(final['step'])-1:\n    p = p + 1\n    row_id.append(p)\n\nfinal['row_id'] = row_id\n#final.reset_index()\n\nlast = pd.DataFrame()\n\nordered_col = ['row_id','series_id', 'step', 'event', 'score']\nfor col in ordered_col:\n    last[col] = final[col]\n\nlast","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"last.to_csv('/kaggle/working/submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}