{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, time  \nfrom pathlib import Path\n\n# Backup previous session\nepoch_time = int(time.time())\nworking_folder = Path('/kaggle/working/')\ncoordinates_folder = working_folder/'coordinates'\nif os.path.exists(coordinates_folder):\n    old_name = str(coordinates_folder)\n    new_name = str(coordinates_folder) + f'_{epoch_time}'\n    os.rename(old_name, new_name)\nimages_folder = working_folder/'images'\nif os.path.exists(images_folder):\n    old_name = str(images_folder)\n    new_name = str(images_folder) + f'_{epoch_time}'\n    os.rename(old_name, new_name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport cv2\nfrom cv2.typing import MatLike\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Set the maximum number of rows to display\npd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 100)\n\nworking_folder = Path('/kaggle/working/')\ncoordinates_folder = working_folder/'coordinates'\nif not os.path.exists(coordinates_folder):\n    coordinates_folder.mkdir()\nimages_folder = working_folder/'images'\nif not os.path.exists(images_folder):\n    images_folder.mkdir()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# min and max obtained from EDA\nANGLEZ_MIN = -90.0\nANGLEZ_MAX = 90.0\nANGLEZ_DER2_MIN = -4.0\nANGLEZ_DER2_MAX = 4.0\nENMO_MIN = 0.0\nENMO_MAX = 1.5\n\nLOG_OFFSET = 1\nY_HEIGHT = 256\nTIME_WIDTH = 1024","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_full = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n\n#sid = '038441c925bb'\n#train_series_full = train_series_full[train_series_full['series_id'] == sid]\n\nprint('train_series_full:')\ntrain_series_full","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\nprint('\\ntrain_events:')\ntrain_events","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocessing\ndef preproc_timestamp(df):\n    df['timestamp'] = pd.to_datetime(df['timestamp'], errors='coerce')\n    sids = df['series_id'].unique().tolist()\n    offsets = {}\n    for sid in sids:\n        group_df = df[df['series_id'] == sid]\n        group_df['timestamp'] = pd.to_datetime(group_df['timestamp'], errors='coerce')\n        earliest_offset_minutes = group_df['timestamp'].min().utcoffset().total_seconds() / 60.0\n        offsets[sid] = earliest_offset_minutes\n    df = df.apply(lambda row: standardize_timezone(row, offsets[row['series_id']]), axis=1)\n    df['timestamp'] = df['timestamp'].apply(lambda x: x.replace(tzinfo=None))\n    return df\n\ndef standardize_timezone(row, offset: float):\n    offset_minutes = row['timestamp'].utcoffset().total_seconds() / 60.0\n    offset_diff = offset - offset_minutes\n    row['timestamp'] += pd.to_timedelta(offset_diff, unit='minutes')\n    return row\n\ndef preproc_train_series(train_series_full, sid):\n    train_series = train_series_full[train_series_full['series_id'] == sid]\n    \n    train_series = preproc_timestamp(train_series)\n\n    # log ENMO\n    offset = 1\n    train_series['enmo_log'] = np.log(train_series['enmo'] + offset)\n\n    ################################################################\n    ############## anglez feature engineering ######################\n    window_size = 60\n\n    # Define linear weights for the WMA with a window size of 60\n    weights = np.arange(1, window_size + 1)\n\n    # Define a custom function to calculate the weighted average\n    def weighted_average(arr):\n        return np.sum(arr * weights[:len(arr)]) / np.sum(weights[:len(arr)])\n\n    # Calculate the WMA using the rolling method and apply the custom function\n    train_series['anglez_wma'] = train_series['anglez'].rolling(window=window_size, min_periods=1).apply(weighted_average, raw=True)\n\n    # Calculate the average using the rolling method with the mean aggregation function\n    train_series['anglez_ave'] = train_series['anglez'].rolling(window=window_size, min_periods=1).mean()\n\n    # Calculate the standard deviation using the rolling method with the std aggregation function\n    #train_series['anglez_std'] = train_series['anglez'].rolling(window=window_size, min_periods=1).std()\n    #train_series['anglez_std'] = train_series['anglez_std'].fillna(0)\n\n    # Calculate the second-order derivative for each non-overlapping window\n    train_series['anglez_der2'] = train_series['anglez'].diff().rolling(window=window_size, min_periods=1).mean().diff()\n    train_series['anglez_der2'] = train_series['anglez_der2'].fillna(0)\n\n    ########### LIMIT VALUES ##############\n    train_series['anglez'] = train_series['anglez'].clip(lower=ANGLEZ_MIN, upper=ANGLEZ_MAX)\n    train_series['anglez_wma'] = train_series['anglez_wma'].clip(lower=ANGLEZ_MIN, upper=ANGLEZ_MAX)\n    train_series['anglez_der2'] = train_series['anglez_der2'].clip(lower=ANGLEZ_DER2_MIN, upper=ANGLEZ_DER2_MAX)\n    train_series['enmo_log'] = train_series['enmo_log'].clip(lower=ENMO_MIN, upper=ENMO_MAX)\n\n    return train_series","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = train_events.dropna()\ntrain_events = preproc_timestamp(train_events)\ntrain_events['night'] = train_events['night'].astype(int)\n\ntrain_events","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate data\ndef process_sample(sid, train_series, train_events):\n    # Filter the DataFrame based on the series_id\n    df_sid = train_series[train_series['series_id'] == sid]\n    #df_sid = preproc_timestamp(df_sid)\n\n    ########### PADDING PRIOR ##############\n    earliest_timestamp = df_sid['timestamp'].min()\n    start_date = earliest_timestamp.date()\n    start_timestamp = pd.Timestamp(f'{start_date} 12:00:00')\n    start_timestamp_1 = pd.Timestamp(f'{start_date} 11:55:00')\n    #start_timestamp_2 = pd.Timestamp(f'{start_date} 12:05:00')\n    if not (earliest_timestamp<start_timestamp and earliest_timestamp>start_timestamp_1):\n        # Assumption: No onset occur during 11:55am-12pm on day 0\n        if earliest_timestamp < start_timestamp_1:\n            start_date = start_date - pd.to_timedelta(1, unit='D')\n            start_timestamp = pd.Timestamp(f'{start_date} 12:00:00')\n        # Add timestamps to the top of df_sid with an interval of 5 seconds until it reaches start_timestamp\n        before_earliest = earliest_timestamp - pd.to_timedelta(5, unit='s')\n        additional_timestamps = pd.date_range(start=start_timestamp, end=before_earliest, freq='5S')\n        additional_data = pd.DataFrame({'timestamp': additional_timestamps})\n        additional_data['series_id'] = sid\n        additional_data['step'] = -1\n        additional_data['anglez'] = 0.0\n        additional_data['enmo'] = 1.0\n\n        # Concatenate additional_data with df_sid\n        df_sid = pd.concat([additional_data, df_sid], ignore_index=True)\n\n    ########### PADDING AFTER ##############\n    latest_timestamp = df_sid['timestamp'].max()\n    end_date = latest_timestamp.date()\n    end_timestamp = pd.Timestamp(f'{end_date} 12:00:00')\n    #end_timestamp_1 = pd.Timestamp(f'{end_date} 11:55:00')\n    end_timestamp_2 = pd.Timestamp(f'{end_date} 12:05:00')\n    if not (latest_timestamp>end_timestamp and latest_timestamp<end_timestamp_2):\n        if latest_timestamp > end_timestamp_2:\n            end_date = end_date + pd.to_timedelta(1, unit='D')\n            end_timestamp = pd.Timestamp(f'{end_date} 12:00:00')\n        after_latest = latest_timestamp + pd.to_timedelta(5, unit='s')\n        additional_timestamps = pd.date_range(start=after_latest, end=end_timestamp, freq='5S')\n        additional_data = pd.DataFrame({'timestamp': additional_timestamps})\n        additional_data['series_id'] = sid\n        additional_data['step'] = -1\n        additional_data['anglez'] = 0.0\n        additional_data['enmo'] = 1.0\n\n        # Concatenate additional_data with df_sid\n        df_sid = pd.concat([df_sid, additional_data], ignore_index=True)\n    \n    df_sid.sort_values('timestamp')\n    \n    # Filter event data based on the series_id\n    sample_events = train_events[train_events['series_id'] == sid]\n    sample_onset = sample_events.loc[sample_events['event'] == 'onset', ['night', 'timestamp']].dropna()\n    sample_wakeup = sample_events.loc[sample_events['event'] == 'wakeup', ['night', 'timestamp']].dropna()\n\n    return df_sid, sample_onset, sample_wakeup\n    \ndef generate_scatterplot_imgs(data, series_id):\n    for day in data:\n        canvas = np.ones((Y_HEIGHT*4, TIME_WIDTH), dtype=np.uint8) * 255\n        for coord in data[day]['anglez']:\n            y, x = map(int, coord)\n            canvas[y, x] = 0\n        for coord in data[day]['anglez_wma']:\n            y, x = map(int, coord)\n            canvas[y, x] = 0\n        for coord in data[day]['anglez_der2']:\n            y, x = map(int, coord)\n            canvas[y, x] = 0\n        for coord in data[day]['enmo_log']:\n            y, x = map(int, coord)\n            canvas[y, x] = 0\n        save_path = images_folder/f'{series_id}_{day}.png'\n        cv2.imwrite(str(save_path), canvas)\n        print(f'png generated: {save_path}')\n        \ndef generate_lineplot_imgs(data, series_id):\n    for day in data:\n        # Create an empty canvas\n        canvas = np.ones((Y_HEIGHT*4, TIME_WIDTH), dtype=np.uint8) * 255\n        # Iterate over each type of data (anglez, anglez_wma, anglez_der2, enmo_log)\n        for data_type in ['anglez', 'anglez_wma', 'anglez_der2', 'enmo_log']:\n            # Extract coordinates and convert to integers\n            coordinates = np.array(data[day][data_type], dtype=int)\n            # Draw lines on the canvas based on the coordinates\n            for i in range(len(coordinates) - 1):\n                start_point = (coordinates[i, 1], coordinates[i, 0])\n                end_point = (coordinates[i + 1, 1], coordinates[i + 1, 0])\n                cv2.line(canvas, start_point, end_point, 0, 1)  # Draw a line (pixel value 0)\n\n        # Save the resulting canvas as a PNG image\n        save_path = images_folder / f'{series_id}_{day}.png'\n        cv2.imwrite(str(save_path), canvas)\n        print(f'png generated: {save_path}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_plot_points(series_id, train_series, train_events):\n    sample_series, sample_onset, sample_wakeup = process_sample(series_id, train_series, train_events)\n\n    raw_earliest_timestamp = sample_series['timestamp'].min()\n    earliest_timestamp = pd.Timestamp(f'{raw_earliest_timestamp.date()} 12:00:00')\n    latest_timestamp = sample_series['timestamp'].max()\n    days_between = (latest_timestamp - earliest_timestamp).days\n    data = {}\n\n    for day in range(days_between):\n        onset_rows = sample_onset[sample_onset['night']==(day+1)]\n        wakeup_rows = sample_wakeup[sample_wakeup['night']==(day+1)]\n        onset_timestamp = None\n        wakeup_timestamp = None\n        data[day] = {\n            'anglez': [],\n            'anglez_wma': [],\n            'anglez_der2': [],\n            'enmo_log': [],\n            'offset': 0.0\n        }\n        start_timestamp = earliest_timestamp + pd.to_timedelta(day, unit='D')\n        end_timestamp = earliest_timestamp + pd.to_timedelta(day + 1, unit='D')\n        if not onset_rows.empty:\n            onset_timestamp = onset_rows.iloc[0]['timestamp']\n            start_timestamp = pd.Timestamp(f'{onset_timestamp.date()} 12:00:00')\n            if onset_timestamp < start_timestamp:\n                start_timestamp = start_timestamp - pd.to_timedelta(1, unit='D')\n            end_timestamp = start_timestamp + pd.to_timedelta(1, unit='D')\n        if not wakeup_rows.empty:\n            wakeup_timestamp = wakeup_rows.iloc[0]['timestamp']\n            if wakeup_timestamp >= end_timestamp:\n                data[day]['offset'] = 6.0  # +6 hours\n                start_timestamp = start_timestamp + pd.to_timedelta(6, unit='hours')\n                end_timestamp = end_timestamp + pd.to_timedelta(6, unit='hours')\n                print(f'earliest_timestamp={earliest_timestamp}')\n                print(f'day={day} start_timestamp={start_timestamp} wakeup_timestamp={wakeup_timestamp} end_timestamp={end_timestamp}')\n                assert wakeup_timestamp < end_timestamp\n\n        rows_df = sample_series[\n            (sample_series['timestamp'] >= start_timestamp) & (sample_series['timestamp'] < end_timestamp)\n        ]\n        \n        if len(rows_df) < TIME_WIDTH:\n            print(f'NOTICE: Skipping series_id {series_id} day {day} due to insufficient rows. len(rows_df)={len(rows_df)}')\n            continue\n\n        # Time Sampling\n        target_frequency = int((24 * 60 * 60 * 1000) / TIME_WIDTH)\n        target_frequency = str(target_frequency) + 'ms'\n        rows_df = rows_df.resample(target_frequency, on='timestamp').last()\n        rows_df = rows_df.iloc[:TIME_WIDTH]\n        rows_df['anglez'] = rows_df['anglez'].fillna(0.0)\n        rows_df['anglez_wma'] = rows_df['anglez_wma'].fillna(0.0)\n        rows_df['anglez_der2'] = rows_df['anglez_der2'].fillna(0.0)\n        rows_df['enmo_log'] = rows_df['enmo_log'].fillna(np.log(1.0 + LOG_OFFSET))\n\n        # Vectorized calculations\n        y_coor_anglez = np.round((rows_df['anglez'].values - ANGLEZ_MIN) / (ANGLEZ_MAX - ANGLEZ_MIN) * (Y_HEIGHT - 1))\n        y_coor_anglez = (Y_HEIGHT - 1) - y_coor_anglez\n\n        y_coor_anglez_wma = np.round((rows_df['anglez_wma'].values - ANGLEZ_MIN) / (ANGLEZ_MAX - ANGLEZ_MIN) * (Y_HEIGHT - 1))\n        y_coor_anglez_wma = (Y_HEIGHT - 1) - y_coor_anglez_wma + Y_HEIGHT\n\n        y_coor_anglez_der2 = np.round((rows_df['anglez_der2'].values - ANGLEZ_DER2_MIN) / (ANGLEZ_DER2_MAX - ANGLEZ_DER2_MIN) * (Y_HEIGHT - 1))\n        y_coor_anglez_der2 = (Y_HEIGHT - 1) - y_coor_anglez_der2 + 2 * Y_HEIGHT\n\n        y_coor_enmo_log = np.round((rows_df['enmo_log'].values - ENMO_MIN) / (ENMO_MAX - ENMO_MIN) * (Y_HEIGHT - 1))\n        y_coor_enmo_log = (Y_HEIGHT - 1) - y_coor_enmo_log + 3 * Y_HEIGHT\n\n        row_count = np.arange(len(rows_df))\n\n        # Append to data dictionary\n        data[day]['anglez'] = np.column_stack((y_coor_anglez, row_count)).tolist()\n        data[day]['anglez_wma'] = np.column_stack((y_coor_anglez_wma, row_count)).tolist()\n        data[day]['anglez_der2'] = np.column_stack((y_coor_anglez_der2, row_count)).tolist()\n        data[day]['enmo_log'] = np.column_stack((y_coor_enmo_log, row_count)).tolist()\n\n    save_path = coordinates_folder / f'{series_id}.json'\n    with open(save_path, 'w') as fp:\n        json.dump(data, fp)\n    print(f'json generated: {save_path}')\n\n    generate_lineplot_imgs(data, series_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.mode.chained_assignment = None  # default='warn'\nsids = train_events['series_id'].unique().tolist()\nprint('len(sids)',len(sids))\nfor i,sid in enumerate(sids):\n    print('i:',i)\n    train_series = preproc_train_series(train_series_full, sid)\n    get_plot_points(sid, train_series, train_events)\nprint('DONE')","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r /kaggle/working/images.zip /kaggle/working/images\n!zip -r /kaggle/working/coordinates.zip /kaggle/working/coordinates","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!rm -rf /kaggle/working/images\n#!rm -rf /kaggle/working/coordinates","metadata":{"execution":{"iopub.status.busy":"2023-12-08T19:49:00.501669Z","iopub.execute_input":"2023-12-08T19:49:00.502223Z","iopub.status.idle":"2023-12-08T19:49:02.713664Z","shell.execute_reply.started":"2023-12-08T19:49:00.502161Z","shell.execute_reply":"2023-12-08T19:49:02.712204Z"},"trusted":true},"execution_count":null,"outputs":[]}]}