{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":152953034,"sourceType":"kernelVersion"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pandas as pd\nfrom datetime import datetime, timezone, timedelta\nimport matplotlib.pyplot as plt\nimport matplotlib\n# So that matplotlib can only be used in backend to write into files\nmatplotlib.use(\"Agg\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-25T18:57:23.785984Z","iopub.execute_input":"2023-11-25T18:57:23.786546Z","iopub.status.idle":"2023-11-25T18:57:24.173468Z","shell.execute_reply.started":"2023-11-25T18:57:23.786501Z","shell.execute_reply":"2023-11-25T18:57:24.172420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"working_folder = Path(\"/kaggle/working/\")\nimages_folder = working_folder/\"images\"\nimages_folder.mkdir()\ndata_folder = Path(\"/kaggle/input/child-mind-institute-detect-sleep-states\")","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:57:24.175259Z","iopub.execute_input":"2023-11-25T18:57:24.175704Z","iopub.status.idle":"2023-11-25T18:57:24.180879Z","shell.execute_reply.started":"2023-11-25T18:57:24.175674Z","shell.execute_reply":"2023-11-25T18:57:24.179877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = pd.read_csv(\"/kaggle/input/outside-data/outside_resource.csv\")\ntrain_series","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:57:24.182561Z","iopub.execute_input":"2023-11-25T18:57:24.183250Z","iopub.status.idle":"2023-11-25T18:58:16.972582Z","shell.execute_reply.started":"2023-11-25T18:57:24.183185Z","shell.execute_reply":"2023-11-25T18:58:16.971538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = train_series['series_id'].unique()\nlen(series_ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:58:16.975468Z","iopub.execute_input":"2023-11-25T18:58:16.976181Z","iopub.status.idle":"2023-11-25T18:58:25.775846Z","shell.execute_reply.started":"2023-11-25T18:58:16.976138Z","shell.execute_reply":"2023-11-25T18:58:25.774811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_val_series_ids = round(0.2 * len(series_ids))\nnum_test_series_ids = round(0.1 * len(series_ids))\n# print(num_val_series_ids+num_test_series_ids,num_val_series_ids,num_test_series_ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:58:25.777158Z","iopub.execute_input":"2023-11-25T18:58:25.778554Z","iopub.status.idle":"2023-11-25T18:58:25.782901Z","shell.execute_reply.started":"2023-11-25T18:58:25.778516Z","shell.execute_reply":"2023-11-25T18:58:25.781878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)\nnp.random.shuffle(series_ids)\n# val_and_test = np.random.choice(series_ids, size=(num_val_series_ids + num_test_series_ids), replace=False)\n# series_ids_in_val = np.random.choice(series_ids, size=(num_val_series_ids), replace=False)\n# series_ids_in_test = list(set(series_ids_in_val).difference(val_and_test))\n\nseries_ids_in_val = series_ids[:num_val_series_ids]\nseries_ids_in_test = series_ids[num_val_series_ids:num_test_series_ids+num_val_series_ids]\nseries_ids_in_train = series_ids[num_test_series_ids+num_val_series_ids : ]","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:58:25.784165Z","iopub.execute_input":"2023-11-25T18:58:25.784477Z","iopub.status.idle":"2023-11-25T18:58:25.797533Z","shell.execute_reply.started":"2023-11-25T18:58:25.784447Z","shell.execute_reply":"2023-11-25T18:58:25.796472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(series_ids_in_val),len(series_ids_in_test))","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:58:25.799274Z","iopub.execute_input":"2023-11-25T18:58:25.799699Z","iopub.status.idle":"2023-11-25T18:58:25.808655Z","shell.execute_reply.started":"2023-11-25T18:58:25.799651Z","shell.execute_reply":"2023-11-25T18:58:25.807637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series['anglez_radians'] = (np.pi / 180) * train_series['anglez']\ntrain_series['cos_anglez'] = np.cos(train_series['anglez_radians'])\ntrain_rows = train_series.loc[train_series['series_id'].isin(series_ids_in_train)].reset_index(drop=True)\ntrain_rows","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:58:25.809897Z","iopub.execute_input":"2023-11-25T18:58:25.810817Z","iopub.status.idle":"2023-11-25T18:58:48.264036Z","shell.execute_reply.started":"2023-11-25T18:58:25.810779Z","shell.execute_reply":"2023-11-25T18:58:48.262984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(data_folder/\"train_events.csv\")\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:58:48.265151Z","iopub.execute_input":"2023-11-25T18:58:48.265457Z","iopub.status.idle":"2023-11-25T18:58:48.309851Z","shell.execute_reply.started":"2023-11-25T18:58:48.265430Z","shell.execute_reply":"2023-11-25T18:58:48.308412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window_properties = []\nall_events = []\nfor i in range(len(series_ids)):\n    print(\"i:\", i)\n    print(\"series_id:\", series_ids[i])\n    # Series contain the data\n    series = train_series.loc[train_series['series_id'] == series_ids[i]].reset_index(drop=True)\n    series['timestamp'] = pd.to_datetime(series['timestamp'])\n    series['timestamp_utc'] = series['timestamp'].map(lambda timestamp: timestamp.astimezone(timezone.utc))\n#     series['anglez_radians'] = (np.pi / 180) * series['anglez']\n#     series['cos_anglez'] = np.cos(series['anglez_radians'])\n    series['enmo'] = np.clip(series['enmo'], 0, 1) # Why\n    min_date_utc = series['timestamp_utc'].dt.date.min()\n    max_date_utc = series['timestamp_utc'].dt.date.max()\n    series_24_hour_windows = {}\n    upper_bound = datetime(year=min_date_utc.year, month=min_date_utc.month, day=min_date_utc.day, hour=20, minute=30, tzinfo=timezone.utc)\n    lower_bound = upper_bound + timedelta(hours=-24) # 8:30pm UTC on the previous day.\n    # Cut every 24 hours from 8:30pm to 8:30pm of the next day \n    while lower_bound < series['timestamp_utc'].max():\n        window_df = series.loc[(series['timestamp_utc'] >= lower_bound) & (series['timestamp_utc'] < upper_bound)].reset_index(drop=True)\n        if len(window_df) > 0:\n            series_24_hour_windows[upper_bound.isoformat()[:-6]] = window_df\n        upper_bound += timedelta(hours=24)\n        lower_bound += timedelta(hours=24)\n\n    #get the corresponding index\n    #Events contain the target/label\n    events = train_events.loc[train_events['series_id'] == series_ids[i]].reset_index(drop=True)\n    events = events.loc[events['timestamp'].notna()].reset_index(drop=True)\n    events['timestamp'] = pd.to_datetime(events['timestamp'])\n    events['timestamp_utc'] = events['timestamp'].map(lambda timestamp: timestamp.astimezone(timezone.utc))\n    events_24_hour_windows = {}\n    upper_bound = datetime(year=min_date_utc.year, month=min_date_utc.month, day=min_date_utc.day, hour=20, minute=30, tzinfo=timezone.utc)\n    lower_bound = upper_bound + timedelta(hours=-24) # 8:30pm UTC on the previous day.\n    while lower_bound < series['timestamp_utc'].max():\n        window_df = events.loc[(events['timestamp_utc'] >= lower_bound) & (events['timestamp_utc'] < upper_bound)].reset_index(drop=True)\n        if len(window_df) > 0:\n            events_24_hour_windows[upper_bound.isoformat()[:-6]] = window_df\n        upper_bound += timedelta(hours=24)\n        lower_bound += timedelta(hours=24)\n    \n    # The keys are the date information before the hour/minute/second mark\n    windows = list(series_24_hour_windows.keys())\n    num_steps_cumulative = 0\n    for window_idx, window in enumerate(windows):        \n        fig = plt.figure(figsize=(14.4, 4))  # (width, height) in inches\n        # What about normalize the two values? \n        plt.plot(series_24_hour_windows[window]['timestamp_utc'], series_24_hour_windows[window]['cos_anglez'], color=\"red\")\n        plt.scatter(\n            series_24_hour_windows[window]['timestamp_utc'], \n            series_24_hour_windows[window]['enmo'], \n#             color=series_24_hour_windows[window]['color'], \n            color = 'green',\n            s=1\n        )\n        # Explore this\n        ax = plt.gca()\n        ax.spines['top'].set_visible(False)\n        ax.spines['right'].set_visible(False)\n        ax.spines['bottom'].set_visible(False)\n        ax.spines['left'].set_visible(False)\n        ax.set_xticks([])\n        ax.set_yticks([])\n        plt.margins(0, 0)\n        plt.subplots_adjust(left=0, right=1, top=1, bottom=0)\n        plt.savefig(images_folder/f\"{series_ids[i]}_{window}.jpg\", bbox_inches=\"tight\", pad_inches=0)\n        plt.clf()\n        plt.cla()\n        plt.close()\n\n        min_ts_window = series_24_hour_windows[window]['timestamp_utc'].min()\n        max_ts_window = series_24_hour_windows[window]['timestamp_utc'].max()\n        num_steps_window = (max_ts_window - min_ts_window).total_seconds() / 5 + 1\n        num_steps_cumulative += num_steps_window\n        window_properties.append({\n            'series_id': series_ids[i], \n            'image_name': f\"{series_ids[i]}_{window}.jpg\", \n            'idx_in_series': window_idx, \n            'num_steps_window': num_steps_window, \n            'num_steps_cumulative': num_steps_cumulative\n        })\n\n        events_window = []\n        if window in events_24_hour_windows:\n            for j in range(len(events_24_hour_windows[window])):\n                event = {}\n                event['series_id'] = series_ids[i]\n                event['image_name'] = f\"{series_ids[i]}_{window}.jpg\"\n                row = events_24_hour_windows[window].iloc[j]\n                event['label'] = row['event']\n                event['step_in_series'] = row['step']\n                event_ts = row['timestamp_utc']\n                event['step_in_window'] = (event_ts - min_ts_window).total_seconds() / 5\n                event['x0'] = round((event['step_in_window'] / num_steps_window) * 1440) - 20\n                event['y0'] = 20\n                event['x1'] = round((event['step_in_window'] / num_steps_window) * 1440) + 20\n                event['y1'] = 380\n                events_window.append(event)\n\n        all_events.extend(events_window)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T18:58:48.313284Z","iopub.execute_input":"2023-11-25T18:58:48.313710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window_properties_df = pd.DataFrame(window_properties)\nwindow_properties_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window_properties_df.to_csv(\"window_properties_8_30pm_utc_cutoff.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotations_df = pd.DataFrame(all_events)\nannotations_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotations_df.to_csv(\"annotations_8_30pm_utc_cutoff.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}