{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pandas as pd\nfrom datetime import datetime, timezone, timedelta\nimport psutil\nimport matplotlib.pyplot as plt\nimport matplotlib\nmatplotlib.use(\"Agg\")","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:02:52.988285Z","iopub.execute_input":"2024-05-21T23:02:52.988770Z","iopub.status.idle":"2024-05-21T23:02:53.501267Z","shell.execute_reply.started":"2024-05-21T23:02:52.988733Z","shell.execute_reply":"2024-05-21T23:02:53.498907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"working_folder = Path(\"/kaggle/working/\")\nimages_folder = working_folder/\"images\"\nimages_folder.mkdir()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:02:53.504050Z","iopub.execute_input":"2024-05-21T23:02:53.504892Z","iopub.status.idle":"2024-05-21T23:02:53.512287Z","shell.execute_reply.started":"2024-05-21T23:02:53.504847Z","shell.execute_reply":"2024-05-21T23:02:53.510863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = Path(\"/kaggle/input/child-mind-institute-detect-sleep-states\")\n!ls {data_folder}","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:02:53.514022Z","iopub.execute_input":"2024-05-21T23:02:53.514698Z","iopub.status.idle":"2024-05-21T23:02:54.641024Z","shell.execute_reply.started":"2024-05-21T23:02:53.514636Z","shell.execute_reply":"2024-05-21T23:02:54.639336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = pd.read_parquet(data_folder/\"train_series.parquet\")\ntrain_series","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:02:54.644964Z","iopub.execute_input":"2024-05-21T23:02:54.645419Z","iopub.status.idle":"2024-05-21T23:03:53.492798Z","shell.execute_reply.started":"2024-05-21T23:02:54.645331Z","shell.execute_reply":"2024-05-21T23:03:53.491133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = train_series['series_id'].unique()\nlen(series_ids)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:03:53.495497Z","iopub.execute_input":"2024-05-21T23:03:53.495919Z","iopub.status.idle":"2024-05-21T23:04:05.557642Z","shell.execute_reply.started":"2024-05-21T23:03:53.495880Z","shell.execute_reply":"2024-05-21T23:04:05.556200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_val_series_ids = round(0.2 * len(series_ids))\nnum_val_series_ids","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:04:05.559054Z","iopub.execute_input":"2024-05-21T23:04:05.559380Z","iopub.status.idle":"2024-05-21T23:04:05.567892Z","shell.execute_reply.started":"2024-05-21T23:04:05.559344Z","shell.execute_reply":"2024-05-21T23:04:05.566520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(1234)\nseries_ids_in_val = np.random.choice(series_ids, size=num_val_series_ids, replace=False)\nseries_ids_in_val","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:04:05.569760Z","iopub.execute_input":"2024-05-21T23:04:05.571002Z","iopub.status.idle":"2024-05-21T23:04:05.593047Z","shell.execute_reply.started":"2024-05-21T23:04:05.570951Z","shell.execute_reply":"2024-05-21T23:04:05.591027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(series_ids_in_val)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:04:05.595339Z","iopub.execute_input":"2024-05-21T23:04:05.596112Z","iopub.status.idle":"2024-05-21T23:04:05.606222Z","shell.execute_reply.started":"2024-05-21T23:04:05.596073Z","shell.execute_reply":"2024-05-21T23:04:05.604726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rows = train_series.loc[~ train_series['series_id'].isin(series_ids_in_val)].reset_index(drop=True)\ntrain_rows","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:04:05.608258Z","iopub.execute_input":"2024-05-21T23:04:05.608752Z","iopub.status.idle":"2024-05-21T23:04:25.649710Z","shell.execute_reply.started":"2024-05-21T23:04:05.608705Z","shell.execute_reply":"2024-05-21T23:04:25.648759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.percentile(train_rows['enmo'], 95)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:04:25.653066Z","iopub.execute_input":"2024-05-21T23:04:25.654157Z","iopub.status.idle":"2024-05-21T23:04:26.982459Z","shell.execute_reply.started":"2024-05-21T23:04:25.654122Z","shell.execute_reply":"2024-05-21T23:04:26.980984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series['large_enmo'] = train_series['enmo'] > np.percentile(train_rows['enmo'], 95)\n(train_series['large_enmo'].value_counts() / len(train_series)) * 100","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:04:26.983662Z","iopub.execute_input":"2024-05-21T23:04:26.984009Z","iopub.status.idle":"2024-05-21T23:04:29.546304Z","shell.execute_reply.started":"2024-05-21T23:04:26.983977Z","shell.execute_reply":"2024-05-21T23:04:29.545493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(data_folder/\"train_events.csv\")\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:04:29.547688Z","iopub.execute_input":"2024-05-21T23:04:29.548232Z","iopub.status.idle":"2024-05-21T23:04:29.600990Z","shell.execute_reply.started":"2024-05-21T23:04:29.548200Z","shell.execute_reply":"2024-05-21T23:04:29.599683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window_properties = []\nall_events = []\nfor i in range(len(series_ids)):\n    print(\"i:\", i)\n    print(\"series_id:\", series_ids[i])\n    series = train_series.loc[train_series['series_id'] == series_ids[i]].reset_index(drop=True)\n    series['color'] = [\"blue\" if large_enmo else \"green\" for large_enmo in series['large_enmo']]\n    series['timestamp'] = pd.to_datetime(series['timestamp'])\n    series['timestamp_utc'] = series['timestamp'].map(lambda timestamp: timestamp.astimezone(timezone.utc))\n    series['anglez_radians'] = (np.pi / 180) * series['anglez']\n    series['cos_anglez'] = np.cos(series['anglez_radians']) #get cosine of anglez \n    series['enmo'] = np.clip(series['enmo'], 0, 1) #norm enmo feature to range(0, 1)\n    min_date_utc = series['timestamp_utc'].dt.date.min()\n    max_date_utc = series['timestamp_utc'].dt.date.max()\n    # Get image with window is 24h of a day (begin at 12:00)\n    series_24_hour_windows = {}\n    upper_bound = datetime(year=min_date_utc.year, month=min_date_utc.month, day=min_date_utc.day, hour=12, minute=0, tzinfo=timezone.utc) #12:00 at min date\n    lower_bound = upper_bound + timedelta(hours=-24) # 12:00 UTC on the previous day.\n    while lower_bound < series['timestamp_utc'].max():\n        window_df = series.loc[(series['timestamp_utc'] >= lower_bound) & (series['timestamp_utc'] < upper_bound)].reset_index(drop=True)\n        if len(window_df) > 0:\n            series_24_hour_windows[upper_bound.isoformat()[:-6]] = window_df\n        upper_bound += timedelta(hours=24)\n        lower_bound += timedelta(hours=24)\n    \n    events = train_events.loc[train_events['series_id'] == series_ids[i]].reset_index(drop=True)\n    events = events.loc[events['timestamp'].notna()].reset_index(drop=True)\n    events['timestamp'] = pd.to_datetime(events['timestamp'])\n    events['timestamp_utc'] = events['timestamp'].map(lambda timestamp: timestamp.astimezone(timezone.utc))\n    # Get events with window is 24h of a day (begin at 12:00)\n    events_24_hour_windows = {}\n    upper_bound = datetime(year=min_date_utc.year, month=min_date_utc.month, day=min_date_utc.day, hour=12, minute=0, tzinfo=timezone.utc)\n    lower_bound = upper_bound + timedelta(hours=-24) # 8:30pm UTC on the previous day.\n    while lower_bound < series['timestamp_utc'].max():\n        window_df = events.loc[(events['timestamp_utc'] >= lower_bound) & (events['timestamp_utc'] < upper_bound)].reset_index(drop=True)\n        if len(window_df) > 0:\n            events_24_hour_windows[upper_bound.isoformat()[:-6]] = window_df\n        upper_bound += timedelta(hours=24)\n        lower_bound += timedelta(hours=24)\n    #save images \n    windows = list(series_24_hour_windows.keys())\n    num_steps_cumulative = 0\n    for window_idx, window in enumerate(windows):        \n        fig = plt.figure(figsize=(14.4, 4))  # (width, height) in inches\n        plt.plot(series_24_hour_windows[window]['timestamp_utc'], series_24_hour_windows[window]['cos_anglez'], color=\"red\") #try anglez cos anglez\n        plt.scatter(\n            series_24_hour_windows[window]['timestamp_utc'], \n            series_24_hour_windows[window]['enmo'], \n            color=series_24_hour_windows[window]['color'], \n            s=1\n        )\n        ax = plt.gca()\n        ax.spines['top'].set_visible(False)\n        ax.spines['right'].set_visible(False)\n        ax.spines['bottom'].set_visible(False)\n        ax.spines['left'].set_visible(False)\n        ax.set_xticks([])\n        ax.set_yticks([])\n        plt.margins(0, 0)\n        plt.subplots_adjust(left=0, right=1, top=1, bottom=0)\n        plt.savefig(images_folder/f\"{series_ids[i]}_{window}.jpg\", bbox_inches=\"tight\", pad_inches=0)\n        plt.clf()\n        plt.cla()\n        plt.close()\n\n        min_ts_window = series_24_hour_windows[window]['timestamp_utc'].min()\n        max_ts_window = series_24_hour_windows[window]['timestamp_utc'].max()\n        num_steps_window = (max_ts_window - min_ts_window).total_seconds() / 5 + 1\n        num_steps_cumulative += num_steps_window\n        window_properties.append({\n            'series_id': series_ids[i], \n            'image_name': f\"{series_ids[i]}_{window}.jpg\", \n            'idx_in_series': window_idx, \n            'num_steps_window': num_steps_window, \n            'num_steps_cumulative': num_steps_cumulative\n        })\n\n        events_window = []\n        if window in events_24_hour_windows:\n            for j in range(len(events_24_hour_windows[window])):\n                event = {}\n                event['series_id'] = series_ids[i]\n                event['image_name'] = f\"{series_ids[i]}_{window}.jpg\"\n                row = events_24_hour_windows[window].iloc[j]\n                event['label'] = row['event']\n                event['step_in_series'] = row['step']\n                event_ts = row['timestamp_utc']\n                event['step_in_window'] = (event_ts - min_ts_window).total_seconds() / 5\n                event['x0'] = round((event['step_in_window'] / num_steps_window) * 1440) - 20\n                event['y0'] = 20\n                event['x1'] = round((event['step_in_window'] / num_steps_window) * 1440) + 20\n                event['y1'] = 380\n                events_window.append(event)\n\n        all_events.extend(events_window)\n    print(\"% of RAM used:\", psutil.virtual_memory().percent)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T23:04:29.602927Z","iopub.execute_input":"2024-05-21T23:04:29.603612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window_properties_df = pd.DataFrame(window_properties)\nwindow_properties_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window_properties_df.to_csv(\"window_properties_12_00pm_utc_cutoff.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotations_df = pd.DataFrame(all_events)\nannotations_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotations_df.to_csv(\"annotations_12_00pm_utc_cutoff.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}