{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30553,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nfrom IPython.display import Markdown\nimport gc\nfrom pandas.api.indexers import FixedForwardWindowIndexer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-12-04T17:32:24.291959Z","iopub.execute_input":"2023-12-04T17:32:24.292626Z","iopub.status.idle":"2023-12-04T17:32:26.675278Z","shell.execute_reply.started":"2023-12-04T17:32:24.292569Z","shell.execute_reply":"2023-12-04T17:32:26.673859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The basic idea is to scan the signal for each hour day by day to find a fixed pattern. These periods seem like a kind of artifical data, so we won't make predictions in these periods.","metadata":{}},{"cell_type":"code","source":"def get_artifical_periods(train_series):\n\n    fake_positions_npy = np.zeros((len(train_series), ))\n    for hour in range(24):\n        hour_df = train_series[train_series.hour==hour]\n        hour_dates = hour_df.date.unique()\n        pattern = 0\n        all_pattern = {}\n\n        n_dates = len(hour_dates)\n        fake_dates = []\n        for i in range(n_dates-1):\n            array_1 = hour_df.loc[hour_df.date==hour_dates[i], 'anglez'].values\n            if len(array_1)==1440:\n                array_1 = array_1[:720]\n            array_2 = hour_df.loc[hour_df.date==hour_dates[i+1], 'anglez'].values\n            if len(array_2)==1440:\n                array_2 = array_2[:720]\n            if len(array_1) != len(array_2):\n                continue\n\n            if len(all_pattern) > 0:\n                for date, patt in all_pattern.items():\n                    nonzero = np.nonzero(patt - array_2)[0]\n                    if len(nonzero) / len(array_2) <= 0.05:\n                        fake_positions = list(hour_df[hour_df.date==hour_dates[i+1]].index)\n                        fake_positions_npy[fake_positions] = 1\n                        fake_positions = list(hour_df[hour_df.date==date].index)\n                        fake_positions_npy[fake_positions] = 1\n                        break\n\n            nonzero = np.nonzero(array_1 - array_2)[0]\n            if len(nonzero) / len(array_2) <= 0.05:\n\n                fake_positions = list(hour_df[hour_df.date==hour_dates[i+1]].index)\n                fake_positions_npy[fake_positions] = 1\n                fake_dates.append(hour_dates[i+1])\n\n                if hour_dates[i] not in fake_dates:\n                    fake_positions = list(hour_df[hour_df.date==hour_dates[i]].index)\n                    fake_positions_npy[fake_positions] = 1\n\n\n                if len(all_pattern) == 0:\n                    all_pattern[hour_dates[i]] = array_1\n                else:\n                    not_same_num = 0\n                    for date, patt in all_pattern.items():\n                        patt_nonzero = np.nonzero(patt - array_1)[0]\n                        if len(patt_nonzero) / len(array_1) > 0.05:\n                            not_same_num += 1\n                    if not_same_num == len(all_pattern):\n                        all_pattern[hour_dates[i]] = array_1\n\n        if len(all_pattern) > 0:\n            min_patt_date = min(list(all_pattern.keys()))\n            min_date = min(list(hour_df.date.unique()))\n            if min_date == min_patt_date:\n                continue\n            for d in range(min_date, min_patt_date):\n                array = hour_df.loc[hour_df.date==d, 'anglez'].values\n                if len(array)==1440:\n                    array = array[:720]\n                if len(array) != 720:\n                    continue\n\n                for date, patt in all_pattern.items():\n                    nonzero = np.nonzero(patt - array)[0]\n                    if len(nonzero) / len(array) <= 0.05:\n                        fake_positions = list(hour_df[hour_df.date==d].index)\n                        fake_positions_npy[fake_positions] = 1\n                        break\n                        \n    return fake_positions_npy","metadata":{"execution":{"iopub.status.busy":"2023-12-04T17:32:26.677672Z","iopub.execute_input":"2023-12-04T17:32:26.678334Z","iopub.status.idle":"2023-12-04T17:32:26.706287Z","shell.execute_reply.started":"2023-12-04T17:32:26.678287Z","shell.execute_reply":"2023-12-04T17:32:26.704263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_series(series):\n    train_series = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", filters=[('series_id','=',series)])\n    train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\").query('series_id == @series')\n    \n    sample_onset = train_events.loc[train_events['event'] == 'onset', 'step'].dropna()\n    sample_wakeup = train_events.loc[train_events['event'] == 'wakeup', 'step'].dropna()\n    #train_series[\"zero_hour\"] = train_series[\"timestamp\"].apply(lambda x: 1 if x[11:19]=='00:00:00' else 0)\n    #sample_zero_hour = train_series[train_series['zero_hour'] == 1].index\n    train_series[\"hour\"] = train_series[\"timestamp\"].apply(lambda x: int(x[11:13]))\n    train_series['date'] = train_series[\"timestamp\"].apply(lambda x: int(x[5:7]+x[8:10]))\n    \n    fake_positions_npy = get_artifical_periods(train_series)\n    # cleaning etc.\n    train_events = train_events.dropna()\n    train_events[\"step\"]  = train_events[\"step\"].astype(\"int\")\n    train_events[\"awake\"] = train_events[\"event\"].replace({\"onset\":1,\"wakeup\":0})\n\n    train = pd.merge(train_series, train_events[['step','awake']], on='step', how='left')\n    train[\"awake\"] = train[\"awake\"].bfill(axis ='rows')\n\n    train.loc[:, 'is_fake'] = fake_positions_npy\n    train.loc[train.is_fake==1, 'awake'] = 2\n    # Result: the last event is always a \"wakeup\"\n    train['awake'] = train['awake'].fillna(1) # awake\n    train[\"awake\"] = train[\"awake\"].astype(\"int\")\n    return train, sample_onset, sample_wakeup","metadata":{"execution":{"iopub.status.busy":"2023-12-04T17:32:26.709045Z","iopub.execute_input":"2023-12-04T17:32:26.709959Z","iopub.status.idle":"2023-12-04T17:32:26.728840Z","shell.execute_reply.started":"2023-12-04T17:32:26.709920Z","shell.execute_reply":"2023-12-04T17:32:26.727434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a list of all the series_id\ntmp = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", columns=[\"series_id\",\"anglez\"])\nseries_id_list = tmp['series_id'].unique().tolist()\ndisplay(len(series_id_list))","metadata":{"execution":{"iopub.status.busy":"2023-12-04T17:32:30.535323Z","iopub.execute_input":"2023-12-04T17:32:30.535771Z","iopub.status.idle":"2023-12-04T17:32:56.567202Z","shell.execute_reply.started":"2023-12-04T17:32:30.535734Z","shell.execute_reply":"2023-12-04T17:32:56.565698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor series_id in series_id_list:\n    train, sample_onset, sample_wakeup = get_train_series(series_id)\n    \n    display(Markdown('###  anglez for series ' + series_id))\n    fig, ax = plt.subplots(figsize=(20, 3))\n    if len(train.awake.unique()) == 3:\n        palette = [\"#ff964f\",\"#6fc276\",\"#34495e\"]\n    elif 0 not in set(train.awake.unique()):\n        palette = [\"#6fc276\",\"#34495e\"]\n    else:\n        palette = [\"#ff964f\",\"#6fc276\"]\n    sns.lineplot(data=train, x=\"step\", y=\"anglez\", hue=\"awake\", linewidth=0.5, palette=palette)\n    \n    for onset in sample_onset:\n        plt.axvline(x=onset, color='r', linestyle='--', label='onset')\n\n    for wakeup in sample_wakeup:\n        plt.axvline(x=wakeup, color='g', linestyle='--', label='wakeup')\n\n    handles, labels = plt.gca().get_legend_handles_labels()\n    new_labels, new_handles = [], []\n    for handle, label in zip(handles, labels):\n        if label not in new_labels:\n            new_handles.append(handle)\n            new_labels.append(label)\n\n    plt.legend(new_handles, new_labels)\n    plt.show();\n    \n    \n    display(Markdown('###  enmo for series ' + series_id))\n    fig, ax = plt.subplots(figsize=(20, 3))\n    sns.lineplot(data=train, x=\"step\", y=\"enmo\", hue=\"awake\", linewidth=0.5, palette=palette)\n    for onset in sample_onset:\n        plt.axvline(x=onset, color='r', linestyle='--', label='onset')\n\n    for wakeup in sample_wakeup:\n        plt.axvline(x=wakeup, color='g', linestyle='--', label='wakeup')\n\n    handles, labels = plt.gca().get_legend_handles_labels()\n    new_labels, new_handles = [], []\n    for handle, label in zip(handles, labels):\n        if label not in new_labels:\n            new_handles.append(handle)\n            new_labels.append(label)\n\n    plt.legend(new_handles, new_labels)\n    plt.show();\n    \n    del train\n    gc.collect();\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T17:33:41.888682Z","iopub.execute_input":"2023-12-04T17:33:41.889156Z","iopub.status.idle":"2023-12-04T19:02:49.229809Z","shell.execute_reply.started":"2023-12-04T17:33:41.889118Z","shell.execute_reply":"2023-12-04T19:02:49.228593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}