{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":152619799,"sourceType":"kernelVersion"},{"sourceId":153811367,"sourceType":"kernelVersion"},{"sourceId":157569355,"sourceType":"kernelVersion"},{"sourceId":157604909,"sourceType":"kernelVersion"},{"sourceId":157869245,"sourceType":"kernelVersion"},{"sourceId":158872676,"sourceType":"kernelVersion"},{"sourceId":158895982,"sourceType":"kernelVersion"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ライブラリ / 汎用関数","metadata":{}},{"cell_type":"code","source":"! pip install japanize_matplotlib --quiet","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:54:27.471697Z","iopub.execute_input":"2024-01-14T12:54:27.472073Z","iopub.status.idle":"2024-01-14T12:54:47.093792Z","shell.execute_reply.started":"2024-01-14T12:54:27.472041Z","shell.execute_reply":"2024-01-14T12:54:47.092346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport pandas.tseries.offsets as offsets\nimport numpy as np\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nimport japanize_matplotlib\nimport seaborn as sns\nimport os\nimport glob\nimport gc\nimport time\nimport IPython.display\nimport collections\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import LSTM, Dense, Conv1D, Conv2D, Conv3D, Dropout\nfrom tensorflow.keras.utils import to_categorical, timeseries_dataset_from_array\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import BinaryCrossentropy\n\nfrom sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score\nfrom sklearn.preprocessing import PolynomialFeatures\nfrom sklearn.linear_model import LinearRegression\n\nfrom scipy.optimize import curve_fit\nfrom scipy.stats import norm, ttest_ind\n\nfrom datetime import timedelta\nfrom scipy.stats import norm\nfrom pathlib import Path\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-14T12:54:47.095960Z","iopub.execute_input":"2024-01-14T12:54:47.096314Z","iopub.status.idle":"2024-01-14T12:55:02.476846Z","shell.execute_reply.started":"2024-01-14T12:54:47.096282Z","shell.execute_reply":"2024-01-14T12:55:02.475683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def graphs(df, id_map, start=0, end=0):\n    if not end:\n        df = df[df['id_map']==id_map].copy().reset_index(drop=True)\n    else:\n        df = df[df['id_map']==id_map].copy().reset_index(drop=True).iloc[start:end]\n    steps = df[df['time'].apply(lambda x: str(x)[-8:]=='00:00:00')]['step']\n    flag = len(df) != df['step'].iat[-1]\n\n    _, axes = plt.subplots(2, 1, figsize=(20,6))\n    axes[0].set_title(id_map)\n    a = 'anglez' if 'anglez' in df.columns else 'a_b_var'\n    e = 'enmo' if 'enmo' in df.columns else 'e_b_mean'        \n    for ax, col in zip(axes.ravel(), [a, e]):\n        if 'state' in df.columns:\n            if col == 'anglez':\n                df_state = df['state'].apply(lambda x: 50 if x==1 else -50 if x==0 else 0)\n            else:\n                df_state = df['state'].apply(lambda x: 2 if x==1 else 0 if x==0 else 1)\n        X = df.index if flag else df['step']\n        ax.plot(X, df[col], linewidth=0.5, marker='', color='0.5', label=col)\n        if 'state' in df.columns:\n            ax.plot(X, df_state, linewidth=1, marker='', color='b', label='state')\n        for step in steps:\n            X = df[df['step']==step].index[0] if flag else step\n            ax.axvline(x=X, linewidth=1, color='r', alpha=0.5)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:55:02.479018Z","iopub.execute_input":"2024-01-14T12:55:02.479914Z","iopub.status.idle":"2024-01-14T12:55:02.494724Z","shell.execute_reply.started":"2024-01-14T12:55:02.479866Z","shell.execute_reply":"2024-01-14T12:55:02.493304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_to_sec(x):\n    return x.timestamp() % SEC_OF_DAY\n\ndef time_to_step(x):\n    return (x.timestamp() % SEC_OF_DAY) / 5","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:55:02.498233Z","iopub.execute_input":"2024-01-14T12:55:02.498702Z","iopub.status.idle":"2024-01-14T12:55:02.509578Z","shell.execute_reply.started":"2024-01-14T12:55:02.498658Z","shell.execute_reply":"2024-01-14T12:55:02.508210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Const","metadata":{}},{"cell_type":"code","source":"HOME = Path('/kaggle')\nORIGIN = HOME / 'input/child-mind-institute-detect-sleep-states'\nCUSTOM = HOME / 'input/zzz-eda-dataset'\nEVAL = HOME / 'input/zzz-eval-1-ipynb'\nWORKING = HOME / 'working'\n\nMODEL = [HOME / 'input/zzz-model-ipynb']\nfor i in range(1,4):\n    path = f'input/zzz-model-{i}-ipynb'\n    MODEL.append(HOME / path)\n\nEVENT_MAP = {'onset':0, 'wakeup':1}\n\n# EXP = ['anglez', 'enmo', 'a_b_var', 'a_a_var', 'e_b_mean', 'e_a_mean', 'pd']\nEXP = ['anglez', 'enmo', 'a_b_var', 'e_b_mean']\nOBJ = ['state']\n\nSTEP_OF_WARMUP = 12 * 60 * 6\nSTEP_OF_RATE = 2\nSTEP_OF_STRIDE = 1\nSTEP_OF_MOVING = 12 * 60 * 6\nSTEP_OF_ROLLING = 12 * 60 * 6\n\nSEC_OF_DAY  = 60 * 60 * 24\nSEC_OF_HOUR = 60 * 60\n\nSTEP_OF_DAY   = 12 * 60 * 24\nSTEP_OF_HOUR  = 12 * 60\nSTEP_OF_30MIN = 12 * 30","metadata":{"execution":{"iopub.status.busy":"2024-01-14T13:19:44.853437Z","iopub.execute_input":"2024-01-14T13:19:44.854732Z","iopub.status.idle":"2024-01-14T13:19:44.865086Z","shell.execute_reply.started":"2024-01-14T13:19:44.854684Z","shell.execute_reply":"2024-01-14T13:19:44.863876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# データ読込","metadata":{}},{"cell_type":"code","source":"%%time\n\ndf = pq.read_table(CUSTOM / 'custom_train_series.parquet').to_pandas()\ndf_test = pq.read_table(ORIGIN / 'test_series.parquet').to_pandas()\ndf_event = pd.read_csv(CUSTOM / 'custom_train_events.csv', index_col=0)\ndf_sample = pd.read_csv(ORIGIN / 'sample_submission.csv', index_col=0)\ndf_id = pd.read_csv(CUSTOM / 'custom_train_stat_groupby_id.csv', index_col=0)\ndf_mean = pd.read_csv(CUSTOM / 'custom_train_series_mean.csv', index_col=0)\ndf_metric = pd.read_csv(CUSTOM / 'metric.csv', index_col=0)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:55:02.525818Z","iopub.execute_input":"2024-01-14T12:55:02.526151Z","iopub.status.idle":"2024-01-14T12:56:07.904836Z","shell.execute_reply.started":"2024-01-14T12:55:02.526116Z","shell.execute_reply":"2024-01-14T12:56:07.903123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id = df_id[['series_id']].to_dict()['series_id']","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:07.907080Z","iopub.execute_input":"2024-01-14T12:56:07.907666Z","iopub.status.idle":"2024-01-14T12:56:07.925953Z","shell.execute_reply.started":"2024-01-14T12:56:07.907619Z","shell.execute_reply":"2024-01-14T12:56:07.924876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_pre(df):\n    df['time'] = df['timestamp'].apply(lambda x: x[:-5]).astype('datetime64[ns]')\n    df = df.merge(df_id[['series_id', 'id_map']], on='series_id', how=\"left\").drop(['series_id', 'timestamp'], axis=1)\n    df.info()\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:07.927384Z","iopub.execute_input":"2024-01-14T12:56:07.927836Z","iopub.status.idle":"2024-01-14T12:56:07.935896Z","shell.execute_reply.started":"2024-01-14T12:56:07.927792Z","shell.execute_reply":"2024-01-14T12:56:07.934552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def moving_var(x_array, term=None, w=STEP_OF_MOVING):\n    w = min(w, len(x_array)//4)\n    m_init = np.mean(x_array[:w]); mean_list = [m_init] #平均の初期値\n    v_init = np.var(x_array[:w]); var_list = [v_init] #分散の初期値\n    for i in range(len(x_array)-w): # 更新式\n        mean_list.append(mean_list[-1] + (-x_array[i] +x_array[i+w])/w)\n        var_list.append(var_list[-1]+mean_list[-2]**2 +(x_array[i+w]**2 - x_array[i]**2)/w  -mean_list[-1]**2)\n    if term == 'before':\n        mean_list = mean_list + [mean_list[-1] for _ in range(w-1)]        \n        var_list = var_list + [var_list[-1] for _ in range(w-1)]\n    if term == 'after':\n        mean_list = [mean_list[0] for _ in range(w-1)] + mean_list\n        var_list = [var_list[0] for _ in range(w-1)] + var_list\n    return mean_list, var_list","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:07.937828Z","iopub.execute_input":"2024-01-14T12:56:07.938294Z","iopub.status.idle":"2024-01-14T12:56:07.951555Z","shell.execute_reply.started":"2024-01-14T12:56:07.938253Z","shell.execute_reply":"2024-01-14T12:56:07.950218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 学習データの加工","metadata":{}},{"cell_type":"code","source":"def add_pd(df, id_map, df_mean=df_mean):\n    df_tmp = df_mean[df_mean['id_map']==id_map][['time', 'w_pd', 'o_pd']].rename(columns={'time':'t'})\n    df['t'] = df['time'].apply(time_to_step).astype('int64')\n    df = df.merge(df_tmp, on='t').sort_values('step').drop('t', axis=1)\n    df['pd'] = df['w_pd'] - df['o_pd']\n    df = df.drop(['o_pd', 'w_pd'], axis=1)\n    return df\n\ndef add_warmup(df, id_map, warmup, col, df_mean=df_mean):\n    if warmup==0: return df\n\n    ts = df['time'].iat[0]\n    ts = ts.timestamp() % SEC_OF_DAY / 5\n    \n    df_tmp = df_mean[df_mean['id_map']==id_map]\n    df_tmp = pd.concat([df_tmp[df_tmp['time']>=ts], df_tmp[df_tmp['time']<ts]])    \n    df_tmp = df_tmp.iloc[-warmup:,:].reset_index(drop=True)\n    df_tmp['step'] = df_tmp.index - warmup\n    df_tmp['pd'] = df_tmp['w_pd'] - df_tmp['o_pd']\n    df_tmp = df_tmp.drop(['o_pd', 'w_pd'], axis=1)\n\n    ts = df['time'].iat[0]\n    ts = [ts + timedelta(seconds=-5*(i+1)) for i in reversed(range(warmup))]\n    df_tmp['time'] = ts\n    df = pd.concat([df_tmp[col], df[col]])\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:07.958428Z","iopub.execute_input":"2024-01-14T12:56:07.958860Z","iopub.status.idle":"2024-01-14T12:56:07.972066Z","shell.execute_reply.started":"2024-01-14T12:56:07.958826Z","shell.execute_reply":"2024-01-14T12:56:07.970738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_test_extract(id_map, warmup, n_val=2, n_test=2, df=df, df_id=df_id):\n    df = df[(df['id_map']==id_map) & (df['state']!=-1)].copy() if 'state' in df.columns else df.copy()\n    df = add_pd(df, id_map).reset_index(drop=True)\n    \n    night   = df_id[df_id['id_map']==id_map]['night'].iat[0]\n    n_train = night - n_test - n_val\n    steps = df_event[(df_event['id_map']==id_map) & (df_event['step']!=-1)]['step']\n\n    if n_test:\n        end    = steps.iat[(n_train+n_val)*2-1]\n        start  = steps.iat[(n_train+n_val)*2]\n        s_test = (end + start) // 2\n    else:\n        s_test = df['step'].iat[-1] + 1\n        \n    if n_val:\n        end   = steps.iat[n_train*2-1]\n        start = steps.iat[n_train*2]\n        s_val = (end + start) // 2\n    else:\n        s_val = s_test\n\n    col = ['id_map', 'step', 'time'] + EXP + (OBJ if 'state' in df.columns else [])\n    df = df[col]\n        \n    df_tmp = df[df['step']<s_val].reset_index(drop=True)\n    train = add_warmup(df_tmp, id_map, warmup, col)\n    df_tmp = df[(df['step']>=s_val) & (df['step']<s_test)].reset_index(drop=True)\n    val = add_warmup(df_tmp, id_map, warmup, col) if n_val else df.head(0)\n    df_tmp = df[df['step']>=s_test].reset_index(drop=True)\n    test  = add_warmup(df_tmp, id_map, warmup, col) if n_test else df.head(0)\n    \n    IPython.display.clear_output()\n    print(f'night: {night}')\n    print(f'train: {n_train}')\n    print(f'val  : {n_val}')\n    print(f'test : {n_test}')\n    \n    return train, val, test","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:07.973632Z","iopub.execute_input":"2024-01-14T12:56:07.974117Z","iopub.status.idle":"2024-01-14T12:56:07.990443Z","shell.execute_reply.started":"2024-01-14T12:56:07.974068Z","shell.execute_reply":"2024-01-14T12:56:07.989181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_window(hour):\n    rolling = 12 * 60 * hour // STEP_OF_STRIDE\n    warmup = STEP_OF_WARMUP + rolling * STEP_OF_STRIDE\n    return rolling, warmup","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:07.992400Z","iopub.execute_input":"2024-01-14T12:56:07.992774Z","iopub.status.idle":"2024-01-14T12:56:08.007589Z","shell.execute_reply.started":"2024-01-14T12:56:07.992731Z","shell.execute_reply":"2024-01-14T12:56:08.006458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(df))\nprint(len(df[df['state']!=-1]))","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:08.009213Z","iopub.execute_input":"2024-01-14T12:56:08.009628Z","iopub.status.idle":"2024-01-14T12:56:17.205191Z","shell.execute_reply.started":"2024-01-14T12:56:08.009595Z","shell.execute_reply":"2024-01-14T12:56:17.203876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習用\ng = df_id[df_id['night']>=20]['id_map'].unique()\nprint(len(g))\nprint(len(df[df['id_map'].isin(g)]))\nprint(len(df[(df['state']!=-1)&(df['id_map'].isin(g))]))","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:17.206973Z","iopub.execute_input":"2024-01-14T12:56:17.208033Z","iopub.status.idle":"2024-01-14T12:56:28.805599Z","shell.execute_reply.started":"2024-01-14T12:56:17.207994Z","shell.execute_reply":"2024-01-14T12:56:28.804504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 検証用\ng = df_id[(df_id['night']<20)&(df_id['night']>0)]['id_map'].unique()\nprint(len(g))\nprint(len(df[df['id_map'].isin(g)]))\nprint(len(df[(df['state']!=-1)&(df['id_map'].isin(g))]))","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:28.807063Z","iopub.execute_input":"2024-01-14T12:56:28.810687Z","iopub.status.idle":"2024-01-14T12:56:38.021887Z","shell.execute_reply.started":"2024-01-14T12:56:28.810639Z","shell.execute_reply":"2024-01-14T12:56:38.020509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graphs(df, 1)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:38.023407Z","iopub.execute_input":"2024-01-14T12:56:38.023795Z","iopub.status.idle":"2024-01-14T12:56:46.331506Z","shell.execute_reply.started":"2024-01-14T12:56:38.023761Z","shell.execute_reply":"2024-01-14T12:56:46.330205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, _, _, = train_test_extract(1, 0, 0, 0)\ngraphs(train, 1)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:46.333105Z","iopub.execute_input":"2024-01-14T12:56:46.334491Z","iopub.status.idle":"2024-01-14T12:56:49.242811Z","shell.execute_reply.started":"2024-01-14T12:56:46.334442Z","shell.execute_reply":"2024-01-14T12:56:49.241597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# モデルの学習","metadata":{}},{"cell_type":"code","source":"warmup = STEP_OF_WARMUP + STEP_OF_ROLLING # stride処理前のstepに対してのwarmup\nstride = STEP_OF_STRIDE\nrate   = STEP_OF_RATE\n\nprint(f'warmup: {warmup}')\nprint(f'stride: {stride}')\nprint(f'rate  : {rate}')","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.244704Z","iopub.execute_input":"2024-01-14T12:56:49.245390Z","iopub.status.idle":"2024-01-14T12:56:49.252190Z","shell.execute_reply.started":"2024-01-14T12:56:49.245353Z","shell.execute_reply":"2024-01-14T12:56:49.251234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def std_arr(arr):\n    return (arr - arr.mean()) / arr.std()\n\ndef std_df(df, cols=EXP):\n    df = df.copy()\n    for col in cols:\n        mean = df[col].mean()\n        std = df[col].std()\n        df[col] = (df[col] - mean) / std\n    return df\n\ndef make_dataset(df, stride=STEP_OF_STRIDE, rate=STEP_OF_RATE, warmup=STEP_OF_WARMUP):\n    ds = timeseries_dataset_from_array(\n        data=df[EXP],\n        targets=df[OBJ][warmup:] if OBJ[0] in df.columns else None,\n        sequence_length=warmup // rate,\n        sampling_rate=rate,\n        sequence_stride=stride\n    )\n    return ds\n\ndef class_weight(df):\n    weight = len(df[df['state']==1]) / len(df[df['state']==0])\n    return {0: weight, 1: 1}\n\ndef handle_history(history, id_map):\n    df = pd.DataFrame.from_dict(history)\n    df['epoch'] = df.index\n    df['id_map'] = id_map\n    return df[['id_map','epoch','loss','accuracy','val_loss','val_accuracy']]\n\ndef prediction(model, df, id_map=-1, stride=STEP_OF_STRIDE, warmup=STEP_OF_WARMUP):\n    ds = make_dataset(std_df(df), stride)\n    pred = model.predict(ds).flatten()\n    pred = pd.Series(pred, index=range(0, len(pred)*stride, stride))\n    plt.figure(figsize=(20,4))\n    plt.plot(pred)\n    plt.plot(df.iloc[warmup:,:]['state'].reset_index(drop=True))\n    plt.title(f\"model: {id_map:03d}, pred: {df['id_map'].iat[0]:03d}\")\n    plt.show()\n    return pred","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.253537Z","iopub.execute_input":"2024-01-14T12:56:49.254092Z","iopub.status.idle":"2024-01-14T12:56:49.270971Z","shell.execute_reply.started":"2024-01-14T12:56:49.254059Z","shell.execute_reply":"2024-01-14T12:56:49.269701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def handle_model(id_map, key=False ,model=False):\n    if not model:\n        model = Sequential([\n            LSTM(64, return_sequences=False),\n            Dropout(0.1),\n            Dense(units=1, activation='sigmoid')\n        ])\n    model.compile(optimizer=Adam(),\n        loss=BinaryCrossentropy(from_logits=False),\n        metrics=[\"binary_crossentropy\"])\n    \n    early_stopping = EarlyStopping(monitor='val_loss', patience=5, mode='min')\n\n    key = ('_'+key) if key else ''\n    checkpoint_callback = ModelCheckpoint(\n        filepath=f'/kaggle/working/model_{id_map:03d}{key}.tf',\n        monitor='val_loss',\n        mode='min',\n        save_best_only=True,\n        save_weights_only=False,\n        verbose=1\n    )\n    \n    return model, early_stopping, checkpoint_callback","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.272779Z","iopub.execute_input":"2024-01-14T12:56:49.273176Z","iopub.status.idle":"2024-01-14T12:56:49.284239Z","shell.execute_reply.started":"2024-01-14T12:56:49.273128Z","shell.execute_reply":"2024-01-14T12:56:49.282846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"モデル作成用: {len(df_id[df_id['night']>=20]['night'])}\")\nprint(f\"検証用　　　: {len(df_id[(df_id['night']<20)&(df_id['night']!=0)]['night'])}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.286180Z","iopub.execute_input":"2024-01-14T12:56:49.286656Z","iopub.status.idle":"2024-01-14T12:56:49.305195Z","shell.execute_reply.started":"2024-01-14T12:56:49.286617Z","shell.execute_reply":"2024-01-14T12:56:49.304174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# models = pd.Series()\n# c = 0\n# for i, id_map in enumerate(df_id[df_id['night']>=20].index):\n#     if i % 50 == 0:\n#         c += 1\n#     path = f\"input/zzz-model-{c}-ipynb/model_{id_map:03d}.tf\"\n#     models[id_map] = tf.keras.models.load_model(HOME / path)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.307007Z","iopub.execute_input":"2024-01-14T12:56:49.307582Z","iopub.status.idle":"2024-01-14T12:56:49.316645Z","shell.execute_reply.started":"2024-01-14T12:56:49.307527Z","shell.execute_reply":"2024-01-14T12:56:49.315469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_pre_score = []\n# for i in range(1, 4):\n#     path = f\"input/zzz-model-{i}-ipynb/pred_score.csv\"\n#     df_tmp = pd.read_csv(HOME / path, index_col=0)\n#     df_pre_score = pd.concat([df_pre_score, df_tmp]) if len(df_pre_score) else df_tmp\n# df_pre_score = df_pre_score.reset_index(drop=True)\n# df_pre_score.sort_values('F1', ascending=False).head(10)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.318186Z","iopub.execute_input":"2024-01-14T12:56:49.318651Z","iopub.status.idle":"2024-01-14T12:56:49.328733Z","shell.execute_reply.started":"2024-01-14T12:56:49.318608Z","shell.execute_reply":"2024-01-14T12:56:49.327448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.hist(df_pre_score['F1'])\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.330247Z","iopub.execute_input":"2024-01-14T12:56:49.331558Z","iopub.status.idle":"2024-01-14T12:56:49.343300Z","shell.execute_reply.started":"2024-01-14T12:56:49.331521Z","shell.execute_reply":"2024-01-14T12:56:49.342131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 変化点の検出","metadata":{}},{"cell_type":"code","source":"def pred_pre(pred, rolling=STEP_OF_ROLLING, stride=STEP_OF_STRIDE):\n    w = STEP_OF_ROLLING // STEP_OF_STRIDE\n    pred = pred.rolling(w).mean()[w-1:].reset_index(drop=True)\n    pred = [[i*stride, i, x-0.5] for i, x in enumerate(pred)]\n    pred = pd.DataFrame(pred, columns=['idx', 'sequence', 'state'])\n    return pred","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.344635Z","iopub.execute_input":"2024-01-14T12:56:49.345395Z","iopub.status.idle":"2024-01-14T12:56:49.356168Z","shell.execute_reply.started":"2024-01-14T12:56:49.345344Z","shell.execute_reply":"2024-01-14T12:56:49.355050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def event_handle(pred, test, stride=STEP_OF_STRIDE, series_id=series_id):\n    series_id = series_id[test['id_map'].iat[0]]\n    d = 12 * 30\n    w = 12 * 60 * 18\n    step = 0\n    state = 'onset'\n    event = []\n    while True:\n        s = min(step + d, pred['idx'].iat[-1])\n        e = min(s + w, pred['idx'].iat[-1])\n        p_tmp = pred[(pred['idx']>=s) & (pred['idx']<=e)]\n        if state == 'onset':\n            i = p_tmp['state'].idxmax()\n        else:\n            i = p_tmp['state'].idxmin()\n        step = p_tmp[p_tmp.index==i]['idx'].iat[0]\n        event.append([series_id, step, state, 0.5])\n        state = 'onset' if state=='wakeup' else 'wakeup'\n        if (state=='onset') & (e==pred['idx'].iat[-1]):\n            break\n    return event","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.359292Z","iopub.execute_input":"2024-01-14T12:56:49.360354Z","iopub.status.idle":"2024-01-14T12:56:49.371090Z","shell.execute_reply.started":"2024-01-14T12:56:49.360286Z","shell.execute_reply":"2024-01-14T12:56:49.369830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_event(pred, event):\n    plt.figure(figsize=(10,4))\n    plt.plot(p_tmp['idx'], p_tmp['state'])\n    for x in event:\n        plt.axvline(x[1], c='r', linewidth=0.5)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.372903Z","iopub.execute_input":"2024-01-14T12:56:49.374004Z","iopub.status.idle":"2024-01-14T12:56:49.387175Z","shell.execute_reply.started":"2024-01-14T12:56:49.373959Z","shell.execute_reply":"2024-01-14T12:56:49.385994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def state_handle(df, event):\n    df = df[df['step']>=0].copy()\n    state = [1 for _ in range(event[0][1])]\n    for i in range(1, len(event)):\n        x = 1 if event[i][2] == 'onset' else 0\n        state += [x for _ in range(event[i][1]-event[i-1][1])]\n    state += [1 for _ in range(event[-1][1], len(df))]\n    df['pred'] = state\n    df.index.name = ''\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.394273Z","iopub.execute_input":"2024-01-14T12:56:49.394653Z","iopub.status.idle":"2024-01-14T12:56:49.403188Z","shell.execute_reply.started":"2024-01-14T12:56:49.394621Z","shell.execute_reply":"2024-01-14T12:56:49.401961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prob_handle(df, pred, stride=STEP_OF_STRIDE):\n    prob = np.array([[x for _ in range(stride)] for x in pred['state']]).flatten()[-len(df):]\n    df['prob'] = prob\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.405016Z","iopub.execute_input":"2024-01-14T12:56:49.405512Z","iopub.status.idle":"2024-01-14T12:56:49.415788Z","shell.execute_reply.started":"2024-01-14T12:56:49.405471Z","shell.execute_reply":"2024-01-14T12:56:49.414728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# モデルの評価（個別）","metadata":{}},{"cell_type":"code","source":"# 実測-予測\n# 睡眠-睡眠：True Positive (TP)\n# 睡眠-起床：False Negative (FN)\n# 起床-睡眠：False Positive (FP)\n# 起床-起床：True Negative (TN)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_score = pd.read_csv(MODEL[0] / 'pred_score.csv', index_col=0)\ndf_score","metadata":{"execution":{"iopub.status.busy":"2024-01-14T14:59:52.546947Z","iopub.execute_input":"2024-01-14T14:59:52.547490Z","iopub.status.idle":"2024-01-14T14:59:52.593392Z","shell.execute_reply.started":"2024-01-14T14:59:52.547450Z","shell.execute_reply":"2024-01-14T14:59:52.592054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = {\n    'Accuracy' : 'Accuracy（正解率）\\nどれだけ正確に予測できているか',\n    'Recall'   : 'Recall（再現率）\\nどれだけ取りこぼしなく予測することができたか',\n    'Precision': 'Precision（適合率）\\n正と予測したものがどれだけ正しかったか',\n    'F1'       : 'F1\\nRecallとPrecisionのバランス'}\n_, axes = plt.subplots(2, 2, figsize=(20,10))\naxes = axes.ravel()\nfor i, (k, v) in enumerate(d.items()):\n    axes[i].hist(df_score[k])\n    axes[i].set_title(v)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:01:32.476617Z","iopub.execute_input":"2024-01-14T15:01:32.477164Z","iopub.status.idle":"2024-01-14T15:01:33.539400Z","shell.execute_reply.started":"2024-01-14T15:01:32.477123Z","shell.execute_reply":"2024-01-14T15:01:33.538191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = {\n    'TP': 'True Positive (TP, 真陽性)\\n睡眠(実)-睡眠(予)',\n    'FN': 'False Negative (FN, 偽陰性)\\n睡眠(実)-起床(予)',\n    'FP': 'False Positive (FP, 偽陽性)\\n起床(実)-睡眠(予)',\n    'TN': 'True Negative (TN, 真陰性)\\n起床(実)-起床(予)'}\n_, axes = plt.subplots(2, 2, figsize=(20,10))\naxes = axes.ravel()\nfor i, (k, v) in enumerate(d.items()):\n    axes[i].hist(df_score[k])\n    axes[i].set_title(v)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T15:01:36.075712Z","iopub.execute_input":"2024-01-14T15:01:36.076147Z","iopub.status.idle":"2024-01-14T15:01:37.101555Z","shell.execute_reply.started":"2024-01-14T15:01:36.076109Z","shell.execute_reply":"2024-01-14T15:01:37.100467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = glob.glob('/kaggle/input/zzz-model-ipynb/pred_???_*')\npreds = pd.Series()\nfor i, path in enumerate(paths):\n    preds[path[-16:-4]] = pd.read_csv(path, index_col=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(preds)):\n    p = preds.iloc[i]\n    plt.figure(figsize=(20,2))\n    plt.title(preds.index[i])\n    plt.plot(p['state'])\n    plt.plot(p['pred'])\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# モデルの評価（バギング）","metadata":{}},{"cell_type":"code","source":"df_score = pd.read_csv(EVAL / 'pred_score.csv', index_col=0)\ndf_score","metadata":{"execution":{"iopub.status.busy":"2024-01-14T13:23:29.399978Z","iopub.execute_input":"2024-01-14T13:23:29.400451Z","iopub.status.idle":"2024-01-14T13:23:29.435595Z","shell.execute_reply.started":"2024-01-14T13:23:29.400404Z","shell.execute_reply":"2024-01-14T13:23:29.434297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = 4\ncols = ['F1', 'Accuracy', 'Recall', 'Precision']\nfor col in cols:\n    _, axes = plt.subplots(1, n, figsize=(20,3))\n    axes = axes.ravel()\n    for i in range(n):\n        m = df_score[df_score['model']==i+1][col].mean()\n        axes[i].hist(df_score[df_score['model']==i+1][col])\n        axes[i].axvline(x=m, color='r', linewidth=1)\n        axes[i].set_title(f\"n={i+1}  :  {col}={m:.3f}\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T13:45:57.617655Z","iopub.execute_input":"2024-01-14T13:45:57.618173Z","iopub.status.idle":"2024-01-14T13:46:02.352743Z","shell.execute_reply.started":"2024-01-14T13:45:57.618136Z","shell.execute_reply":"2024-01-14T13:46:02.351568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# paths = glob.glob('/kaggle/input/zzz-model-ipynb/pred_???_*')\n# preds = pd.Series()\n# for i, path in enumerate(paths):\n#     preds[path[-16:-4]] = pd.read_csv(path, index_col=0)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T12:56:49.416673Z","iopub.execute_input":"2024-01-14T12:56:49.416979Z","iopub.status.idle":"2024-01-14T12:56:49.426876Z","shell.execute_reply.started":"2024-01-14T12:56:49.416952Z","shell.execute_reply":"2024-01-14T12:56:49.425670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = glob.glob('/kaggle/input/zzz-eval-1-ipynb/pred_???_*')\npreds = pd.Series()\nfor i, path in enumerate(paths):\n    preds[path[-14:-4]] = pd.read_csv(path, index_col=0)","metadata":{"execution":{"iopub.status.busy":"2024-01-14T13:21:24.290271Z","iopub.execute_input":"2024-01-14T13:21:24.290713Z","iopub.status.idle":"2024-01-14T13:23:29.397686Z","shell.execute_reply.started":"2024-01-14T13:21:24.290679Z","shell.execute_reply":"2024-01-14T13:23:29.396373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = preds['pred_111_1']\nplt.figure(figsize=(20,3))\nplt.plot(p['state'])\nplt.plot(p['pred'])\nplt.yticks([0,1],['睡眠','起床'])\nplt.tick_params(labelbottom=False)\nplt.tick_params(labelsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T14:05:33.193344Z","iopub.execute_input":"2024-01-14T14:05:33.194340Z","iopub.status.idle":"2024-01-14T14:05:33.514674Z","shell.execute_reply.started":"2024-01-14T14:05:33.194279Z","shell.execute_reply":"2024-01-14T14:05:33.513592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = preds['pred_061_1']\nplt.figure(figsize=(20,3))\nplt.plot(p['state'])\nplt.plot(p['pred'])\nplt.yticks([0,1],['睡眠','起床'])\nplt.tick_params(labelbottom=False)\nplt.tick_params(labelsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T14:48:40.475234Z","iopub.execute_input":"2024-01-14T14:48:40.475833Z","iopub.status.idle":"2024-01-14T14:48:40.811501Z","shell.execute_reply.started":"2024-01-14T14:48:40.475786Z","shell.execute_reply":"2024-01-14T14:48:40.810375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = preds['pred_125_1']\nplt.figure(figsize=(20,3))\nplt.plot(p['state'])\nplt.plot(p['pred'])\nplt.yticks([0,1],['睡眠','起床'])\nplt.tick_params(labelbottom=False)\nplt.tick_params(labelsize=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T14:48:50.458101Z","iopub.execute_input":"2024-01-14T14:48:50.458566Z","iopub.status.idle":"2024-01-14T14:48:50.766797Z","shell.execute_reply.started":"2024-01-14T14:48:50.458528Z","shell.execute_reply":"2024-01-14T14:48:50.765585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(preds)):\n    p = preds.iloc[i]\n    plt.figure(figsize=(20,2))\n    plt.title(preds.index[i])\n    plt.plot(p['state'])\n    plt.plot(p['pred'])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-14T14:58:28.542897Z","iopub.execute_input":"2024-01-14T14:58:28.543439Z","iopub.status.idle":"2024-01-14T14:58:41.344702Z","shell.execute_reply.started":"2024-01-14T14:58:28.543397Z","shell.execute_reply":"2024-01-14T14:58:41.342921Z"},"trusted":true},"execution_count":null,"outputs":[]}]}