{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7193425,"sourceType":"datasetVersion","datasetId":4158512}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport time\nimport psutil\nfrom contextlib import contextmanager\nimport pickle\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport seaborn as sns\nimport lightgbm as lgb\n\ndef show_memory_usage(name = \"unknown\"):\n    vm = psutil.virtual_memory()\n    print(f\"[MEMUSE] memory usage (in {name}): {vm.used/1024/1024:.2f}MB ({vm.percent}%)\")\n\n@contextmanager\ndef timer(name: str):\n    show_memory_usage(f\"before {name}\")\n    s = time.time()\n    yield\n    elapsed = time.time() - s\n    print(f\"[{name}] {elapsed:.3f}sec\")\n    show_memory_usage(f\"after {name}\")\n    \nclass Config:\n    is_train = True\n    is_debug = True\n    is_rerun = os.getenv('KAGGLE_IS_COMPETITION_RERUN')\n    src_path = \"/kaggle/input/cmisleep-2ndplace-kmat\"\n    train_path = '/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet'\n    train_target_path = '/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv'\n    test_path = '/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet'\n    sample_submission_path = '/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv'\n    \n    lgbm_path = \"/kaggle/input/cmisleep-2ndplace-kmat/trained_model/lgbm_models\"\n    # weights\n    lgbm_paths_2nd = [[os.path.join(lgbm_path, \"lgb_acm_0_th30\"),\n                       os.path.join(lgbm_path, \"lgb_acm_0_th50\"),\n                       os.path.join(lgbm_path, \"lgb_acm_0_th70\"),\n                      ],\n                      [os.path.join(lgbm_path, \"lgb_acm_1_th30\"),\n                       os.path.join(lgbm_path, \"lgb_acm_1_th50\"),\n                       os.path.join(lgbm_path, \"lgb_acm_1_th70\"),\n                      ],\n                     ]\n    lgbm_sub_paths_2nd = [[os.path.join(lgbm_path, \"lgb_simple_0\")],\n                          [os.path.join(lgbm_path, \"lgb_simple_1\")]]\n    \n    lgbm_path_third = os.path.join(lgbm_path, \"lgb_third\")\n    \n    preprocess_dir = 'data_processed/'\n    pred_dirs = ['preds_0/', 'preds_1/']\n    steps_per_sec = 0.2\n    step_for_a_day = 60 * steps_per_sec * 60 * 24\n    step_for_30min = 60 * steps_per_sec * 30\n    step_for_15min = 60 * steps_per_sec * 15\n    step_for_1min = 60 * steps_per_sec\n    data_for_debug = 100\n    \n    pred_overlap = 0.5\n    pred_batch = 48\n\ncfg = Config()\nos.makedirs(cfg.preprocess_dir, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:44:24.534369Z","iopub.execute_input":"2023-12-13T14:44:24.534778Z","iopub.status.idle":"2023-12-13T14:44:24.547864Z","shell.execute_reply.started":"2023-12-13T14:44:24.534746Z","shell.execute_reply":"2023-12-13T14:44:24.546664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext cython","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:42:26.604774Z","iopub.execute_input":"2023-12-13T14:42:26.605169Z","iopub.status.idle":"2023-12-13T14:42:27.610409Z","shell.execute_reply.started":"2023-12-13T14:42:26.605133Z","shell.execute_reply":"2023-12-13T14:42:27.609288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%cython\nimport numpy as np\ncimport numpy as cnp\ncimport cython\n\ndef cumsum_morethan_zero(cnp.ndarray[cnp.float64_t, ndim=1] x):\n    cdef int i, n\n    n = x.shape[0]\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] y = np.zeros(n)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] y_rev = np.zeros(n)\n    y[0] = x[0]\n    for i in range(1, n):\n        if x[i] == 0:\n            y[i] = 0\n        else:\n            y[i] = y[i-1] + x[i]\n    y_rev[-1] = y[-1]\n    for i in range(n-2, -1, -1):\n        if y_rev[i+1] > y[i]:\n            if x[i] == 0:\n                y_rev[i] = 0\n            else:\n                y_rev[i] = y_rev[i+1]\n        else:\n            y_rev[i] = y[i]\n    return y_rev\n\ndef easy_convolve(cnp.ndarray[cnp.float64_t, ndim=1] x, int filter_size):\n    \"\"\"\n    padding same, kernel is ones\n    \"\"\"\n    cdef int i, j, n, p, m\n    m = filter_size - 1\n    p = m // 2\n    n = x.shape[0]\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] x_p = np.zeros(n+2*p)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] y = np.zeros(n)\n    x_p[p:n+p] = x\n\n    for j in range(filter_size):\n        y[0] += x_p[j]\n\n    for i in range(1, n):# filter_size, n+p+p-filter_size+1):\n        y[i] = x_p[i+m] + y[i-1] - x_p[i-1]\n    return y\n\ndef minimum(cnp.ndarray[cnp.float64_t, ndim=1] x, cnp.float64_t maxval):\n    cdef int i, n\n    n = x.shape[0]\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] y = np.zeros(n)\n    for i in range(n):\n        y[i] = min(x[i], maxval)\n    return y\n\ndef easy_closing(cnp.ndarray[cnp.float64_t, ndim=1] x, int filter_size):\n    \"\"\"\n    closing = dilation -> erosion\n    padding same, kernel is ones, x is 0 or 1\n    \"\"\"\n    x = easy_convolve(x, filter_size)\n    x = minimum(x, 1)\n    x = 1 - x\n    x = easy_convolve(x, filter_size)\n    x = minimum(x, 1)\n    x = 1 - x\n    return x\n\ndef easy_closing_q(cnp.ndarray[cnp.float64_t, ndim=1] x, int filter_size):\n    \"\"\"\n    closing = dilation -> erosion\n    padding same, kernel is ones, x is 0 or 1\n    \"\"\"\n    cdef int i, j, n, p, m\n    m = filter_size - 1\n    p = m // 2\n    n = x.shape[0]\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] x_p = np.zeros(n+2*p)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] y_p = np.zeros(n+2*p)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] y = np.zeros(n)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] z = np.zeros(n)\n    \n    x_p[p:n+p] = x\n    for j in range(filter_size):\n        y[0] += x_p[j]\n    for i in range(1, n):# filter_size, n+p+p-filter_size+1):\n        y[i] = x_p[i+m] + y[i-1] - x_p[i-1]\n    for i in range(n):\n        y[i] = 1 - min(y[i], 1)\n    \n    y_p[p:n+p] = y\n    for j in range(filter_size):\n        z[0] += y_p[j]\n    for i in range(1, n):# filter_size, n+p+p-filter_size+1):\n        z[i] = y_p[i+m] + z[i-1] - y_p[i-1]\n    for i in range(n):\n        z[i] = 1 - min(z[i], 1)\n    \n    return z\n\n\ndef detect_peak(cnp.ndarray[cnp.float64_t, ndim=1] x, int k):\n    cdef int n = x.shape[0]\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] max_array = np.zeros(n, dtype=np.float64)\n    cdef cnp.ndarray[cnp.int32_t, ndim=1] max_indices = np.zeros(n, dtype=np.int32)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] result = np.zeros(n, dtype=np.float64)\n    cdef int i, j, start, end, max_index\n    \n    # calculate max values in each window\n    for i in range(n):\n        start = max(0, i - k)\n        end = min(n, i + k + 1)\n        max_index = start\n        for j in range(start, end):\n            if x[j] > x[max_index]:\n                max_index = j\n        max_array[i] = x[max_index]\n        max_indices[i] = max_index\n    \n    # set peak values to 1\n    for i in range(n):\n        if x[i] == max_array[max_indices[i]]:\n            result[i] = 1.0\n    \n    return max_array\n\ndef detect_peak_r(cnp.ndarray[cnp.float64_t, ndim=1] x, int k):\n    cdef int n = x.shape[0]\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] max_array = np.zeros(n, dtype=np.float64)\n    cdef cnp.ndarray[cnp.int32_t, ndim=1] max_indices = np.zeros(n, dtype=np.int32)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] result = np.zeros(n, dtype=np.float64)\n    cdef int i, j, start, end, max_index\n    \n    # calculate max values in first window\n    max_index = 0\n    for i in range(k):\n        if x[i] > x[max_index]:\n            max_index = i\n    max_array[k-1] = x[max_index]\n    max_indices[k-1] = max_index\n    \n    # calculate max values in each window\n    for i in range(k, n):\n        start = i - k\n        end = i\n        if max_index == start - 1:\n            max_index = start\n            for j in range(start, end):\n                if x[j] > x[max_index]:\n                    max_index = j\n        else:\n            if x[i] > x[max_index]:\n                max_index = i\n        max_array[i] = x[max_index]\n        max_indices[i] = max_index\n    \n    # set peak values to 1\n    for i in range(n):\n        if x[i] == max_array[max_indices[i]]:\n            result[i] = 1.0\n    \n    return max_array\n\n@cython.boundscheck(False)\n@cython.wraparound(False)\ndef detect_peak_kmat(cnp.ndarray[cnp.float64_t, ndim=1] x, int k):\n    cdef int n = x.shape[0]\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] max_array = np.zeros(n, dtype=np.float64)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] result_val = np.zeros(n, dtype=np.float64)\n    cdef cnp.ndarray[cnp.float64_t, ndim=1] result = np.zeros(n, dtype=np.float64)\n    cdef int i, j, start, end, max_index, half_k\n\n    half_k = k // 2\n    for i in range(half_k, n-half_k):\n        result[i] = 1\n        result_val[i] = x[i]\n        for j in range(1, half_k+1):\n            if x[i] < x[i-j]:\n                result[i] = 0\n                result_val[i] = 0\n                break\n            if x[i] < x[i+j]:\n                result[i] = 0\n                result_val[i] = 0\n                break\n    return result, result_val\n","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:42:27.613530Z","iopub.execute_input":"2023-12-13T14:42:27.614242Z","iopub.status.idle":"2023-12-13T14:42:31.494423Z","shell.execute_reply.started":"2023-12-13T14:42:27.614193Z","shell.execute_reply":"2023-12-13T14:42:31.493007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def timestamp_to_step_single_id(df_train_id, df_target_id=None):\n    \"\"\"\n    Convert timestamp to step\n    timestepは2018-08-14T22:26:00-0400といった形で与えられる\n    \n    \"\"\"\n    offset_date = df_train_id['timestamp'].iloc[0:1].str.split('T', expand=True)[0]\n    offset_time = df_train_id['timestamp'].iloc[0:1].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    offset = pd.to_datetime(offset_date + ' ' + offset_time)\n    offset_step = offset.dt.hour * 60 * 12 + offset.dt.minute * 12 + offset.dt.second / 5\n    df_train_id[\"daily_step\"] = (df_train_id['step'] + offset_step.values[0]) % cfg.step_for_a_day\n    if df_target_id is not None:\n        df_target_id[\"daily_step\"] = (df_target_id['step'] + offset_step.values[0]) % cfg.step_for_a_day\n    # print((df['step'].iloc[-1] + offset_step) % cfg.step_for_a_day)\n    return df_train_id, df_target_id\n\ndef sample_train_target_by_id(df_train, df_target, id_no):\n    return df_train[df_train['series_id']==id_no], df_target[df_target['series_id']==id_no]\n\n\ndef find_sensor_error(df_train_id, column='anglez'):\n    \n    def padding(x, left, right, value=0):\n        return np.pad(x, (left, right), 'constant', constant_values=(value, value))\n\n\n    x = df_train_id['daily_step'].values\n    y = df_train_id[column].values\n    mask = np.ones_like(y)\n    pad_left = int(x[0])\n    pad_right = int(cfg.step_for_a_day) -1 - int(x[-1]) % int(cfg.step_for_a_day)\n    \n    x = padding(x, pad_left, pad_right, value=0)\n    y = padding(y, pad_left, pad_right, value=0)\n    y_dif = padding(np.abs(y[1:] - y[:-1]), 1, 0, value=0)\n    mask = padding(mask, pad_left, pad_right, value=0)\n\n\n    # reshape to (num_day, step_for_a_day)\n    x = x.reshape(-1, int(cfg.step_for_a_day))\n    y = y.reshape(-1, int(cfg.step_for_a_day))\n    y_dif = y_dif.reshape(-1, int(cfg.step_for_a_day))\n    mask = mask.reshape(-1, int(cfg.step_for_a_day))\n    day_counter = np.cumsum(mask, axis=0) # (num_day, step_for_a_day)\n\n    # メモリ注意。\n    delta_matrix = y[np.newaxis, :, :] - y[:, np.newaxis, :] # (num_day, num_day, step_for_a_day)\n    mask_matrix = mask[np.newaxis, :, :] * mask[:, np.newaxis, :] # (num_day, num_day, step_for_a_day)\n    delta_matrix = (delta_matrix==0) * mask_matrix # where delta is zero except for the padding area\n    \n    delta_matrix = np.sum(delta_matrix, axis=0) - 1 # (num_day, step_for_a_day). 同じデータ同士の差は0になるので-1\n    mask_matrix = np.sum(mask_matrix, axis=0) # (num_day, step_for_a_day)\n    nan_counter = np.cumsum(delta_matrix > 0, axis=0) # (num_day, step_for_a_day) 初回のnan風なものは案外ラベルがついている？\n    nan_exist_other_day = np.any(delta_matrix > 0, axis=0, keepdims=True) # (1, step_for_a_day)\n    nan_exist_other_day = np.tile(nan_exist_other_day, (delta_matrix.shape[0], 1)) # (num_day, step_for_a_day)\n\n    maybe_not_nan = delta_matrix == 0\n    valid = mask * maybe_not_nan\n    y_valid = y_dif * maybe_not_nan # (num_day, step_for_a_day) ★一時的にy_difを使う。\n    y_sum = np.sum(y_valid, axis=0, keepdims=True) # (1, step_for_a_day)\n    y_mean = y_sum / (np.sum(valid, axis=0, keepdims=True)+1e-7) # (1, step_for_a_day)\n    y_dev = (y_valid - y_mean) * valid # (num_day, step_for_a_day)\n    y_std = np.sqrt(np.sum(y_dev**2, axis=0, keepdims=True) / (np.sum(valid, axis=0, keepdims=True)+1e-7)) # (1, step_for_a_day)\n    # mean, stdをtileして、yと同じshapeにする\n    y_mean = np.tile(y_mean, (y.shape[0], 1))\n    y_std = np.tile(y_std, (y.shape[0], 1)) # (num_day, step_for_a_day)\n\n\n    # flatten\n    delta_matrix = delta_matrix.reshape(-1)\n    mask_matrix = mask_matrix.reshape(-1)\n    day_counter = day_counter.reshape(-1)\n    nan_counter = nan_counter.reshape(-1)\n    nan_exist_other_day = nan_exist_other_day.reshape(-1)\n    y_mean = y_mean.reshape(-1)\n    y_std = y_std.reshape(-1)\n\n    # left, rightのpadding部分を除く\n    start = pad_left\n    end = -pad_right if pad_right > 0 else len(delta_matrix)\n    delta_matrix = delta_matrix[start:end]\n    mask_matrix = mask_matrix[start:end]\n    day_counter = day_counter[start:end]\n    nan_counter = nan_counter[start:end]\n    nan_exist_other_day = nan_exist_other_day[start:end]\n    y_mean = y_mean[start:end]\n    y_std = y_std[start:end]\n\n    df_train_id[column+\"_numrepeat\"] = delta_matrix\n    df_train_id[column+\"_daycount\"] = mask_matrix\n    df_train_id[column+\"_daycounter\"] = day_counter\n    df_train_id[column+\"_nancounter\"] = nan_counter\n    df_train_id[column+\"_nanexist\"] = nan_exist_other_day\n    df_train_id[column+\"_daymean\"] = y_mean\n    df_train_id[column+\"_daystd\"] = y_std\n\n    # smoothing 5step closing    \n    df_train_id[column+\"_simpleerror\"] = easy_closing_q((delta_matrix > 0).astype(np.float64),  5) # 5step\n    df_train_id[column+\"_simpleerror_span\"] = cumsum_morethan_zero(df_train_id[column+\"_simpleerror\"].values)\n    df_train_id[column+\"_simpleerror_v2\"] = df_train_id[column+\"_simpleerror\"].astype(int) + (df_train_id[column+\"_simpleerror_span\"] > cfg.step_for_30min).astype(int) + (df_train_id[column+\"_simpleerror_span\"] > (cfg.step_for_30min*2)).astype(int) # \n\n    return df_train_id\n\ndef make_binary_target(df_train_id, df_target_id, id_no):\n    df_train_id['target'] = -1\n    \n    current_state = -1\n    current_step = 0\n    # state would continues no less than 1 or 2hour \n    continue_length = int(cfg.step_for_30min*4)\n\n    for total_step, event_type, night_no in zip(df_target_id['step'].values, df_target_id['event'].values, df_target_id['night'].values):\n        tmp = current_state\n        data_exist = not np.isnan(total_step)\n        if event_type == 'onset' and data_exist:\n            if current_state == 1 or current_state == -1:\n                df_train_id.loc[(df_train_id['step'] >= current_step) & (df_train_id['step'] < total_step), 'target'] = 1\n            if current_state == -2: # nan before onset. \n                df_train_id.loc[(df_train_id['step'] >= np.maximum(total_step - continue_length, 0)) & (df_train_id['step'] < total_step), 'target'] = 1\n            current_state = 0\n            current_step = total_step\n        elif event_type == 'wakeup' and data_exist:\n            if current_state == 0 or current_state == -1:\n                df_train_id.loc[(df_train_id['step'] >= current_step) & (df_train_id['step'] < total_step), 'target'] = 0\n            if current_state == -2: # nan before wakeup. \n                df_train_id.loc[(df_train_id['step'] >= np.maximum(total_step - continue_length, 0)) & (df_train_id['step'] < total_step), 'target'] = 0\n            current_state = 1\n            current_step = total_step\n        elif not data_exist: # nan\n            if current_state == 1: # nan after wakeup\n                df_train_id.loc[(df_train_id['step'] >= current_step) & (df_train_id['step'] < current_step+continue_length), 'target'] = 1\n            elif current_state == 0: # nan after onset\n                df_train_id.loc[(df_train_id['step'] >= current_step) & (df_train_id['step'] < current_step+continue_length), 'target'] = 0\n\n            current_state = -2\n        # print(tmp, \"->\", event_type, \"->\", current_state, data_exist, total_step)\n\n    if current_state == 1: # nan after wakeup\n        df_train_id.loc[(df_train_id['step'] >= current_step) & (df_train_id['step'] < current_step+continue_length), 'target'] = 1\n        end_step_for_train = current_step+continue_length\n    elif current_state == 0: # nan after onset\n        df_train_id.loc[(df_train_id['step'] >= current_step) & (df_train_id['step'] < current_step+continue_length), 'target'] = 0\n        end_step_for_train = current_step+continue_length\n    \n    elif current_state == -2:\n        end_step_for_train = (night_no - 1) * cfg.step_for_a_day\n\n    \n    return df_train_id, end_step_for_train\n\n\ndef prerprocess_inputs_for_mlmodel(df_train_id):\n    df_train_id[\"daily_step\"] = (df_train_id[\"daily_step\"] / cfg.step_for_a_day).astype(np.float32)\n    df_train_id[\"anglez\"] = (df_train_id[\"anglez\"] / 90).astype(np.float32)\n    df_train_id[\"anglez_simpleerror\"] = df_train_id[\"anglez_simpleerror\"].astype(np.float32)\n    # anglez_shimpleerror_spanはlog1pをとっておく\n    df_train_id[\"anglez_simpleerror_span\"] = np.clip(np.log1p(df_train_id[\"anglez_simpleerror_span\"]) / 10, 0, 1).astype(np.float32)\n    df_train_id[\"anglez_nanexist\"] = df_train_id[\"anglez_nanexist\"].astype(np.float32)\n    df_train_id[\"anglez_daystd\"] = np.clip(df_train_id[\"anglez_daystd\"] / 90, 0, 2).astype(np.float32)\n    df_train_id[\"anglez_daymean\"] = np.clip(df_train_id[\"anglez_daymean\"] / 90, 0, 2).astype(np.float32)\n    df_train_id[\"anglez_daycounter\"] = (df_train_id[\"anglez_daycounter\"] / df_train_id[\"anglez_daycounter\"].max()).astype(np.float32)\n    df_train_id[\"anglez_nancounter\"] = (df_train_id[\"anglez_nancounter\"]>2).astype(np.float32)\n    max_daystd = np.max(df_train_id[\"anglez_daystd\"].values)\n    max_daymean = np.max(df_train_id[\"anglez_daymean\"].values)\n    if max_daystd > 10:\n        print(\"max_daystd > 10\", max_daystd)\n    if max_daymean > 10:\n        print(\"max_daymean > 10\", max_daymean)\n        \n\n    df_train_id[\"enmo\"] = np.clip(np.log1p(df_train_id[\"enmo\"]), 0, 5).astype(np.float32)\n\n    if \"target\" in df_train_id.columns:\n        df_train_id[\"target\"] = df_train_id[\"target\"].astype(np.float32)\n    return df_train_id\n\ndef make_dataset(df_train_id, id_no, df_target_id=None):\n    if df_target_id is None:\n        make_train_data = False\n    else:\n        make_train_data = True\n    file_path = os.path.join(cfg.preprocess_dir, f\"id_{id_no}_feature.npy\")\n    file_step_path = os.path.join(cfg.preprocess_dir, f\"id_{id_no}_step.npy\")\n    file_path_meta = os.path.join(cfg.preprocess_dir, f\"id_{id_no}_meta.json\")\n    train_columns = ['daily_step', 'anglez', 'anglez_simpleerror', 'anglez_simpleerror_span', \n    'anglez_nanexist', 'anglez_daystd', 'anglez_daymean', \"anglez_daycounter\", \"anglez_nancounter\", \n    'enmo']\n    # if make_train_data:\n    #     train_columns += ['target']\n    \n    # with timer(f\"select_dataset {id_no}\"):\n    #     df_train_id, df_target_id = sample_train_target_by_id(df_train, df_target, id_no)\n    # with timer(f\"timestamp_to_step {id_no}\"):\n    # df_train_id = timestamp_to_step(df_train_id)\n    # df_target_id = timestamp_to_step(df_target_id)\n    df_train_id, df_target_id = timestamp_to_step_single_id(df_train_id, df_target_id)\n    # with timer(f\"find_sensor_error {id_no}\"):\n    df_train_id = find_sensor_error(df_train_id, column='anglez')\n    if make_train_data:\n        # with timer(f\"make_binary_target {id_no}\"):\n        df_train_id, end_step_for_train = make_binary_target(df_train_id, df_target_id, id_no)\n        meta_data = {\"end_step_for_train\": end_step_for_train}\n\n    df_train_id = prerprocess_inputs_for_mlmodel(df_train_id)\n\n    # numpyに変換して保存する\n    np.save(file_path, df_train_id[train_columns].values)\n    np.save(file_step_path, df_train_id[\"step\"].values)\n    if make_train_data:\n        with open(file_path_meta, 'w') as f:\n            json.dump(meta_data, f, indent=4)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:42:31.496887Z","iopub.execute_input":"2023-12-13T14:42:31.497318Z","iopub.status.idle":"2023-12-13T14:42:31.560330Z","shell.execute_reply.started":"2023-12-13T14:42:31.497278Z","shell.execute_reply":"2023-12-13T14:42:31.559133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport os\nimport gc\n\nimport pandas as pd\nimport numpy as np\n\n\ndef timestamp_to_step(df):\n    \"\"\"\n    Convert timestamp to step\n    timestepは2018-08-14T22:26:00-0400といった形で与えられる\n    \"\"\"\n    df['date'] = df['timestamp'].str.split('T', expand=True)[0]\n    df['time'] = df['timestamp'].str.split('T', expand=True)[1].str.split('-', expand=True)[0]\n    df['timestamp_dt'] = pd.to_datetime(df['date']+' '+df['time'])\n    df['daily_step'] = df['timestamp_dt'].dt.hour * 60 * 12 + df['timestamp_dt'].dt.minute * 12 + df['timestamp_dt'].dt.second / 5\n    return df\n\n\ndef data_preparation(load_path, load_path_target=None, save_dir = \"cacheTMP/\"):\n    os.makedirs(save_dir, exist_ok=True)\n\n    # series_idsの代わり目のindexを取得し、それをもとにparquetのファイルを分割して読み込み、処理をおこなう\n    # df = pd.read_parquet(load_path) -> これはOOM\n    series_ids = pd.read_parquet(load_path, columns=[\"series_id\"])[\"series_id\"].values\n    unique_ids, unique_indices = np.unique(series_ids, return_index=True)\n    unique_indices = np.sort(unique_indices)\n    sorted_ids = series_ids[unique_indices]\n    start_indices = unique_indices\n    end_indices = np.append(unique_indices[1:], len(series_ids))\n    del series_ids\n    \n    if load_path_target is None:\n        df_target = None\n    else:\n        df_target = pd.read_csv(cfg.train_target_path)\n\n    load_step = 5\n    starts = np.arange(0, len(sorted_ids), load_step)\n    ends = np.append(starts[1:], len(sorted_ids))\n    for start, end in zip(starts, ends):\n        chunk_series_ids = sorted_ids[start:end]\n        chunk_start_ind = start_indices[start:end] - start_indices[start]\n        chunk_end_ind = end_indices[start:end] - start_indices[start]        \n        print(f\"start: {start}, end: {end}\")\n        filters = [(\"series_id\", \"in\", chunk_series_ids)]\n        chunk = pd.read_parquet(load_path, filters=filters)\n        for start_index, end_index, series_id in zip(chunk_start_ind, chunk_end_ind, chunk_series_ids):\n            chunk_sub = chunk.iloc[start_index:end_index].copy()\n            make_dataset(chunk_sub, series_id, df_target) # npyで保存\n            # chunk_sub.to_parquet(os.path.join(save_dir, f\"{series_id}.parquet\"), index=False)\n        if cfg.is_train and cfg.is_debug:\n            if end>=cfg.data_for_debug:\n                break\n    del chunk\n    del chunk_sub\n    del end_indices, start_indices, sorted_ids, unique_indices, unique_ids\n    gc.collect()\n    \n","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:42:31.562154Z","iopub.execute_input":"2023-12-13T14:42:31.562586Z","iopub.status.idle":"2023-12-13T14:42:31.582924Z","shell.execute_reply.started":"2023-12-13T14:42:31.562548Z","shell.execute_reply":"2023-12-13T14:42:31.581572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport glob\n\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\n\nSRC_PATH = os.path.join(cfg.src_path, 'src')\nsys.path.insert(1, SRC_PATH)\n\nfrom train_1dcnn import SleepAwake, set_seeds\n    \ndef build_model(weight_file, length, model_type=\"normal\"):\n    K.clear_session()\n    \n    \n    model_params = {\"input_shape\": (length, 10),\n                    \"output_shape\": (length, 1),  \n                    \"weight_file\": weight_file,\n                    \"is_train_model\": False,\n                    \"model_type\":model_type}\n    sa_model = SleepAwake(**model_params)\n    return sa_model\n\n    \n\ndef infer_predict_twin(load_dir, save_dir_0, save_dir_1, overlap=0.9):\n    show_memory_usage(\"before model load\")\n    set_seeds(111)\n\n    weight_files_0 = [os.path.join(cfg.src_path, f\"trained_model/exp00_run_00_SplitStem5foldSEED111controledStride_fold{i}/final_weights.h5\") for i in range(5)]\n    weight_files_0 += [os.path.join(cfg.src_path, f\"trained_model/exp00_run_00_SplitStem5foldSEED42controledStride_fold{i}/final_weights.h5\") for i in range(5)]\n\n    weight_files_1 = [os.path.join(cfg.src_path, f\"trained_model/exp00_run_01_SplitStem5foldSEED111normal_fold{i}/final_weights.h5\") for i in range(5)]\n    weight_files_1 += [os.path.join(cfg.src_path, f\"trained_model/exp00_run_01_SplitStem5foldSEED42normal_fold{i}/final_weights.h5\") for i in range(5)]\n\n    \n    sa_models_0 = [build_model(weight_file=w, length=2880 * 5, model_type=\"contstride\") for w in weight_files_0]\n    num_ensemble_0 = len(sa_models_0)\n\n    sa_models_1 = [build_model(weight_file=w, length=1024 * 14, model_type=\"normal\") for w in weight_files_1]\n    num_ensemble_1 = len(sa_models_1)\n\n    npy_files = sorted(glob.glob(os.path.join(load_dir, \"*feature.npy\")))\n    npy_files_step = sorted(glob.glob(os.path.join(load_dir, \"*step.npy\")))\n    show_memory_usage(\"before inference\")\n    if not os.path.exists(save_dir_0):\n        os.makedirs(save_dir_0)\n    if not os.path.exists(save_dir_1):\n        os.makedirs(save_dir_1)\n    for i, (f, fs) in enumerate(zip(npy_files, npy_files_step)):\n        show_memory_usage(f\"inference {i}\")\n\n        stt = time.time()\n        features = np.load(f)# [:end_step]\n        steps = np.load(fs)# [:end_step]\n        inputs = features[:,:] # features[:,:-1] if is_train else features[:,:]\n        daily_step = features[:, 0]\n        pred_nan = features[:, 2]\n        nan_span = features[:, 3]\n        nan_counter = features[:, -4]\n        \n        # 2/4 models        \n        # preds, preds_switch = sa_models[0].overlap_predict(inputs, overlap=overlap) # oom\n        preds, preds_switch = sa_models_0[0].overlap_predict_no_oom(inputs, overlap=overlap,  max_batch = cfg.pred_batch)\n        for m in sa_models_0[1:]:\n            preds_ensemble, preds_switch_ensemble = m.overlap_predict_no_oom(inputs, overlap=overlap, max_batch=cfg.pred_batch)\n            preds += preds_ensemble\n            preds_switch += preds_switch_ensemble\n        preds /= num_ensemble_0\n        preds_switch /= num_ensemble_0\n        \n        # print(\"pred time\", time.time()-stt)\n        stt = time.time()\n\n        # parquetで保存。csvよりも軽いはやい\n        save_path = save_dir_0 + os.path.basename(f).replace(\".npy\", \".parquet\")\n        # df = pd.DataFrame({\"step\": steps, \"daily_step\": daily_step, \"pred_awake\": preds.flatten(), \"pred_switch\": preds_switch.flatten(), \"pred_nan\": pred_nan})\n        STRIDE = 1\n        df = pd.DataFrame({\"step\": steps[:len(preds_switch)*STRIDE][::STRIDE],\n                                \"daily_step\": daily_step[:len(preds_switch)*STRIDE][::STRIDE], \n                                \"pred_awake\": preds.flatten().astype(np.float32), \n                                \"pred_switch10p\": preds_switch[:,0].astype(np.float32), \n                                \"pred_switch10\": preds_switch[:,1].astype(np.float32), \n                                \"pred_switch8\": preds_switch[:,2].astype(np.float32),\n                                \"pred_switch6\": preds_switch[:,3].astype(np.float32),\n                                \"pred_switch4\": preds_switch[:,4].astype(np.float32),\n                                \"pred_switch2\": preds_switch[:,5].astype(np.float32),\n                                \"pred_nan\": pred_nan[:len(preds_switch)*STRIDE][::STRIDE].astype(np.float32),\n                                \"pred_nan_span\": nan_span[:len(preds_switch)*STRIDE][::STRIDE].astype(np.float32),\n                                \"pred_nan_counter\": nan_counter[:len(preds_switch)*STRIDE][::STRIDE].astype(np.float32),\n                                })\n        df.to_parquet(save_path, index=False)\n        \n        del df, preds, preds_switch, preds_ensemble, preds_switch_ensemble\n        \n        # 2/4 models\n        preds, preds_switch = sa_models_1[0].overlap_predict_no_oom(inputs, overlap=overlap,  max_batch = 32)\n        for m in sa_models_1[1:]:\n            preds_ensemble, preds_switch_ensemble = m.overlap_predict_no_oom(inputs, overlap=overlap, max_batch=32)\n            preds += preds_ensemble\n            preds_switch += preds_switch_ensemble\n        preds /= num_ensemble_1\n        preds_switch /= num_ensemble_1\n        \n        # print(\"pred time\", time.time()-stt)\n        stt = time.time()\n\n        save_path = save_dir_1 + os.path.basename(f).replace(\".npy\", \".parquet\")\n        STRIDE = 1\n        df = pd.DataFrame({\"step\": steps[:len(preds_switch)*STRIDE][::STRIDE],\n                                \"daily_step\": daily_step[:len(preds_switch)*STRIDE][::STRIDE], \n                                \"pred_awake\": preds.flatten().astype(np.float32), \n                                \"pred_switch10p\": preds_switch[:,0].astype(np.float32), \n                                \"pred_switch10\": preds_switch[:,1].astype(np.float32), \n                                \"pred_switch8\": preds_switch[:,2].astype(np.float32),\n                                \"pred_switch6\": preds_switch[:,3].astype(np.float32),\n                                \"pred_switch4\": preds_switch[:,4].astype(np.float32),\n                                \"pred_switch2\": preds_switch[:,5].astype(np.float32),\n                                \"pred_nan\": pred_nan[:len(preds_switch)*STRIDE][::STRIDE].astype(np.float32),\n                                \"pred_nan_span\": nan_span[:len(preds_switch)*STRIDE][::STRIDE].astype(np.float32),\n                                \"pred_nan_counter\": nan_counter[:len(preds_switch)*STRIDE][::STRIDE].astype(np.float32),\n                                })\n        df.to_parquet(save_path, index=False)\n        \n        del df, preds, preds_switch, preds_ensemble, preds_switch_ensemble\n        del features, steps, inputs, daily_step, pred_nan\n        if i%2==0:\n            gc.collect()\n    del sa_models_0\n    del sa_models_1\n\n            ","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:46:35.506841Z","iopub.execute_input":"2023-12-13T14:46:35.507274Z","iopub.status.idle":"2023-12-13T14:46:35.538732Z","shell.execute_reply.started":"2023-12-13T14:46:35.507245Z","shell.execute_reply":"2023-12-13T14:46:35.537867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## POSTPROCESS","metadata":{}},{"cell_type":"code","source":"import glob\nimport os\nimport time\n\nimport tensorflow as tf\nimport scipy\nimport numpy as np\nimport pandas as pd\n\n\n\ndef detect_peak(array, valid_mask, kernel_size=5, threshold = 0.05):\n    peak_mask, peak_array = detect_peak_kmat(array.astype(np.float64), int(kernel_size))\n    pred_mask = peak_mask * valid_mask\n    peak_array = peak_array * valid_mask\n    peak_indices = np.where(peak_array>0)[0]\n    peak_val = peak_array[peak_indices]\n    mask_over_threshold = peak_val > threshold\n    peak_indices = peak_indices[mask_over_threshold]\n    peak_val = peak_val[mask_over_threshold]\n    peak_results = {\"peak_array\": peak_array, \"peak_indices\": peak_indices, \"peak_val\": peak_val}\n    return peak_results\n\ndef gaussian_smooth(array, sigma):#\n    if sigma<=0:\n        return array\n    array = scipy.ndimage.gaussian_filter1d(array, sigma, mode='reflect')\n    return array\n\n\ndef smooth_awake(pred_awake, gauss_factor):\n    \"\"\"\n    gaussian smooth\n    \"\"\"\n    \n    return smooth_pred_awake\n\ndef find_event(event_indices, pred_awake, before_length, after_length, awake_threshold=0.5, sleep_threshold=0.5, awake_sleep_diff_threshold=0, lengthlist_for_2ndstage=None, pred_nan=None, pred_scores=None):\n    \"\"\"\n    event stepの前後の状態の平均値をとり、それが閾値を超えているかどうかで状態変化を判定する\n    TODO 平均じゃないほうがいいかも？\n    \"\"\"\n    before_states = []\n    after_states = []\n    event_types = []\n    for event_index in event_indices:\n        before_state = pred_awake[max(0, event_index-before_length):event_index].mean()\n        after_state = pred_awake[event_index+1:min(event_index+after_length+1, len(pred_awake))].mean()\n        before_states.append(before_state)\n        after_states.append(after_state)\n        if before_state > after_state + awake_sleep_diff_threshold:\n            if before_state > awake_threshold and after_state < sleep_threshold:\n                event_type = \"onset\"\n            else:\n                event_type = \"nan\"\n        elif after_state > before_state + awake_sleep_diff_threshold:\n            if before_state < sleep_threshold and after_state > awake_threshold:\n                event_type = \"wakeup\"\n            else:\n                event_type = \"nan\"\n        else:\n            event_type = \"nan\"\n        event_types.append(event_type)\n    event_results = {\"event_indices\": event_indices, \"before_states\": before_states, \"after_states\": after_states, \"event_types\": event_types}\n\n\n    if lengthlist_for_2ndstage is not None:\n        for length in lengthlist_for_2ndstage:\n            event_results[f\"before_states_feat_{length}\"] = [pred_awake[max(0, event_index-length):event_index].mean() for event_index in event_indices]\n            event_results[f\"after_states_feat_{length}\"] = [pred_awake[event_index+1:min(event_index+length+1, len(pred_awake))].mean() for event_index in event_indices]\n            if pred_nan is not None:\n                event_results[f\"before_nan_feat_{length}\"] = [pred_nan[max(0, event_index-length):event_index].mean() for event_index in event_indices]\n                event_results[f\"after_nan_feat_{length}\"] = [pred_nan[event_index+1:min(event_index+length+1, len(pred_awake))].mean() for event_index in event_indices]\n            if pred_scores is not None:\n                for i, col in enumerate(['pred_switch', \"pred_switch10p\", \"pred_switch10\", \"pred_switch8\", \"pred_switch6\", \"pred_switch4\", \"pred_switch2\"]):\n                    # event_results[f\"before_{col}_{length//2}\"] = [pred_scores[max(0, event_index-length//2):event_index,i].mean() for event_index in event_indices]\n                    # event_results[f\"after_{col}_{length//2}\"] = [pred_scores[event_index+1:min(event_index+length//2+1, len(pred_awake)),i].mean() for event_index in event_indices]\n                    event_results[f\"befaf_{col}_{length//2}\"] = [pred_scores[max(0, event_index-length//2):event_index,i].mean()-pred_scores[event_index+1:min(event_index+length//2+1, len(pred_awake)),i].mean() for event_index in event_indices]\n \n    return event_results\n\ndef add_night_group(out_df_single):\n    # 10000step (abou 2 pm) -> night += 1\n    out_df_single[\"offset_step\"] = out_df_single['step'] + out_df_single['daily_step'].iloc[0] * cfg.step_for_a_day - out_df_single['step'].iloc[0]\n    out_df_single[\"night\"] = 1 + (out_df_single[\"offset_step\"] + cfg.step_for_a_day - 10000) // cfg.step_for_a_day\n    return out_df_single\n\ndef add_night_features(out_df_single, columns=[\"pred_nan_counter\", \"pred_nan_span\", \"pred_nan\"]):\n    \"\"\"\n    groupby night features for 2nd stage\n    \"\"\"\n    out_df_single = add_night_group(out_df_single)\n    for col in columns:\n        out_df_single[f\"{col}_mean\"] = out_df_single.groupby(\"night\")[col].transform(\"mean\")\n        out_df_single[f\"{col}_min\"] = out_df_single.groupby(\"night\")[col].transform(\"min\")\n        out_df_single[f\"{col}_max\"] = out_df_single.groupby(\"night\")[col].transform(\"max\")\n    # aggregationで実施してからconcatする場合\n    # df_agg = out_df_single.groupby(\"night\")[columns].agg([\"mean\", \"min\", \"max\"])\n    # df_agg.columns = [f\"{col}_{agg}\" for col, agg in df_agg.columns]\n    # df_agg = df_agg.reset_index()\n    # out_df_single = pd.merge(out_df_single, df_agg, on=\"night\", how=\"left\")\n    \n    out_df_single = out_df_single.drop(columns=[\"offset_step\", \"night\"])\n    return out_df_single\n\n\ndef avoid_6_vals(out_df_single, pred_switch, window_size=20):\n    \"\"\"\n    6の倍数のところは評価指標上僅かに損をする。(GTが1分単位であり、評価指標が12または6の倍数であるため)\n    予測結果のstepが6の倍数の場合は、そこからどちらかにずらすと得をする。\n    \"\"\"\n    # out_df_single['step'] = out_df_single['step'] + ((out_df_single['step']%6) == 0).astype(int)\n    out_df_single.reset_index(drop=True, inplace=True)\n    out_df_single[\"is_6_multiple\"] = (out_df_single['step']%6) == 0\n    shifts = np.zeros(len(out_df_single))\n    for i in range(len(out_df_single)-1):\n        if out_df_single['is_6_multiple'].iloc[i] == True:\n            index = out_df_single['peak_indices'].iloc[i]\n            before = pred_switch[max(0, index-window_size):index].mean()\n            after = pred_switch[index+1:min(index+window_size+1, len(pred_switch))].mean()\n            if before > after:\n                # out_df_single['step'].iloc[i] -= 1\n                # out_df_single.at[i, 'step']. -= 1\n                shifts[i] = -1\n            else:\n                # out_df_single['step'].iloc[i] += 1\n                # out_df_single.at[i, 'step']. += 1\n                shifts[i] = 1\n    out_df_single['step'] = out_df_single['step'] + shifts.astype(int)\n\n    # drop column\n    out_df_single = out_df_single.drop(columns=[\"is_6_multiple\"])\n    return out_df_single\n\n\ndef predict_averaging(predictions, weights):\n    \n    pred = (predictions * np.array(weights).reshape(1,-1)).sum(axis=1)# / np.sum(weights)\n\n    return pred\n\n\ndef make_dataset_for_second_model(df, df_target=None, peak_kernel_size=30, peak_threshold=0.05, awake_threshold=0.5, awake_sleep_diff_threshold=0, sleep_threshold=0.5, event_before_length=30, event_after_length=30, prior_conf_dev=0.1, averaging_weight=[0,1,0.5,0,0,0]):\n\n    lengthlist_for_2ndstage = [12, 24, 60, 120, 240, 360, 720] # used to make features of state\n    night_agg_cols = [\"pred_nan_counter\", \"pred_nan_span\", \"pred_nan\", \"pred_switch2\", \"pred_switch\", \"pred_awake\"]\n    \n    df[\"pred_switch\"] = predict_averaging(df[[\"pred_switch10p\", \"pred_switch10\", \"pred_switch8\", \"pred_switch6\", \"pred_switch4\", \"pred_switch2\"]].values, weights=averaging_weight)\n    df[\"pred_switch\"] = gaussian_smooth(df[\"pred_switch\"].values, sigma=4)\n    df = add_night_features(df, columns=night_agg_cols)\n    nonnan_mask = 1 - df[\"pred_nan\"].values\n    peak_results = detect_peak(df['pred_switch'].values, nonnan_mask, kernel_size=int(1+2*peak_kernel_size*cfg.step_for_1min), threshold=peak_threshold)\n    event_results = find_event(peak_results[\"peak_indices\"], df['pred_awake'].values, int(event_before_length*cfg.step_for_1min), int(event_after_length*cfg.step_for_1min), awake_threshold=awake_threshold, sleep_threshold=sleep_threshold, \n                                awake_sleep_diff_threshold=awake_sleep_diff_threshold, \n                                lengthlist_for_2ndstage=lengthlist_for_2ndstage,\n                                pred_nan=df['pred_nan'].values,\n                              pred_scores=df[['pred_switch', \"pred_switch10p\", \"pred_switch10\", \"pred_switch8\", \"pred_switch6\", \"pred_switch4\", \"pred_switch2\"]].values,)\n    out_df_single = pd.DataFrame({\"step\": df['step'].values[peak_results[\"peak_indices\"]], \n            \"peak_indices\": peak_results[\"peak_indices\"], \n            \"daily_step\": df['daily_step'].values[peak_results[\"peak_indices\"]], \n            \"event\": event_results['event_types'], \n            \"score\": peak_results['peak_val'],\n            \"score_10p\": df['pred_switch10p'].values[peak_results[\"peak_indices\"]],\n            \"score_10\": df['pred_switch10'].values[peak_results[\"peak_indices\"]],\n            \"score_8\": df['pred_switch8'].values[peak_results[\"peak_indices\"]],\n            \"score_6\": df['pred_switch6'].values[peak_results[\"peak_indices\"]],\n            \"score_4\": df['pred_switch4'].values[peak_results[\"peak_indices\"]],\n            \"score_2\": df['pred_switch2'].values[peak_results[\"peak_indices\"]],\n            \"pred_nan\": df['pred_nan'].values[peak_results[\"peak_indices\"]],\n            \"pred_nan_counter\": df['pred_nan_counter'].values[peak_results[\"peak_indices\"]],\n            \"pred_nan_span\": df['pred_nan_span'].values[peak_results[\"peak_indices\"]],\n            \"pred_awake\": df['pred_awake'].values[peak_results[\"peak_indices\"]],\n            \"before_states\": event_results['before_states'],\n            \"after_states\": event_results['after_states'],\n            })\n    for length in lengthlist_for_2ndstage:\n        out_df_single[f\"before_states_feat_{length}\"] = event_results[f\"before_states_feat_{length}\"]\n        out_df_single[f\"after_states_feat_{length}\"] = event_results[f\"after_states_feat_{length}\"]\n        out_df_single[f\"before_nan_feat_{length}\"] = event_results[f\"before_nan_feat_{length}\"]\n        out_df_single[f\"after_nan_feat_{length}\"] = event_results[f\"after_nan_feat_{length}\"]\n        for i, col in enumerate(['pred_switch', \"pred_switch10p\", \"pred_switch10\", \"pred_switch8\", \"pred_switch6\", \"pred_switch4\", \"pred_switch2\"]):\n            out_df_single[f\"befaf_{col}_{length//2}\"] = event_results[f\"befaf_{col}_{length//2}\"]\n    for col in night_agg_cols:\n        out_df_single[f\"{col}_mean\"] = df[f\"{col}_mean\"].values[peak_results[\"peak_indices\"]]\n        out_df_single[f\"{col}_min\"] = df[f\"{col}_min\"].values[peak_results[\"peak_indices\"]]\n        out_df_single[f\"{col}_max\"] = df[f\"{col}_max\"].values[peak_results[\"peak_indices\"]]\n                    \n    out_df_single = out_df_single[out_df_single['event']!=\"nan\"]\n    if len(out_df_single) > 0:\n        out_df_single = add_night_group(out_df_single)\n\n    if len(out_df_single) > 0:\n        out_df_single = avoid_6_vals(out_df_single, df['pred_switch'].values)\n    \n    # add_target(not used in inference)\n    def diff_step_to_AP(diff_step):\n        return np.sum(np.array([diff_step < threshold for threshold in [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]]), axis=0)\n    if df_target is not None:\n        if len(df_target) >0:\n            diff_step_matrix = df_target['step'].values.reshape(-1,1) - out_df_single['step'].values.reshape(1,-1)\n            diff_step_matrix = np.abs(diff_step_matrix)\n            min_diff_step = np.min(diff_step_matrix, axis=0)\n            out_df_single['min_diff_step'] = min_diff_step\n            out_df_single['best_ap'] = diff_step_to_AP(min_diff_step)/ 10.\n        else:\n            out_df_single['min_diff_step'] = np.nan\n            out_df_single['best_ap'] = 0\n\n    return out_df_single\n\n\ndef run_postprocess_and_prepare_2nd(pred_dir, pp_params, save_path=\"df_second_model.feather\"):\n    pred_files = glob.glob(os.path.join(pred_dir, \"*.parquet\"))\n    out_df = []\n    for file in pred_files:\n        id_no = os.path.basename(file).split(\"_\")[1]\n        df_pred_id = pd.read_parquet(file)\n        out_df_single = make_dataset_for_second_model(df_pred_id, **pp_params)\n        \n        out_df_single['series_id'] = id_no\n        out_df.append(out_df_single)\n    # out_df = pd.concat(out_df).reset_index(drop=True).reset_index().rename(columns={\"index\": \"row_id\"})\n    \n    out_df = pd.concat(out_df).reset_index(drop=True)\n    float32_cols = [col for col in out_df.columns if out_df[col].dtype==\"float64\"]\n    out_df[float32_cols] = out_df[float32_cols].astype(np.float32)\n    out_df.to_feather(save_path)\n    return out_df\n\n\ndef evaluate_ap(out_df, train_events):\n    # out_df = pd.concat(out_df).reset_index(drop=True).reset_index().rename(columns={\"index\": \"row_id\"})\n    \n    series_ids = out_df['series_id'].unique()\n    solution =  train_events.loc[train_events['series_id'].isin(series_ids)].copy()\n    solution = solution[~np.isnan(solution['step'])]\n    total_score = event_detection_ap(solution, out_df.copy(), tolerances)\n    return total_score","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:46:37.113567Z","iopub.execute_input":"2023-12-13T14:46:37.114091Z","iopub.status.idle":"2023-12-13T14:46:37.188187Z","shell.execute_reply.started":"2023-12-13T14:46:37.114046Z","shell.execute_reply":"2023-12-13T14:46:37.187249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def make_features_2ndstage(df, cfg, phase=\"train\"):\n\n\n    drop_cols = []\n    added_cols = []\n    score_keys = ['score', 'score_10p', 'score_10', 'score_8', 'score_6', 'score_4', 'score_2']\n    \n    df[\"event\"] = (df[\"event\"]==\"wakeup\").astype(int)\n    # daily_step for night\n    df[\"daily_step_sleep\"] = (df[\"daily_step\"] + 0.5) % 1\n\n    # change of state between before and after\n    lengthlist = [12, 24, 60, 120, 240, 360, 720]\n    for length in lengthlist:\n        df[f\"state_diff_{length}\"] = df[f\"before_states_feat_{length}\"] - df[f\"after_states_feat_{length}\"]\n        df[f\"nan_diff_{length}\"] = df[f\"before_nan_feat_{length}\"] - df[f\"after_nan_feat_{length}\"]\n\n    # largest score\n    for key in score_keys:\n        df[f\"max_{key}_sne\"] = df.groupby([\"series_id\", \"night\", \"event\"])[key].transform(\"max\")\n        df[f\"max_{key}_sne_diff\"] = df[f\"max_{key}_sne\"] - df[key]\n        df[f\"max_{key}_sne_is_peak\"] = (df[f\"max_{key}_sne_diff\"] == 0).astype(int)\n        df[f\"sum_{key}_sne\"] = df.groupby([\"series_id\", \"night\", \"event\"])[key].transform(\"sum\")\n        df[f\"mean_{key}_sne\"] = df.groupby([\"series_id\", \"night\", \"event\"])[key].transform(\"mean\")\n        drop_cols.append(f\"max_{key}_sne_is_peak\")\n        added_cols += [f\"max_{key}_sne\", f\"max_{key}_sne_diff\", f\"max_{key}_sne_is_peak\"]\n\n    # largest score\n    for key in score_keys:\n        df[f\"max_{key}_sn\"] = df.groupby([\"series_id\", \"night\"])[key].transform(\"max\")\n        df[f\"max_{key}_sn_diff\"] = df[f\"max_{key}_sn\"] - df[key]\n        added_cols += [f\"max_{key}_sn\", f\"max_{key}_sn_diff\"]\n    \n    for key in score_keys:\n        df_peak = df[df[f\"max_{key}_sne_is_peak\"] == 1]\n        \n        df_peak = df_peak.groupby([\"series_id\", \"event\"])[f\"max_{key}_sne\"].agg([\"mean\", \"std\"]).reset_index()\n        df_peak.columns = [\"series_id\", \"event\", f\"max_{key}_sne_mean\", f\"max_{key}_sne_std\"]\n        df = df.merge(df_peak, on=[\"series_id\", \"event\"], how=\"left\")\n        # normalize\n        df[f\"{key}_relative_to_peak\"] = (df[key] - df[f\"max_{key}_sne_mean\"]) / df[f\"max_{key}_sne_std\"]\n\n        added_cols += [f\"max_{key}_sne_mean\", f\"max_{key}_sne_std\", f\"{key}_relative_to_peak\"]\n\n    # daily_step at peak\n    for key in score_keys:\n        df_peak = df[df[f\"max_{key}_sne_is_peak\"] == 1].copy()\n        df_peak = df_peak.rename(columns={\"daily_step\": f\"peak_daily_step_{key}\", \"daily_step_sleep\": f\"peak_daily_step_sleep_{key}\"})\n        df_peak[f\"peak_daily_step_{key}_mean\"] = df_peak.groupby([\"series_id\", \"event\"])[f\"peak_daily_step_{key}\"].transform(\"mean\")\n        df_peak[f\"peak_daily_step_sleep_{key}_mean\"] = df_peak.groupby([\"series_id\", \"event\"])[f\"peak_daily_step_sleep_{key}\"].transform(\"mean\") # scoreが高いものだけに限定してもいいのかもな…。\n\n        df = df.merge(df_peak[[\"series_id\", \"night\", \"event\", f\"peak_daily_step_{key}\", f\"peak_daily_step_sleep_{key}\", f\"peak_daily_step_{key}_mean\", f\"peak_daily_step_sleep_{key}_mean\"]], on=[\"series_id\", \"night\", \"event\"], how=\"left\")\n        df[f\"step_dist_from_peak_{key}\"] = df[\"daily_step\"] - df[f\"peak_daily_step_{key}\"]\n        df[f\"step_dist_from_peak_sleep_{key}\"] = df[\"daily_step_sleep\"] - df[f\"peak_daily_step_sleep_{key}\"]\n        df[f\"step_dist_from_peak_{key}_mean\"] = df[\"daily_step\"] - df[f\"peak_daily_step_{key}_mean\"]\n        df[f\"step_dist_from_peak_sleep_{key}_mean\"] = df[\"daily_step_sleep\"] - df[f\"peak_daily_step_sleep_{key}_mean\"]\n\n        added_cols += [f\"peak_daily_step_{key}\", f\"peak_daily_step_sleep_{key}\", f\"peak_daily_step_{key}_mean\", f\"peak_daily_step_sleep_{key}_mean\", f\"step_dist_from_peak_{key}\", f\"step_dist_from_peak_sleep_{key}\", f\"step_dist_from_peak_{key}_mean\", f\"step_dist_from_peak_sleep_{key}_mean\"]\n        \n\n    # other event at same night\n    df_flip = df.copy()\n    flip_columns = [f\"max_{key}_sne\" for key in score_keys]+ [f\"sum_{key}_sne\" for key in score_keys] + [f\"peak_daily_step_{key}\" for key in score_keys] + [f\"peak_daily_step_sleep_{key}\" for key in score_keys]\n    df_flip[\"event\"] = 1 - df_flip[\"event\"] #.apply(lambda x: \"onset\" if x == \"wakeup\" else \"wakeup\")\n    df_flip = df_flip.groupby([\"series_id\", \"event\", \"night\"])[flip_columns].max().reset_index()\n    df_flip.columns = [\"series_id\", \"event\", \"night\"] + [f\"{c}_flip\" for c in flip_columns]\n    df = df.merge(df_flip, on=[\"series_id\", \"night\", \"event\"], how=\"left\")\n\n    for key in score_keys:\n        df[f\"peak_daily_step_{key}_sleep_duration_01\"] = df[\"daily_step\"] + 0.5 - df[f\"peak_daily_step_sleep_{key}_flip\"]\n        df[f\"peak_daily_step_{key}_sleep_duration_10\"] = df[f\"peak_daily_step_{key}_flip\"] + 0.5 - df[\"daily_step_sleep\"]\n        added_cols += [f\"peak_daily_step_{key}_sleep_duration_01\", f\"peak_daily_step_{key}_sleep_duration_10\"]\n\n    for key in ['score', 'score_10p', 'score_10', 'score_8', 'score_6', 'score_4', 'score_2']:\n        df[f\"rank_{key}_sne\"] = df.groupby([\"series_id\", \"night\", \"event\"])[key].transform(\"rank\")\n\n    # sort by scoreでsort -> accumulated score\n    df = df.sort_values([\"series_id\", \"night\", \"event\", \"score\"], ascending=False).reset_index(drop=True)\n    for key in ['score', 'score_10p', 'score_10', 'score_8', 'score_6', 'score_4', 'score_2']:\n        df[f\"cumsum_{key}_sne\"] = df.groupby([\"series_id\", \"night\", \"event\"])[key].transform(\"cumsum\")\n        # 累積の積ももとめる。\n        # df[f\"tmp\"] = 1 - df[key]\n        # df[f\"cumprod_{key}_sne\"] = df.groupby([\"series_id\", \"night\", \"event\"])[\"tmp\"].transform(\"cumprod\")\n        # drop_cols.append(\"tmp\")\n\n        # df[f\"cumsum_{key}_sne_rate\"] = df[key] / df[f\"cumsum_{key}_sne\"]\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:46:37.190290Z","iopub.execute_input":"2023-12-13T14:46:37.190674Z","iopub.status.idle":"2023-12-13T14:46:37.222276Z","shell.execute_reply.started":"2023-12-13T14:46:37.190609Z","shell.execute_reply":"2023-12-13T14:46:37.221157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LabelEncoders:\n    def __init__(self):\n        self.encoders = {}\n    \n    def fit(self, df):\n        for c in df:\n            if df[c].dtype.name == \"object\":\n                enc = self.encoders.get(c, LabelEncoder())\n                enc.fit(df[c])\n                self.encoders[c] = enc\n                \n    def transform(self, df):\n        for c in df:\n            if c in self.encoders:\n                df[c] = self.encoders[c].transform(df[c])\n        return df\n    \n    def fit_one(self, s):\n        enc = self.encoders.get(s.name, LabelEncoder())\n        enc.fit(s)\n        self.encoders[s.name] = enc\n        \n    def transform_one(self, s):\n        if s.name in self.encoders:\n            return self.encoders[s.name].transform(s)\n        else:\n            return s\n\n    def fit_transform(self, df):\n        self.fit(df)\n        return self.transform(df)\n                \n    def fit_transform_one(self, s):\n        self.fit_one(s)\n        return self.transform_one(s)\n\nclass LGBMSerializer:\n    def __init__(self,\n                 booster: lgb.CVBooster,\n                 encoders: LabelEncoders,\n                 ):\n        self.booster = booster\n        self.encoders = encoders\n        \n    \n    def to_file(self, filename: str):\n        model = {\n            \"boosters\": [b.model_to_string() for b in self.booster.boosters],\n            \"best_iteration\": self.booster.best_iteration,\n            \n        }\n        \n        with open(f\"{filename}_model.json\", \"w\") as f:\n            json.dump(model, f)\n            \n        with open(f\"{filename}_encoder.bin\", \"wb\") as f:\n            pickle.dump(self.encoders, f)\n            \n    @classmethod\n    def from_file(cls, filename: str) -> \"TrainedModel\":\n        \n        with open(f\"{filename}_model.json\", \"r\") as f:\n            model = json.load(f)\n\n        cvbooster = lgb.CVBooster()\n        cvbooster.boosters = [lgb.Booster(model_str=b) for b in model[\"boosters\"]]\n        cvbooster.best_iteration = model[\"best_iteration\"]\n        for b in cvbooster.boosters:\n            b.best_iteration = cvbooster.best_iteration\n            \n        with open(f\"{filename}_encoder.bin\", \"rb\") as f:\n            encoders = pickle.load(f)\n            \n        return cls(cvbooster, encoders)\n            \n    \ndef run_2nd_stage(df, pretrained_dir):\n    # make features and predict \n    df = make_features_2ndstage(df, cfg, phase=\"test\")\n    serializer = LGBMSerializer.from_file(pretrained_dir)\n    cvbooster = serializer.booster\n    encoder = serializer.encoders\n    feature_cols = cvbooster.feature_name()[0]\n    predicted = np.array(cvbooster.predict(encoder.transform(df[feature_cols]))).mean(axis=0)\n    \n    # make submission\n    sub_df = df[[\"series_id\", \"step\", \"event\", \"score\"]].copy()\n    sub_df = sub_df.reset_index(drop=True).reset_index().rename(columns={\"index\": \"row_id\"})\n    # wakeupとonsetに書き換える\n    sub_df[\"event\"] = sub_df[\"event\"].apply(lambda x: \"onset\" if x == 0 else \"wakeup\")\n    sub_df[\"score\"] = ((sub_df[\"score\"]**0.8) + predicted) / 2\n    return sub_df\n\ndef run_2nd_stage_twin(df, pretrained_dir, pretrained_dir_sub, save_dir, exist_third_stage=False):\n    # make features and predict \n    df = make_features_2ndstage(df, cfg, phase=\"test\")\n    # sort by score\n    df = df.sort_values([\"series_id\", \"night\", \"event\", \"score\"], ascending=False).reset_index(drop=True)\n    \n    serializer = LGBMSerializer.from_file(pretrained_dir)\n    cvbooster = serializer.booster\n    encoder = serializer.encoders\n    feature_cols = cvbooster.feature_name()[0]\n    predicted = np.array(cvbooster.predict(encoder.transform(df[feature_cols]))).mean(axis=0)\n\n    serializer = LGBMSerializer.from_file(pretrained_dir_sub)\n    cvbooster = serializer.booster\n    encoder = serializer.encoders\n    feature_cols = cvbooster.feature_name()[0]\n    predicted_sub = np.array(cvbooster.predict(encoder.transform(df[feature_cols]))).mean(axis=0)\n    \n    \n    # make submission\n    if exist_third_stage:\n        sub_df = df.copy()\n    else:\n        sub_df = df[[\"series_id\", \"step\", \"event\", \"night\", \"score\"]].copy()\n    \n    \n    def postprocess_2nd_stage(df, oof_acm, oof_simple, weights=[1, 0.4, 1]):\n        print(\"need to be sorted by score in advance\")\n        df[\"origin_score\"] = df[\"score\"].copy()\n        df[\"event\"] = df[\"event\"].apply(lambda x: \"onset\" if x == 0 else \"wakeup\")\n        df[\"pred_best_ap_until\"] = oof_acm\n        df[\"pred_best_ap_until\"] = df.groupby([\"series_id\", \"night\", \"event\"])[\"pred_best_ap_until\"].transform(lambda x: np.maximum.accumulate(x)) # avoid minus diff\n        df[\"pred_score_acm\"] = df.groupby([\"series_id\", \"night\", \"event\"])[\"pred_best_ap_until\"].diff().fillna(df[\"pred_best_ap_until\"])\n        df[\"pred_score_simple\"] = oof_simple\n        df[\"score\"] = (df[\"pred_score_acm\"]*weights[0] + df[\"pred_score_simple\"]*weights[1]  + df[\"origin_score\"]*weights[2]) / sum(weights)\n        return df\n    \n    sub_df = postprocess_2nd_stage(sub_df, predicted, predicted_sub, weights=[1, 0.4, 1])\n    sub_df.to_feather(save_dir)\n    \n    if not exist_third_stage:\n        # sub_df = sub_df.drop(columns=[\"pred_score_simple\", \"pred_score_acm\", \"pred_best_ap_until\", \"origin_score\"])\n        # else:\n        sub_df = sub_df.drop(columns=[\"pred_score_simple\", \"pred_score_acm\", \"pred_best_ap_until\", \"origin_score\", \"night\"])\n        sub_df = sub_df.reset_index(drop=True).reset_index().rename(columns={\"index\": \"row_id\"})\n    return sub_df\n\ndef run_2nd_stage_twin_ensemble(df, pretrained_dirs, pretrained_dirs_sub, save_path, exist_third_stage=False):\n    # make features and predict \n    df = make_features_2ndstage(df, cfg, phase=\"test\")\n    # sort by score\n    df = df.sort_values([\"series_id\", \"night\", \"event\", \"score\"], ascending=False).reset_index(drop=True)\n    predicted = []\n    for pretrained_dir in pretrained_dirs:\n        serializer = LGBMSerializer.from_file(pretrained_dir)\n        cvbooster = serializer.booster\n        encoder = serializer.encoders\n        feature_cols = cvbooster.feature_name()[0]\n        predicted.append(np.array(cvbooster.predict(encoder.transform(df[feature_cols]))).mean(axis=0))\n    predicted = np.array(predicted).mean(axis=0)\n    \n    predicted_sub = []\n    for pretrained_dir_sub in pretrained_dirs_sub:\n        serializer = LGBMSerializer.from_file(pretrained_dir_sub)\n        cvbooster = serializer.booster\n        encoder = serializer.encoders\n        feature_cols = cvbooster.feature_name()[0]\n        predicted_sub.append(np.array(cvbooster.predict(encoder.transform(df[feature_cols]))).mean(axis=0))\n    predicted_sub = np.array(predicted_sub).mean(axis=0)\n\n    \n    # make submission\n    if exist_third_stage:\n        sub_df = df.copy()\n    else:\n        sub_df = df[[\"series_id\", \"step\", \"event\", \"night\", \"score\"]].copy()\n    \n    \n    def postprocess_2nd_stage(df, oof_acm, oof_simple, weights=[1, 0.4, 1]):\n        print(\"need to be sorted by score in advance\")\n        df[\"origin_score\"] = df[\"score\"].copy()\n        df[\"event\"] = df[\"event\"].apply(lambda x: \"onset\" if x == 0 else \"wakeup\")\n        df[\"pred_best_ap_until\"] = oof_acm\n        df[\"pred_best_ap_until\"] = df.groupby([\"series_id\", \"night\", \"event\"])[\"pred_best_ap_until\"].transform(lambda x: np.maximum.accumulate(x)) # avoid minus diff\n        df[\"pred_score_acm\"] = df.groupby([\"series_id\", \"night\", \"event\"])[\"pred_best_ap_until\"].diff().fillna(df[\"pred_best_ap_until\"])\n        df[\"pred_score_simple\"] = oof_simple\n        df[\"score\"] = (df[\"pred_score_acm\"]*weights[0] + df[\"pred_score_simple\"]*weights[1]  + df[\"origin_score\"]*weights[2]) / sum(weights)\n        return df\n    \n    sub_df = postprocess_2nd_stage(sub_df, predicted, predicted_sub, weights=[1, 0.4, 1])\n    sub_df.to_feather(save_path)\n    \n    if not exist_third_stage:\n        # sub_df = sub_df.drop(columns=[\"pred_score_simple\", \"pred_score_acm\", \"pred_best_ap_until\", \"origin_score\"])\n        # else:\n        sub_df = sub_df.drop(columns=[\"pred_score_simple\", \"pred_score_acm\", \"pred_best_ap_until\", \"origin_score\", \"night\"])\n        sub_df = sub_df.reset_index(drop=True).reset_index().rename(columns={\"index\": \"row_id\"})\n    return sub_df\n\n\n# ensemble after 2nd stage\ndef weighted_fusion_ensemble(df_0, df_1, distance_threshold=100):\n    weight_wo_fusion = 0.5\n    large_val = 1e8\n    series_ids = df_0['series_id'].unique()\n    out_df = []\n    for series_id in series_ids:\n        df_0_id = df_0[df_0['series_id']==series_id].copy()\n        df_1_id = df_1[df_1['series_id']==series_id].copy()\n        df_0_id = df_0_id.sort_values(\"score\", ascending=False).reset_index(drop=True)\n        df_1_id = df_1_id.sort_values(\"score\", ascending=False).reset_index(drop=True)\n        \n        steps_0 = df_0_id['step'].values.copy() # base\n        steps_1 = df_1_id['step'].values.copy()\n        scores_0 = df_0_id['score'].values.copy() # base\n        scores_1 = df_1_id['score'].values.copy()\n        not_assigned_df = []\n        for step, score in zip(steps_1, scores_1):\n            dists = np.abs(steps_0 - step)\n            argmin = np.argmin(dists)\n            min_dist = dists[argmin]\n            if min_dist < distance_threshold:\n                f_step = steps_0[argmin]\n                f_score = scores_0[argmin]\n                add_step = step\n                add_score = score\n                \n                # new_score = (f_score + add_score) / 2\n                new_score = (f_score * f_score + add_score * add_score) / (f_score + add_score)\n                new_step = (f_step * f_score + add_step * add_score) / (f_score + add_score)\n                df_0_id.loc[argmin, \"score\"] = new_score\n                df_0_id.loc[argmin, \"step\"] = new_step\n                steps_0[argmin] = large_val # large val to avoid assign again\n            else:\n                not_assigned = df_1_id[df_1_id['step']==step].copy()\n                not_assigned['score'] = score * weight_wo_fusion # not assigned\n                not_assigned_df.append(not_assigned)\n        df_0_id.loc[steps_0!=large_val, \"score\"] *= weight_wo_fusion # not assigned\n        out_df.append(df_0_id)\n        if len(not_assigned_df) >0:\n            not_assigned_df = pd.concat(not_assigned_df)\n            out_df.append(not_assigned_df)\n    out_df = pd.concat(out_df).reset_index(drop=True) # .reset_index() # .rename(columns={\"index\": \"row_id\"})\n    return out_df\n\ndef round_step(df):\n    df[\"step\"] = df[\"step\"].astype(int) + (df[\"step\"] % 6 < 1).astype(int)\n    return df\n\n\n\ndef run_3rd_stage(df_second, pretrained_dir, \n                  mins_offset = [2,4,8,16], #32],#[2, 3, 9, 30, 60, 1440],\n                 ):\n    def drop_duplicate_1minstep(df):\n        df[\"step_1min\"] = df[\"step\"] // cfg.step_for_1min\n        df = df.drop_duplicates([\"series_id\", \"event\", \"step_1min\"])\n        df = df.drop(columns=[\"step_1min\"])\n        return df\n    \n    def add_offset_events(pred_df):\n        \"\"\"\n        Use as many prediction as possible for Average Precision.\n        \"\"\"\n        pred_df[\"offset_from_original\"] = 0\n        pred_concat = [pred_df.copy()]\n        for offset in mins_offset:\n            pred_df_c = pred_df.copy()\n            pred_df_c[\"step\"] = pred_df_c[\"step\"] - cfg.step_for_1min * offset\n            pred_df_c[\"offset_from_original\"] = - offset\n            pred_concat.append(pred_df_c)\n            pred_df_c = pred_df.copy()\n            pred_df_c[\"step\"] = pred_df_c[\"step\"] + cfg.step_for_1min * offset\n            pred_df_c[\"offset_from_original\"] = offset\n            pred_concat.append(pred_df_c)\n        pred_concat = pd.concat(pred_concat, ignore_index=True)\n        # If some predictions are in the same minutes, keep the higher.\n        pred_concat[\"step_1min\"] = pred_concat[\"step\"] // cfg.step_for_1min\n        # pred_concat = pred_concat.sort_values([\"series_id\", \"event\", \"step_1min\", \"score\"], ascending=False)\n        pred_concat = pred_concat.drop_duplicates([\"series_id\", \"event\", \"step_1min\"])\n        pred_concat = pred_concat.drop(columns=[\"step_1min\"])\n        # pred_concat = pred_concat.drop(columns=[\"step_1min\", \"row_id\"])\n        # pred_concat = pred_concat.reset_index().rename(columns={\"index\": \"row_id\"})\n        return pred_concat\n    \n    def add_large_offset_events(pred_df):\n        mins_offset = [-1440,-60,60,1440]\n        pred_df[\"offset_from_original\"] = 0\n        pred_concat = [pred_df.copy()]\n        for offset in mins_offset:\n            pred_df_c = pred_df.copy()\n            pred_df_c[\"step\"] = pred_df_c[\"step\"] + cfg.step_for_1min * offset\n            pred_df_c[\"score\"] *= 0.0001 # low score\n            pred_df_c[\"offset_from_original\"] = offset\n            pred_concat.append(pred_df_c)\n        pred_concat = pd.concat(pred_concat, ignore_index=True)\n        # If some predictions are in the same minutes, keep the higher.\n        pred_concat[\"step_1min\"] = pred_concat[\"step\"] // cfg.step_for_1min\n        pred_concat = pred_concat.drop_duplicates([\"series_id\", \"event\", \"step_1min\"])\n        pred_concat = pred_concat.drop(columns=[\"step_1min\"])\n        return pred_concat\n    \n\n    def add_feat_third_stage_ensemble(df, list_pred_files):\n        \n        num_ensemble = len(list_pred_files)\n\n        out_df = []\n        for files in zip(*list_pred_files):\n            df_pred_id = pd.read_parquet(files[0])\n            cols = [c for c in df_pred_id.columns if \"pred_switch\" in c] + [\"pred_awake\"]\n            for pf in files[1:]:\n                df_pred_id_1 = pd.read_parquet(pf)    \n                for c in cols:\n                    df_pred_id[c] += df_pred_id_1[c]\n            for c in cols:\n                df_pred_id[c] /= num_ensemble\n            cols = df_pred_id.columns\n            new_cols = [col + \"_third\" for col in cols]\n            series_id = files[0].split(\"id_\")[-1].split(\"_\")[0]\n            df_series = df[df[\"series_id\"] == series_id].copy()\n            steps = df_series[\"step\"].values.astype(int)\n            steps = np.clip(steps, 0, len(df_pred_id)-1)\n            df_series[new_cols] = df_pred_id.values[steps]\n            out_df.append(df_series)\n        out_df = pd.concat(out_df, ignore_index=True)\n        return out_df      \n\n    \n    df_third = add_offset_events(df_second)\n    df_third = df_third[df_third[\"offset_from_original\"] != 0]\n    \n    pred_files_1st_stage = [glob.glob(os.path.join(cfg.pred_dirs[i], \"*.parquet\")) for i in range(len(cfg.pred_dirs))]\n    df_third = add_feat_third_stage_ensemble(df_third, pred_files_1st_stage)\n    \n    serializer = LGBMSerializer.from_file(pretrained_dir)\n    cvbooster = serializer.booster\n    encoder = serializer.encoders\n    feature_cols = cvbooster.feature_name()[0]\n    predicted = np.array(cvbooster.predict(encoder.transform(df_third[feature_cols]))).mean(axis=0)\n    # df_third[\"score\"] = df_third[\"pred_switch10_third\"] * 0.085 + predicted * 0.110\n    print(\"changed weights of 3rd stage\")\n    df_third[\"score\"] = predicted * 0.40\n    df_third = df_third[df_third[\"score\"]>0.0008]\n    df_third = pd.concat([add_large_offset_events(df_second), \n                          df_third], ignore_index=True)[[\"series_id\", \"step\", \"event\", \"score\"]]\n    df_third = df_third.drop_duplicates([\"series_id\", \"event\", \"step\"], keep=\"first\")\n    df_third = df_third.sort_values([\"series_id\", \"step\"])\n    df_third = df_third.reset_index(drop=True).reset_index().rename(columns={\"index\": \"row_id\"})\n    return df_third\n\ndef remove_out_of_range(df, min_step=0):\n    return df[df[\"step\"] > min_step].drop(columns=[\"row_id\"]).reset_index(drop=True).reset_index().rename(columns={\"index\": \"row_id\"})\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:46:37.299107Z","iopub.execute_input":"2023-12-13T14:46:37.299478Z","iopub.status.idle":"2023-12-13T14:46:37.373861Z","shell.execute_reply.started":"2023-12-13T14:46:37.299448Z","shell.execute_reply":"2023-12-13T14:46:37.372665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pandas.api.types\nfrom typing import Dict, List, Tuple\n\n# tolerances in steps\ntolerances = {\n    \"onset\":  [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n    \"wakeup\": [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n}\nseries_id_column_name = \"series_id\"\ntime_column_name = \"step\"\nevent_column_name = \"event\"\nscore_column_name = \"score\"\nuse_scoring_intervals = None\n\ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n        series_id_column_name: str,\n        time_column_name: str,\n        event_column_name: str,\n        score_column_name: str,\n        use_scoring_intervals: bool = False,\n) -> float:\n    \n    # Validate metric parameters\n    assert len(tolerances) > 0, \"Events must have defined tolerances.\"\n    assert set(tolerances.keys()) == set(solution[event_column_name]).difference({'start', 'end'}),\\\n        (f\"Solution column {event_column_name} must contain the same events \"\n         \"as defined in tolerances.\")\n    assert pd.api.types.is_numeric_dtype(solution[time_column_name]),\\\n        f\"Solution column {time_column_name} must be of numeric type.\"\n\n    # Validate submission format\n    for column_name in [\n        series_id_column_name,\n        time_column_name,\n        event_column_name,\n        score_column_name,\n    ]:\n        if column_name not in submission.columns:\n            raise ParticipantVisibleError(f\"Submission must have column '{target_name}'.\")\n\n    if not pd.api.types.is_numeric_dtype(submission[time_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{time_column_name}' must be of numeric type.\"\n        )\n    if not pd.api.types.is_numeric_dtype(submission[score_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{score_column_name}' must be of numeric type.\"\n        )\n\n    # Set these globally to avoid passing around a bunch of arguments\n    globals()['series_id_column_name'] = series_id_column_name\n    globals()['time_column_name'] = time_column_name\n    globals()['event_column_name'] = event_column_name\n    globals()['score_column_name'] = score_column_name\n    globals()['use_scoring_intervals'] = use_scoring_intervals\n\n    return event_detection_ap(solution, submission, tolerances)\n\ndef event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n) -> float:\n\n    # Ensure solution and submission are sorted properly\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution\n            .query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index='interval',\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, ['start', 'end']]\n            .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths[series_id_column_name].unique()],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    return mean_ap\n\ndef match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\"\"\"\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted\n\ndef precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n\ndef average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:46:37.376422Z","iopub.execute_input":"2023-12-13T14:46:37.376930Z","iopub.status.idle":"2023-12-13T14:46:37.414900Z","shell.execute_reply.started":"2023-12-13T14:46:37.376890Z","shell.execute_reply":"2023-12-13T14:46:37.413683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_many_events(pred_df):\n    \"\"\"\n    Use as many prediction as possible for Average Precision.\n    \"\"\"\n    \n    mins_offset = [3, 9, 30, 2, 60, 1440]\n    score_rate = [0.01, 0.008, 0.005, 0.002, 0.0001, 0.00001]\n\n    pred_concat = [pred_df.copy()]\n    for offset, ratio in zip(mins_offset, score_rate):\n        pred_df_c = pred_df.copy()\n        pred_df_c[\"step\"] = pred_df_c[\"step\"] - cfg.step_for_1min * offset\n        pred_df_c[\"score\"] = pred_df_c[\"score\"] * ratio\n        pred_concat.append(pred_df_c)\n        pred_df_c = pred_df.copy()\n        pred_df_c[\"step\"] = pred_df_c[\"step\"] + cfg.step_for_1min * offset\n        pred_df_c[\"score\"] = pred_df_c[\"score\"] * ratio\n        pred_concat.append(pred_df_c)\n    pred_concat = pd.concat(pred_concat, ignore_index=True)\n\n    print(pred_concat.shape)\n    # If some predictions are in the same minutes, keep the higher.\n    pred_concat[\"step_1min\"] = pred_concat[\"step\"] // cfg.step_for_1min\n    pred_concat = pred_concat.sort_values([\"series_id\", \"event\", \"step_1min\", \"score\"], ascending=False)\n    pred_concat = pred_concat.drop_duplicates([\"series_id\", \"event\", \"step_1min\"])\n    pred_concat = pred_concat.drop(columns=[\"step_1min\", \"row_id\"])\n    pred_concat = pred_concat.reset_index().rename(columns={\"index\": \"row_id\"})\n    print(pred_concat.shape)\n    return pred_concat\n\n\ndef submit_sample():\n    sample = pd.read_csv(cfg.sample_submission_path)\n    sample.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:46:37.416744Z","iopub.execute_input":"2023-12-13T14:46:37.417086Z","iopub.status.idle":"2023-12-13T14:46:37.430337Z","shell.execute_reply.started":"2023-12-13T14:46:37.417060Z","shell.execute_reply":"2023-12-13T14:46:37.429376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\npp_params = {'awake_threshold': 0.01118579240197301, \n             'sleep_threshold': 0.9741809981418406, \n             'awake_sleep_diff_threshold': 0.0036734165080717115, \n             'peak_kernel_size': 4.689964359639259, \n             'peak_threshold': 0.05135826117540523, \n             'event_before_length': 68.38372582053368, \n             'event_after_length': 99.69200455879107, \n             'prior_conf_dev': 0.29102092096947463,\n             \"averaging_weight\": [0.333, 0.333, 0.333, 0.00, 0, 0]}\nuse_third_stage = True\n\n\nif not cfg.is_rerun and cfg.is_train==False:\n    print(\"submit sample\")\n    #submit_sample()\n    out_df = pd.read_csv(cfg.sample_submission_path)\nelse:\n    \n    show_memory_usage(\"before data preparation\")\n    data_preparation(cfg.train_path if cfg.is_train else cfg.test_path, \n                     load_path_target=None, \n                     save_dir = cfg.preprocess_dir)\n    show_memory_usage(\"aft data preparation 1\")\n    gc.collect()\n    show_memory_usage(\"aft data preparation 2\")\n    infer_predict_twin(load_dir=cfg.preprocess_dir, save_dir_0=cfg.pred_dirs[0], save_dir_1=cfg.pred_dirs[1], overlap=cfg.pred_overlap)\n    show_memory_usage(\"aft NN infer\")\n    # preprocess_dirを捨てる\n    shutil.rmtree(cfg.preprocess_dir)\n\n    gc.collect()\n    show_memory_usage(\"aft NN infer 2\")\n    \n    for i in range(2):\n        out_df = run_postprocess_and_prepare_2nd(cfg.pred_dirs[i], pp_params, save_path=\"df_second_model.feather\")\n        show_memory_usage(\"aft postprocess 1\")\n        gc.collect()\n        show_memory_usage(\"aft postprocess 2\")\n        out_df = run_2nd_stage_twin_ensemble(out_df, pretrained_dirs=cfg.lgbm_paths_2nd[i], pretrained_dirs_sub=cfg.lgbm_sub_paths_2nd[i], save_path=f\"df_second_model_{i}.feather\", exist_third_stage=use_third_stage)\n        show_memory_usage(\"aft 2nd stage 1\")\n        del out_df\n        gc.collect()\n    \n    df_2nd_0 = pd.read_feather(f\"df_second_model_0.feather\")\n    df_2nd_1 = pd.read_feather(f\"df_second_model_1.feather\")\n\n    df_2nd_wbf = weighted_fusion_ensemble(df_2nd_0, df_2nd_1, distance_threshold=100)\n    df_2nd_wbf = round_step(df_2nd_wbf)\n    print(df_2nd_wbf.shape, df_2nd_0.shape, df_2nd_1.shape)\n    \n    if cfg.is_train:\n        train_events = pd.read_csv(cfg.train_target_path)\n        print(evaluate_ap(df_2nd_0[[\"series_id\", \"step\", \"event\", \"score\"]].sort_values([\"series_id\", \"step\"]).copy(), train_events))\n        print(evaluate_ap(df_2nd_1[[\"series_id\", \"step\", \"event\", \"score\"]].sort_values([\"series_id\", \"step\"]).copy(), train_events))\n        print(evaluate_ap(df_2nd_wbf[[\"series_id\", \"step\", \"event\", \"score\"]].sort_values([\"series_id\", \"step\"]).copy(), train_events))\n\n    show_memory_usage(\"aft 2nd stage 2\")\n    if use_third_stage:\n        out_df = run_3rd_stage(df_2nd_wbf, pretrained_dir=cfg.lgbm_path_third)\n    else:\n        out_df = add_many_events(df_2nd_wbf)\n    out_df = remove_out_of_range(out_df)\n    \n    \n    show_memory_usage(\"aft add event 1\")\n    gc.collect()\n    show_memory_usage(\"aft add event 2\")\n    if cfg.is_train:\n        train_events = pd.read_csv(cfg.train_target_path)\n        print(evaluate_ap(out_df, train_events))\n\n    shutil.rmtree(cfg.pred_dirs[0])\n    shutil.rmtree(cfg.pred_dirs[1])\n    \nprint(out_df.shape)\ndisplay(out_df.head())\nsub_columns = [\"row_id\", \"series_id\", \"step\", \"event\", \"score\"]\nout_df[\"step\"] = out_df[\"step\"].astype(int)\nout_df[sub_columns].to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-13T14:46:37.431982Z","iopub.execute_input":"2023-12-13T14:46:37.432344Z","iopub.status.idle":"2023-12-13T15:04:04.571371Z","shell.execute_reply.started":"2023-12-13T14:46:37.432311Z","shell.execute_reply":"2023-12-13T15:04:04.570479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}