{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7123142,"sourceType":"datasetVersion","datasetId":3897339}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"###### General Imports\n##########################\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport multiprocessing as mp\nimport lightgbm as lgb\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow.parquet as pq\nimport pyarrow as pa \n\n## Turn off warnings\nimport os, warnings, joblib, shutil\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-12-04T17:47:32.618991Z","iopub.execute_input":"2023-12-04T17:47:32.619353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pandas.api.types\nfrom typing import Dict, List, Tuple\n\n\n##### Eval Func\n##########################\n\n# tolerances in steps\nall_steps_init = [1,3,6, 12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\nall_steps = [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n\ntolerances = {\n    \"onset\":  all_steps,\n    \"wakeup\": all_steps,\n}\n\nseries_id_column_name = \"series_id\"\ntime_column_name = \"step\"\nevent_column_name = \"event\"\nscore_column_name = \"score\"\nuse_scoring_intervals = None\n\ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        use_scoring_intervals: bool = False,\n) -> float:\n    \n    tolerances = {\n    \"onset\":  all_steps,\n    \"wakeup\": all_steps,\n    }\n\n    series_id_column_name = \"series_id\"\n    time_column_name = \"step\"\n    event_column_name = \"event\"\n    score_column_name = \"score\"\n    use_scoring_intervals = None\n\n    # Validate metric parameters\n    assert len(tolerances) > 0, \"Events must have defined tolerances.\"\n    assert set(tolerances.keys()) == set(solution[event_column_name]).difference({'start', 'end'}),\\\n        (f\"Solution column {event_column_name} must contain the same events \"\n         \"as defined in tolerances.\")\n    assert pd.api.types.is_numeric_dtype(solution[time_column_name]),\\\n        f\"Solution column {time_column_name} must be of numeric type.\"\n\n    # Validate submission format\n    for column_name in [\n        series_id_column_name,\n        time_column_name,\n        event_column_name,\n        score_column_name,\n    ]:\n        if column_name not in submission.columns:\n            raise ParticipantVisibleError(f\"Submission must have column '{target_name}'.\")\n\n    if not pd.api.types.is_numeric_dtype(submission[time_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{time_column_name}' must be of numeric type.\"\n        )\n    if not pd.api.types.is_numeric_dtype(submission[score_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{score_column_name}' must be of numeric type.\"\n        )\n\n    # Set these globally to avoid passing around a bunch of arguments\n    globals()['series_id_column_name'] = series_id_column_name\n    globals()['time_column_name'] = time_column_name\n    globals()['event_column_name'] = event_column_name\n    globals()['score_column_name'] = score_column_name\n    globals()['use_scoring_intervals'] = use_scoring_intervals\n\n    return event_detection_ap(solution, submission, tolerances)\n\ndef event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n) -> float:\n\n    # Ensure solution and submission are sorted properly\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution\n            .query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index='interval',\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, ['start', 'end']]\n            .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths[series_id_column_name].unique()],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    return mean_ap\n\ndef match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\"\"\"\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted\n\ndef precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n\ndef average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##### Data Vars\n##########################\nNUM_CORES   = mp.cpu_count()\n\nTEST_PATH   = 'raw/test_series.parquet'\nTRAIN_PATH  = 'raw/train_series.parquet'\nLABELS_PATH = 'raw/train_events.csv'\n\nEXTRACT        = False\nLOCAL_RUN      = False\nLIMITED_RUN    = False\n\nINFERENCE      = True\nIS_KAGGLE      = True\nMAKE_TRANSFORM = True\nDROP_DATA      = True\n\nVER       = '201'\n\nif IS_KAGGLE:\n    TEST_PATH   = '/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet'\n    TRAIN_PATH  = '/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet'\n    LABELS_PATH = '/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv'\n\nMAIN_FILE = TEST_PATH\nif LOCAL_RUN:\n    MAIN_FILE = TRAIN_PATH\n\nprint('Num Cores:', NUM_CORES)\nprint('Train:', TRAIN_PATH)\nprint('Test: ', TEST_PATH)\nprint('Main: ', MAIN_FILE)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"########################### Create Dirs\n###########################################################\nfor cdir in ['raw','data','models','predictions','data/labels', 'data/series','data/fe_k','data/labels_k','data/fe_f','data/fe_k_base','data/split', 'raw/aux']:\n    \n    if os.path.exists(cdir) and DROP_DATA:\n        shutil.rmtree(cdir)\n        \n    if not os.path.exists(cdir):\n        os.makedirs(cdir)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"########################### Unzip\n##############################################\nif EXTRACT:\n    shutil.unpack_archive('data.zip', 'raw')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"########################### All IDx\n###########################################################\nALL_IDX = pd.read_parquet(MAIN_FILE, columns=['series_id'])['series_id'].unique()\n\nif LIMITED_RUN:\n    ALL_IDX = ALL_IDX[:10]\n    \nIDX_MAP = {k:v for v,k in enumerate(ALL_IDX)}\nIDX_MAP","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"########################### Utils\n###########################################################\ndef load_and_split_data(data_path):\n    \n    parquet_schema = pa.schema(\n        [\n            (\"series_id\", pa.string()),\n            (\"step\", pa.int32()),\n            (\"timestamp\", pa.string()),\n            (\"anglez\", pa.float32()),\n            (\"enmo\", pa.float32()),\n        ]\n    )\n\n    parquet_file = ParquetFile(data_path)\n    data = parquet_file.read().to_pandas()\n    grouped_data = data.groupby('series_id')\n\n    for series_id, group in tqdm(grouped_data):\n        table = pa.Table.from_pandas(group, schema=parquet_schema)\n        pq.ParquetWriter((f'data/split/{series_id}.parquet'), parquet_schema).write_table(table)\n        \ndef make_base(cur_id):\n    \n    new_id = IDX_MAP[cur_id]\n    \n    file_path = f'data/split/{cur_id}.parquet'\n    df = pd.read_parquet(file_path)\n    \n    ## Minify Series Id\n    df['series_id'] = new_id\n    df['series_id'] = df['series_id'].astype(np.int16)\n    \n    ## Conforme sorting\n    df = df.sort_values(by=['step'])\n    df = df.reset_index(drop=True)\n    \n    ## Summer time check !TODO test set may have over shifts - easy to check by probing\n    df['is_summer_time'] = df['timestamp'].fillna('-').str.contains('-0400').astype(np.int8)\n    \n    ## Keep Local time\n    df['timestamp'] = pd.to_datetime(df['timestamp'].apply(lambda x: str(x)[:-5]))\n    \n    ## Save\n    df.to_pickle(f'data/series/{cur_id}.pkl')\n    os.remove(file_path) \n    \n    ## Create labels if it's a train transform\n    if LOCAL_RUN:\n        df = pd.read_csv(LABELS_PATH)\n        df = df[df['series_id']==cur_id]\n        df['series_id'] = new_id\n        df['series_id'] = df['series_id'].astype(np.int16)\n        df['night']     = df['night'].astype(np.int16)\n        df['timestamp'] = pd.to_datetime(df['timestamp'].apply(lambda x: str(x)[:-5]))\n        df = df.sort_values(by=['night','timestamp'])\n        df = df.reset_index(drop=True)\n        df.to_pickle(f'data/labels/{cur_id}.pkl')","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##### Base Loop\n##########################\n\nif MAKE_TRANSFORM:\n    ##\n    load_and_split_data(MAIN_FILE)\n    print('Parquet splitted')\n\n    ##\n    pool = mp.Pool(NUM_CORES)\n    pool.map(make_base, ALL_IDX)\n    print('Base transform done')\n    pool.close()","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##### Utils 2\n##########################\n\ndef base_transform(cur_id):\n    \n    ##########################\n    # Load Data\n    ########################## \n    df = pd.read_pickle(f'data/series/{cur_id}.pkl')\n    if LOCAL_RUN:\n        labels_df = pd.read_pickle(f'data/labels/{cur_id}.pkl')\n    \n    ##########################\n    # Target\n    ##########################  \n    if LOCAL_RUN:\n        labels_df = labels_df[~labels_df['step'].isna()]\n        labels_df['step'] = labels_df['step']\n        del labels_df['timestamp']\n\n        # Merge\n        df = df.merge(labels_df[['step','event']], on=['step'], how='left')\n\n        # Binary Targets\n        df['target_o'] = (df['event']=='onset').astype(np.int8)\n        df['target_w'] = (df['event']=='wakeup').astype(np.int8)\n        del df['event']\n        \n        df['grp_t'] = df[['target_o','target_w']].sum(axis=1)\n        df['grp_t'] = df['grp_t'].cumsum()\n        df['grp_t'] = (df['grp_t']%2!=0).astype(int)\n\n        for i in [2, 6, 12, 36,  60, 90, 120, 150, 180, 240, 300, 360]: \n            df[f'target_o_x{i}'] = (df['target_o'].rolling((i*2+1),1, center=True).max()).astype(np.int8)\n            df[f'target_w_x{i}'] = (df['target_w'].rolling((i*2+1),1, center=True).max()).astype(np.int8)\n            \n        \n    ##########################\n    # Pure Hour\n    ##########################   \n    temp_df = df[['series_id','step']]\n    temp_df = temp_df[((df['timestamp'].dt.minute==0)&(df['timestamp'].dt.hour==16)).cumsum()>0]\n    \n    df['normailzed_step'] = df['step'] - temp_df['step'].min()\n    df['normailzed__hour'] = df['normailzed_step'] // (12 * 60)\n    \n    df['dt_month']    = df['timestamp'].dt.month\n    \n    df['dt_minute']   = (df['timestamp'].dt.minute).astype(np.int8)\n    \n    df['dt_hour']     = df['normailzed__hour'] % 24\n    df['dt_day']      = df['normailzed__hour']//24\n    df['dt_weekday']  = (df['timestamp'].dt.dayofweek).astype(np.int8)\n    \n    ## Todo - change it\n    df['dt_step']     = df.groupby(['normailzed__hour'])['normailzed__hour'].transform('cumcount')\n    df['dt_step']     = np.where(df['dt_day']<=0, df['dt_step'] + (719 - df.groupby(['normailzed__hour'])['dt_step'].transform('max')), df['dt_step'])\n        \n    df['dt_hour_time'] = df['dt_hour']*60 + df['dt_minute']\n\n    \n    del df['normailzed__hour'], df['normailzed_step']\n    \n    ##########################\n    # New cols\n    ##########################  \n    temp_anglez = df['anglez'].values.copy()\n    df['anglez'] = np.abs(df['anglez'])\n    \n\n    ##########################\n    # Checks by groups\n    ##########################    \n    df['fe__check_a'] = (df.groupby(['dt_hour','dt_step','anglez'])['dt_step'].transform('count')>1).astype(int)\n    df['group'] = ((df['fe__check_a'] - df['fe__check_a'].shift()).abs() > 0).cumsum()\n    df['group_len'] = df.groupby(['group'])['step'].transform(lambda x: len(x))\n    \n    ###\n    df['fe__check_c'] = df['anglez'].rolling(21,21, center=True).std().rolling(360,360).quantile(0.2).shift(-180)\n    df['fe__check_c'] /= df['fe__check_c'].max()\n    df['fe__check_c'] = df['fe__check_c'].clip(0.1, None)\n    df['group_c'] = ((df['fe__check_c'] - df['fe__check_c'].shift()).abs() > 0).cumsum()\n    df['group_len_c'] = df.groupby(['group_c'])['step'].transform(lambda x: len(x))\n\n    df['fe__check_c'] = ((df['fe__check_c']==0.1)&(df['group_len']>720)).astype(int)\n    df['group_c'] = ((df['fe__check_c'] - df['fe__check_c'].shift()).abs() > 0).cumsum()\n\n    df['fe__check_cc'] = df['fe__check_c'].rolling(241,241, center=True).max().fillna(0).astype(np.int8)\n    df['group_c'] = ((df['fe__check_cc'] - df['fe__check_cc'].shift()).abs() > 0).cumsum()\n    df['group_len_c'] = df.groupby(['group_c'])['step'].transform(lambda x: len(x))\n\n    \n    \n    ##########################\n    # Checks by groups V2\n    ##########################    \n   \n    ## Noise\n    df['noise_check'] = (df.groupby(['dt_hour','dt_step','anglez'])['dt_step'].transform('count')>1).astype(int)\n    df['noise_group'] = ((df['noise_check'] - df['noise_check'].shift()).abs() > 0).cumsum()\n    df['noise_group__len'] = df.groupby(['noise_group'])['step'].transform(lambda x: len(x))\n   \n    df['clean_noise'] = ((df['noise_check']>0)&(df['noise_group__len']>=1440)).astype(np.int8)\n    df['clean_noise_group'] = ((df['clean_noise'] - df['clean_noise'].shift()).abs() > 0).cumsum()\n    df['clean_noise_group__len'] = df.groupby(['clean_noise_group'])['step'].transform(lambda x: len(x))\n    df['clean_noise'] = df['clean_noise'].rolling(360*6 + 1, 360*6 + 1, center=True).min().fillna(0).astype(np.int8)\n\n    if df.tail(12 * 60 * 6)['clean_noise'].mean()>0.5:\n        df['clean_noise'] = np.where(df['clean_noise_group'] == df['clean_noise_group'].max(), 1, df['clean_noise'])\n   \n    ## Periods\n    threshold = 0.2\n    df['periods']  = df['anglez'].rolling(21,21, center=True).std().rolling(361,361).quantile(0.2).shift(-180)\n    df['periods'] /= df['periods'].max()\n    df['periods']  = df['periods'].clip(threshold, None)\n    df['periods']  = (df['periods']==threshold).astype(int)\n    df['periods_group'] = ((df['periods'] - df['periods'].shift()).abs() > 0).cumsum()\n    df['periods_group__len'] = df.groupby(['periods_group'])['step'].transform(lambda x: len(x))\n   \n    ## Smoothing\n    df['smooth_periods'] = ((df['periods']==1)&(df['periods_group__len']>=360)).astype(int)\n    df['smooth_group'] = ((df['smooth_periods'] - df['smooth_periods'].shift()).abs() > 0).cumsum()\n    df['smooth_group__len'] = df.groupby(['smooth_group'])['step'].transform(lambda x: len(x))\n    df['smooth_group__len'] = df['smooth_group__len'].clip(None,8640)\n   \n    ## Biggest group\n    df['biggest_periods'] = (df['smooth_group__len']>=df.groupby(['dt_day'])['smooth_group__len'].transform('max') / 4).astype(int)\n    df['biggest_periods'] = ((df['biggest_periods']==1)&(df['smooth_periods']==1)).astype(int)\n    df['biggest_periods'] = df['biggest_periods'].rolling(360*8 + 1, 360*8 + 1, center=True).max().fillna(0).astype(np.int8)\n   \n    df['biggest_group'] = ((df['biggest_periods'] - df['biggest_periods'].shift()).abs() > 0).cumsum()\n    df['biggest_group__len'] = df.groupby(['biggest_group'])['step'].transform(lambda x: len(x))\n       \n    df['biggest_periods'] = (df['biggest_periods']==1)&(df.groupby(['biggest_group'])['clean_noise'].transform('mean')<1)\n    df['biggest_periods'] = df['biggest_periods'].astype(np.int8)\n   \n    ##\n    df['fixed_periods'] = (df['biggest_periods'].shift(17280) + df['biggest_periods'].shift(-17280))>=2\n    df['fixed_periods'] = df['fixed_periods'].astype(int)\n    df['fixed_periods'] = np.where(df['biggest_group__len']>=12000, df['fixed_periods'], df['biggest_periods'])\n    df['fixed_periods_group'] = ((df['fixed_periods'] - df['fixed_periods'].shift()).abs() > 0).cumsum()\n    df['fixed_periods_group__len'] = df.groupby(['fixed_periods_group'])['step'].transform(lambda x: len(x))\n           \n    \n    ##########################\n    # Diff\n    ##########################\n    df['anglez'] = temp_anglez\n    #df['anglez__diff'] = np.abs(df['anglez'] - df['anglez'].rolling(5,1, center=True).mean())\n    #df['anglez__diff_2'] = np.round(df['anglez']).rolling(61,1).apply(lambda x: len(set(x)))\n \n\n    ##########################\n    # Drop cols\n    ##########################\n    #if LOCAL_RUN:\n    #    df['temp_anglez'] = df['anglez']\n    #    df['temp_enmo']   = df['enmo']\n    \n    drop_cols = ['timestamp','is_summer_time','c_anglez']\n    for col in drop_cols:\n        if col in list(df):\n            del df[col]\n         \n    ##########################\n    # Minify\n    ##########################\n    for col in list(df):\n        if str(df[col].dtype)[:3] == 'int':\n            if df[col].max()<127 and df[col].min()>-128:\n                df[col] = df[col].astype(np.int8)\n            elif df[col].max()<32767 and df[col].min()>-32768:\n                df[col] = df[col].astype(np.int16)\n            else:\n                df[col] = df[col].astype(np.int32)\n\n    ##########################\n    # Save\n    ##########################            \n    df.to_pickle(f'data/fe_k_base/{cur_id}.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if MAKE_TRANSFORM:\n    pool = mp.Pool(NUM_CORES)\n    pool.map(base_transform, ALL_IDX)\n    print('Base transform done')\n    pool.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##### Features\n##########################\n\ndef make_feats(cur_id):\n    \n    ##########################\n    # Load Data\n    ########################## \n    df = pd.read_pickle(f'data/fe_k_base/{cur_id}.pkl')\n \n    ##########################\n    # Feats\n    ########################## \n    \n    ## Abs\n    df['anglez'] = np.abs(df['anglez'])\n    df['anglez_r'] = np.floor(df['anglez'])\n\n    df['enmo_r'] = np.floor(df['enmo']*100)\n    \n    ## Diff\n    df['anglez__diff'] = np.abs(df['anglez'] - df['anglez'].rolling(5,1, center=True).mean())    \n    df['enmo__diff'] = np.abs(df['enmo'] - df['enmo'].rolling(5,1, center=True).mean())\n    \n    df['anglez_rr'] = np.floor(df['anglez__diff'])\n\n    ##########################\n    # Base\n    ##########################  \n    df['grp_step_x6'] = (df['dt_step']//6)*6    \n    df['grp_step_x12'] = (df['dt_step']//12)*12    \n    \n    aux_df = df[[\n         'dt_month',\n         'dt_minute',\n         'dt_weekday',\n         'step',\n         'dt_hour_time',\n         'fe__check_a',\n         'group',\n         'group_len',\n         'fe__check_c',\n         'group_c',\n         'group_len_c',\n         'fe__check_cc',\n         'noise_check',\n         'noise_group',\n         'noise_group__len',\n         'clean_noise',\n         'clean_noise_group',\n         'clean_noise_group__len',\n         'periods',\n         'periods_group',\n         'periods_group__len',\n         'smooth_periods',\n         'smooth_group',\n         'smooth_group__len',\n         'biggest_periods',\n         'biggest_group',\n         'biggest_group__len',\n         'fixed_periods',\n    ]]\n   \n    if LOCAL_RUN:\n        agg_func = {\n            'step': ['min'],\n            'dt_step': ['min'],\n\n            'target_o': ['max'],\n            'target_w': ['max'],\n            'target_o_x12': ['max'],\n            'target_w_x12': ['max'],\n            'target_o_x6': ['max'],\n            'target_w_x6': ['max'],\n        }\n    else:\n        agg_func = {\n        'step': ['min'],\n        'dt_step': ['min'],\n    }\n\n    df_base = df.groupby(['series_id','dt_day','dt_hour','grp_step_x6']).agg(agg_func)\n    df_base.columns = [c[0] for c in list(df_base)]\n    df_base = df_base.reset_index()\n\n    \n    \n    ##########################\n    # Feats 12x\n    ##########################     \n    agg_feats = {\n        'anglez': ['std'],\n        'anglez__diff': ['mean'],\n        'anglez_r': ['nunique'],\n        'anglez_rr': ['nunique'],\n        'enmo_r': ['nunique',],\n        'step': ['min'],\n    }    \n    feats_12x_df = df.groupby(['series_id','dt_day','dt_hour','grp_step_x12']).agg(agg_feats)\n    feats_12x_df.columns = ['__'.join(c) if c[0] not in ['step'] else c[0] for c in list(feats_12x_df)]\n    feats_12x_df = feats_12x_df.reset_index(drop=True)    \n    \n\n    ##########################\n    # Lags x12\n    ##########################  \n    for p_column in ['anglez__diff__mean','anglez_r__nunique','enmo_r__nunique']:     \n        for i in [2, 4, 12, 24, 30, 40, 60, 120, 240, 480, 180]:\n                        \n            feats_12x_df[f'lag__12x__{p_column}__roll_mean_{i}_c'] = feats_12x_df[p_column].rolling(i+1, i+1, center=True).mean()\n            feats_12x_df[f'lag__12x__{p_column}__roll_mean_{i}_l'] = feats_12x_df[p_column].rolling(i, i).mean().shift()\n            feats_12x_df[f'lag__12x__{p_column}__roll_mean_{i}_r'] = feats_12x_df[p_column].rolling(i, i).mean().shift(-i-1)\n        \n    ##        \n    feats_12x_df = feats_12x_df[['step']+[c for c in list(feats_12x_df) if ('lag__' in c) or ('qnt__' in c)]]       \n          \n        \n    ##########################\n    # Feats 6x\n    ##########################     \n    agg_feats = {\n        'anglez': ['std'],\n        'anglez__diff': ['mean'],\n        'enmo__diff': ['mean'],\n        'anglez_r': ['nunique'],\n        'anglez_rr': ['nunique'],\n        'enmo_r': ['nunique'],\n        'step': ['min'],\n    }    \n    feats_6x_df = df.groupby(['series_id','dt_day','dt_hour','grp_step_x6']).agg(agg_feats)\n    feats_6x_df.columns = ['__'.join(c) if c[0] not in ['step'] else c[0] for c in list(feats_6x_df)]\n    feats_6x_df = feats_6x_df.reset_index(drop=True)    \n    \n    ##\n    for i in range(1,7):\n        feats_6x_df[f'qnt__anglez_r__grp{i}'] = (feats_6x_df['anglez_rr__nunique']>i).astype(int)\n        feats_6x_df[f'qnt__anglez_r__grp{i}'] = ((feats_6x_df[f'qnt__anglez_r__grp{i}'] - feats_6x_df[f'qnt__anglez_r__grp{i}'].shift()).abs() > 0).cumsum()\n        \n        feats_6x_df[f'qnt__anglez_r__grp{i}_min'] = feats_6x_df['step'] - feats_6x_df.groupby([f'qnt__anglez_r__grp{i}'])['step'].transform('min')\n        feats_6x_df[f'qnt__anglez_r__grp{i}_max'] = feats_6x_df['step'] - feats_6x_df.groupby([f'qnt__anglez_r__grp{i}'])['step'].transform('max')\n    \n        feats_6x_df[f'qnt__anglez_r__grp{i}'] = feats_6x_df.groupby([f'qnt__anglez_r__grp{i}'])['step'].transform(lambda x: len(x))\n    \n    \n    ##########################\n    # Lags x6\n    ##########################  \n    for p_column in ['anglez__diff__mean', 'anglez_r__nunique','enmo_r__nunique']:    #enmo__diff__mean  \n        for i in [2, 4, 12, 24, 30, 40, 60, 120, 240, 480, 180]:\n            \n            i = i*2\n            \n            feats_6x_df[f'lag__6x__{p_column}__roll_mean_{i}_c'] = feats_6x_df[p_column].rolling(i+1, i+1, center=True).mean()\n            feats_6x_df[f'lag__6x__{p_column}__roll_mean_{i}_l'] = feats_6x_df[p_column].rolling(i, i).mean().shift()\n            feats_6x_df[f'lag__6x__{p_column}__roll_mean_{i}_r'] = feats_6x_df[p_column].rolling(i, i).mean().shift(-i-1)\n\n            if p_column in ['anglez__diff__mean']:\n                \n                feats_6x_df[f'lag__6x__{p_column}__roll_std_{i}_c'] = feats_6x_df[p_column].rolling(i+1, i+1, center=True).std()\n                feats_6x_df[f'lag__6x__{p_column}__roll_std_{i}_l'] = feats_6x_df[p_column].rolling(i, i).std().shift()\n                feats_6x_df[f'lag__6x__{p_column}__roll_std_{i}_r'] = feats_6x_df[p_column].rolling(i, i).std().shift(-i-1)  \n                \n                feats_6x_df[f'lag__6x__{p_column}__roll_mean_{i}_d_prev'] = feats_6x_df[f'lag__6x__{p_column}__roll_mean_{i}_c'].shift(1440)\n                feats_6x_df[f'lag__6x__{p_column}__roll_mean_{i}_d_next'] = feats_6x_df[f'lag__6x__{p_column}__roll_mean_{i}_c'].shift(-1440)\n    \n    ##        \n    feats_6x_df = feats_6x_df[['step']+[c for c in list(feats_6x_df) if ('lag__' in c) or ('qnt__' in c)]]       \n  \n\n    ##########################\n    # Merge\n    ########################## \n    df = df_base.merge(aux_df, on=['step'], how='left')\n    df = df.merge(feats_6x_df, on=['step'], how='left')\n    df = df.merge(feats_12x_df, on=['step'], how='left')\n    \n    \n    ##########################\n    # Group features\n    ##########################       \n    df['fe__group_step_min'] = df['step'] - df.groupby(['group'])['step'].transform('min')\n    df['fe__group_step_max'] = df['step'] - df.groupby(['group'])['step'].transform('max')    \n    \n    df['fe__group_step_min_c'] = df['step'] - df.groupby(['group_c'])['step'].transform('min')\n    df['fe__group_step_max_c'] = df['step'] - df.groupby(['group_c'])['step'].transform('max')    \n\n    df['fe__group_step_min_d'] = df['step'] - df.groupby(['biggest_group'])['step'].transform('min')\n    df['fe__group_step_max_d'] = df['step'] - df.groupby(['biggest_group'])['step'].transform('max')    \n\n    df['fe__group_step_min_e'] = df['step'] - df.groupby(['smooth_group'])['step'].transform('min')\n    df['fe__group_step_max_e'] = df['step'] - df.groupby(['smooth_group'])['step'].transform('max')    \n    \n    df['fe__group_step_min_f'] = df['step'] - df.groupby(['clean_noise_group'])['step'].transform('min')\n    df['fe__group_step_max_f'] = df['step'] - df.groupby(['clean_noise_group'])['step'].transform('max')    \n        \n    df['fe__day_group_max'] = df['group_len_c'] / df.groupby(['dt_day'])['group_len_c'].transform('max')\n\n    df['fe__day_group_num'] = df.groupby(['dt_day'])['group_c'].transform('nunique')\n    df['fe__day_group_nights'] = df.groupby(['group_c'])['dt_day'].transform('nunique')\n    \n    \n    ##########################\n    # Filter\n    ########################## \n    df['c'] = ((~((df['fe__check_a']==1)&(df['group_len']>20)))).astype(np.int8)\n    df = df[df['c']==1]\n\n    ##########################\n    # Minification\n    ##########################  \n    for col in list(df):\n        if ('lag__' in col):\n            df[col] = df[col].astype(np.float16)\n            \n    ##########################\n    # Export\n    ##########################  \n    #df.to_pickle(f'data/ver_{VER}/fe_s/{cur_id}.pkl')\n    return df\n  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def post_process(df, event_type='onset'):\n    df = df.copy()\n\n    df['c1'] = df.groupby(['series_id','dt_day'])['score'].transform('max')\n    df['c2'] = np.where(df['score']==df['c1'], df['step'], 0)\n    df['c2'] = df.groupby(['series_id','dt_day'])['c2'].transform('max')\n    df['c3'] = np.where(np.abs(df['step']-df['c2'])<60, df['score'], df['score']*0.5)\n    \n    df['score'] = df['c3']\n    df['score'] = np.where(df['c']==1, df['score'], 0.)\n            \n    df = (df.pivot(index=['series_id', 'step'], columns='event', values='score').sort_index(level=[1, 0]))\n    df = df.reset_index()\n    val_df = df.sort_values(['series_id', 'step']).copy().reset_index(drop=True)\n\n    window_first = 3\n    window_size = [5]\n    ignore_width = [13]\n    val_df = df.sort_values(['series_id', 'step']).copy().reset_index(drop=True)\n\n    if True:\n        if True:\n            results_df = []\n            for _, gdf in val_df.groupby('series_id'):\n                for k, event in enumerate([event_type]):\n                    # Moving averages are used to smooth out the data.\n                    prob_arr = gdf[f\"{event}\"].rolling(window=window_first, center=True).mean().fillna(0).rolling(\n                        window=window_size[k], center=True).mean().fillna(0).values\n                    gdf['rolling_prob'] = prob_arr\n                    sort_arr = np.argsort(-prob_arr)\n                    rank_arr = np.empty_like(sort_arr)\n                    rank_arr[sort_arr] = np.arange(len(sort_arr))\n                    # index list for detected action\n                    idx_list = []\n                    for i in range(len(prob_arr)):\n                        this_idx = sort_arr[i]\n                        if this_idx >= 0:\n                            # Add maximam index to index_list\n                            idx_list.append(this_idx)\n                            for parity in (-1, 1):\n                                for j in range(1, ignore_width[k] + 1):\n                                    ex_idx = this_idx + j * parity\n                                    if ex_idx >= 0 and ex_idx < len(prob_arr):\n                                        # Exclude frames near this_idx where the action occurred.\n                                        sort_arr[rank_arr[ex_idx]] = -1\n                    this_df = gdf.iloc[idx_list].reset_index(drop=True).reset_index().rename(columns={'index': 'rank'})[\n                        ['rank', 'series_id', 'step', 'rolling_prob']]\n                    this_df['event'] = event\n\n                    this_df = this_df.loc[this_df['rolling_prob'] > np.quantile(this_df['rolling_prob'], 0.85)].reset_index(\n                        drop=True)\n\n                    results_df.append(this_df)\n\n            results_df = pd.concat(results_df)\n\n            results_df['step'] = results_df['step'] - 0.5\n            results_df['score'] = results_df['rolling_prob']\n            \n    return results_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##### Do Preds\n##########################\nestimator_o = lgb.Booster(model_file=(f'/kaggle/input/sleep-models/ver_{VER}/estimator_o.lgb'))\nestimator_w = lgb.Booster(model_file=(f'/kaggle/input/sleep-models/ver_{VER}/estimator_w.lgb'))\n\ntarget_enc = pd.read_csv(f'/kaggle/input/sleep-models/ver_{VER}/target_encoded_test.csv')\n\nall_preds = []\nall_labels = []\n\nfor cur_id in tqdm(ALL_IDX):\n    df = make_feats(cur_id)\n\n    df = df.merge(target_enc, on=['dt_hour_time'], how='left')\n    \n    if len(df)>0:\n\n        out_o = df[['series_id','dt_day','dt_hour','dt_minute','step','c']]\n        out_o['event'] = 'onset'\n        \n        out_w = df[['series_id','dt_day','dt_hour','dt_minute','step','c']]\n        out_w['event'] = 'wakeup'\n\n        out_o['score'] = estimator_o.predict(df[estimator_o.feature_name()]) \n        out_w['score'] = estimator_w.predict(df[estimator_w.feature_name()]) \n\n        out_o = post_process(out_o, event_type='onset')\n        out_w = post_process(out_w, event_type='wakeup')\n\n        df = pd.concat([out_o, out_w])\n        df = df[['series_id','step','event','score']]\n\n        ###\n        all_preds.append(df)\n\n        if LOCAL_RUN:\n            labels_df = pd.read_pickle(f'data/labels/{cur_id}.pkl')\n            labels_df = labels_df.dropna()\n            all_labels.append(labels_df)\n    else:\n        print(cur_id)\n## Concat\nif len(all_preds)>0:\n    all_preds = pd.concat(all_preds)\nelse:\n    all_preds = pd.DataFrame(columns=['series_id','step','event','score'], )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##### Dummy check score\n##########################\nif LOCAL_RUN:\n    all_labels = pd.concat(all_labels)\n    print('Valid score', score(all_labels[~all_labels['step'].isna()], all_preds))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##### Export\n##########################\nif not LOCAL_RUN:\n    os.system('rm -rf /kaggle/working/*')\n\n    all_preds = all_preds[['series_id','step','event','score']]\n    reverse_dict = {v:k for k,v in IDX_MAP.items()}\n    all_preds['series_id'] = all_preds['series_id'].map(reverse_dict)\n    all_preds = all_preds.sort_values(by=['series_id','step']).reset_index(drop=True)\n    all_preds = all_preds.reset_index(names='row_id')\n    all_preds.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_preds.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}