{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom scipy.stats import skew, kurtosis, rankdata","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:27:31.745858Z","iopub.execute_input":"2023-11-14T10:27:31.746369Z","iopub.status.idle":"2023-11-14T10:27:33.319678Z","shell.execute_reply.started":"2023-11-14T10:27:31.746326Z","shell.execute_reply":"2023-11-14T10:27:33.318751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Dict, List, Tuple\n\nseries_id_column_name = \"series_id\"\ntime_column_name = \"step\"\nevent_column_name = \"event\"\nscore_column_name = \"score\"\nuse_scoring_intervals = None\n\ndef precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n\ndef average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])\n\ndef match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\"\"\"\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted\n\ndef event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n) -> float:\n\n    # Ensure solution and submission are sorted properly\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution\n            .query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index='interval',\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, ['start', 'end']]\n            .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths[series_id_column_name].unique()],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    return mean_ap\n\ndef validation(events_true, events_predict_val):   \n    if len(events_predict_val) > 0:\n        print(event_detection_ap(events_true, events_predict_val, tolerances = {\n                \"onset\": [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n                \"wakeup\": [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n                }))\n    return events_predict_val","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:27:33.323772Z","iopub.execute_input":"2023-11-14T10:27:33.324119Z","iopub.status.idle":"2023-11-14T10:27:33.408048Z","shell.execute_reply.started":"2023-11-14T10:27:33.324095Z","shell.execute_reply":"2023-11-14T10:27:33.407114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"steps_in_minute = 12\nsteps_in_hour = steps_in_minute * 60","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:27:33.409067Z","iopub.execute_input":"2023-11-14T10:27:33.409407Z","iopub.status.idle":"2023-11-14T10:27:33.422076Z","shell.execute_reply.started":"2023-11-14T10:27:33.409366Z","shell.execute_reply":"2023-11-14T10:27:33.421281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"power = 3","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:27:33.424061Z","iopub.execute_input":"2023-11-14T10:27:33.424347Z","iopub.status.idle":"2023-11-14T10:27:33.431973Z","shell.execute_reply.started":"2023-11-14T10:27:33.424305Z","shell.execute_reply":"2023-11-14T10:27:33.431167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Preprocessing:\n    def __init__(self):\n        self.power = power\n        \n\n    def label_data_point_detection_binary(self, data, events):\n        data = data.copy()\n        data['sleeping'] = 0\n        \n        data.loc[data[data['step'].isin(events['step'] - (events['step'] % self.power))].index, 'sleeping'] = 1\n        return data  \n    \n    def label_data_point_detection_categorical(self, data, events):\n        data = data.copy()\n        data['sleeping'] = 0\n        onsets =  events.loc[events['event'] == 'onset'][['series_id', 'step']]   \n        data['index'] = data.index\n        steps_index = pd.Index(data.merge(onsets, on=['series_id', 'step'])['index'].values)\n        data.loc[steps_index, 'sleeping'] = 1\n        \n        wakeups =  events.loc[events['event'] == 'wakeup'][['series_id', 'step']]\n        steps_index = pd.Index(data.merge(wakeups, on=['series_id', 'step'])['index'].values) \n        data.loc[steps_index, 'sleeping'] = 2\n        \n        data = data.drop(['index'], axis=1)\n        \n        return data\n    \n    '''\n    def get_direction(self, data):\n        mask = data['anglez'] < 0\n\n        data['anglez_direction'] = 1\n        data.loc[mask, 'anglez_direction'] = -1\n        \n        data['anglez'] = data['anglez'].abs()\n\n        return data\n    \n    \n    def get_corr(self, data, lag):\n        n = len(data[0])  # Assuming all rows in data have the same length\n\n        # Create arrays for x and y by shifting the data by the specified lag\n        x = data[:, :-lag]\n        y = data[:, lag:]\n\n        # Calculate ranks for x and y\n        rank_x = np.apply_along_axis(rankdata, axis=1, arr=x)\n        rank_y = np.apply_along_axis(rankdata, axis=1, arr=y)\n\n        # Calculate the differences between ranks\n        rank_diff_x = rank_x - np.mean(rank_x, axis=1, keepdims=True)\n        rank_diff_y = rank_y - np.mean(rank_y, axis=1, keepdims=True)\n\n        # Calculate the Spearman correlation coefficients\n        numerator = np.sum(rank_diff_x * rank_diff_y, axis=1)\n        denominator_x = np.sqrt(np.sum(rank_diff_x**2, axis=1))\n        denominator_y = np.sqrt(np.sum(rank_diff_y**2, axis=1))\n\n        spearman_corr = numerator / (denominator_x * denominator_y)\n\n        return spearman_corr\n    '''\n   \n    def compress(self, data, use_power=True):\n        if use_power:\n            power =  self.power\n        else:\n            power =  2\n        data = data.copy()\n        \n        \n        anglez = data['anglez'].to_numpy()\n        enmo = data['enmo'].to_numpy()  \n        sleeping = data['sleeping'].to_numpy() \n        steps = data['step'].to_numpy()  \n        \n        anglez_reshaped = np.pad(anglez, (0, power - (anglez.size % power) if (anglez.size % power) != 0 else 0), mode='edge').reshape(-1, power)\n        enmo_reshaped = np.pad(enmo, (0, power - (enmo.size % power) if (anglez.size % power) != 0 else 0), mode='edge').reshape(-1, power)\n        sleeping_reshaped = np.pad(sleeping, (0, power - (sleeping.size % power) if (anglez.size % power) != 0 else 0), mode='constant').reshape(-1, power)\n        steps_reshaped = np.pad(steps, (0, power - (steps.size % power) if (anglez.size % power) != 0 else 0), mode='edge').reshape(-1, power)\n        \n        data = data.loc[::power]\n        \n        data.loc[:, 'anglez'] = np.median(anglez_reshaped, axis=-1)\n        data.loc[:, 'enmo'] = np.median(enmo_reshaped, axis=-1) \n        data.loc[:, 'sleeping'] = np.max(sleeping_reshaped, axis=-1) \n        data.loc[:, 'step'] = np.max(steps_reshaped, axis=-1) \n               \n        data = data.reset_index(drop=True)\n        return data\n\n    def rolling_features(self, data):        \n        rolling_window = steps_in_hour * 8 // self.power\n        \n        data.loc[:, 'anglez'] = data['anglez']\n        data.loc[:, 'enmo'] = data['enmo']\n    \n        anglez_roll = data['anglez'].rolling(rolling_window, min_periods=0)\n        enmo_roll = data['enmo'].rolling(rolling_window, min_periods=0)\n      \n        data.loc[:, 'anglez_mean_diff'] = anglez_roll.mean() - anglez_roll.mean().shift(-rolling_window).fillna(method='ffill')\n        data.loc[:, 'enmo_mean_diff'] = enmo_roll.mean() - enmo_roll.mean().shift(-rolling_window).fillna(method='ffill')\n        \n        data.loc[:, 'anglez_sdt_diff'] = data['anglez'].rolling(rolling_window, min_periods=0).std() - anglez_roll.std().shift(-rolling_window).fillna(method='ffill')\n        data.loc[:, 'enmo_sdt_diff'] = data['enmo'].rolling(rolling_window, min_periods=0).std() - enmo_roll.std().shift(-rolling_window).fillna(method='ffill')\n\n        data.loc[:, 'anglez_skew_diff'] = data['enmo'].rolling(rolling_window, min_periods=0).skew() - anglez_roll.skew().shift(-rolling_window).fillna(method='ffill')\n        data.loc[:, 'enmo_skew_diff'] = data['enmo'].rolling(rolling_window, min_periods=0).skew() - enmo_roll.skew().shift(-rolling_window).fillna(method='ffill')\n        \n        data.loc[:, 'anglez_kurt_diff'] = data['enmo'].rolling(rolling_window, min_periods=0).kurt() - anglez_roll.kurt().shift(-rolling_window).fillna(method='ffill')\n        data.loc[:, 'enmo_kurt_diff'] = data['enmo'].rolling(rolling_window, min_periods=0).kurt()- enmo_roll.kurt().shift(-rolling_window).fillna(method='ffill')      \n        \n\n        #anglez_percentiles = np.percentile(anglez_reshaped, 75, axis=-1) - np.percentile(anglez_reshaped, 25, axis=-1)\n        #enmo_percentiles = np.percentile(enmo_reshaped, 75, axis=-1) - np.percentile(enmo_reshaped, 25, axis=-1)\n        #data.loc[:, 'anglez_iqr'] = anglez_percentiles\n        #data.loc[:, 'enmo_iqr'] = enmo_percentiles\n        #data.loc[:, 'pos_prop'] = (np.sum(anglez_reshaped > 0, axis=-1) == 1).sum() / power\n        \n        return data\n    \n    \n    def preprocess(self, data, events, mode='train'):\n            preprocessed_data = pd.DataFrame()\n\n            data = preprocessing.label_data_point_detection_categorical(data, events)\n            \n            data = preprocessing.compress(data)\n            \n            data = preprocessing.rolling_features(data)\n            \n            sleeping = data['sleeping']\n            data = data.drop(['sleeping'], axis=1)\n            data['sleeping'] = sleeping\n\n            data = data.fillna(0)\n            \n            return data\n    \npreprocessing = Preprocessing()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:27:33.433157Z","iopub.execute_input":"2023-11-14T10:27:33.433433Z","iopub.status.idle":"2023-11-14T10:27:33.460829Z","shell.execute_reply.started":"2023-11-14T10:27:33.433410Z","shell.execute_reply":"2023-11-14T10:27:33.459956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import isnan\nfrom tqdm.contrib import tzip\n\ndef drop_na_data(data, events, indexes, prev_indexes, next_indexes):\n    count = 0\n    series_id_cached = ''\n    sub_data = []\n    indexes_drop = np.array([])\n    series_ids_prev, series_ids_next, steps_prev, steps_next, curr_index = events.loc[prev_indexes]['series_id'], events.loc[next_indexes]['series_id'], events.loc[prev_indexes]['step'], events.loc[next_indexes]['step'], indexes\n    for (series_id_prev, series_id_next, step_prev, step_next) in tzip(series_ids_prev, series_ids_next, steps_prev, steps_next):\n        if (series_id_prev != series_id_next) and (curr_index  != series_id_prev):\n            step_prev = 0\n            series_id_prev = series_id_next\n        if  (series_id_prev != series_id_next) and (curr_index  == series_id_prev):\n            steps_next = sub_data['step'].max()\n            series_id_next = series_id_prev\n        \n        if (series_id_prev == series_id_next) and not isnan(step_prev)  and not isnan(step_next) and (step_prev + steps_in_hour*16 < step_next):\n            if series_id_cached != series_id_prev:\n                series_id_cached = series_id_prev\n                sub_data = data.query('series_id == @series_id_prev')\n\n            index_prev = sub_data[sub_data['step'] == step_prev].index.values\n            index_next =  sub_data[sub_data['step'] == step_next].index.values\n\n            indexes = np.arange(index_prev+steps_in_hour*8, index_next-steps_in_hour*8)\n            if np.max(indexes) - np.min(indexes) < steps_in_hour * 36:\n                indexes_drop = np.concatenate((indexes_drop, indexes))\n        else:\n            count += 1\n    print('Non proccessed Nans occurances', count)\n    print('Total', series_ids_prev.size)\n    return data[~data.index.isin(indexes_drop)]\n    \n\ndef drop_na_event(data, events):\n    onsets = events[events['event'] == 'onset']\n    indexes = onsets[onsets['step'].isna()].index.to_numpy()\n    prev_indexes = events.index.intersection(indexes-1)\n    next_indexes = events.index.intersection(indexes+2)\n    #data = drop_na_data(data, events, indexes, prev_indexes, next_indexes)\n    events = events.drop(indexes)\n    #events = events.drop(indexes + 1)\n    \n    wakeup = events[events['event'] == 'wakeup']\n    indexes = wakeup[wakeup['step'].isna()].index.to_numpy()\n    prev_indexes = events.index.intersection(indexes-2)\n    next_indexes = events.index.intersection(indexes+1)\n    #data = drop_na_data(data, events, indexes, prev_indexes, next_indexes)\n    events = events.drop(indexes)\n    #events = events.drop(indexes - 1)\n    \n    return data, events\n\n\n\ndef drop_non_symetric_events(data, events):\n    onsets = events[events['event'] == 'onset']\n    indexes = onsets[onsets['step'].isna()].index.to_numpy()\n    indexes = indexes[events.loc[indexes+1]['step'].notna().values]\n    nan_series_onset = events.loc[indexes]['series_id'].unique()\n\n\n    \n    wakeup = events[events['event'] == 'wakeup']\n    indexes = wakeup[wakeup['step'].isna()].index.to_numpy()\n    indexes = indexes[events.loc[indexes-1]['step'].notna().values]\n    nan_series_wakeup = events.loc[indexes]['series_id'].unique()\n    \n    data, events = drop_empty_series(data, events, nan_series_onset)\n    data, events = drop_empty_series(data, events, nan_series_wakeup)\n    return data, events\n\n# Second array is contradictarary addition to droping\ndef drop_empty_series(data, events, empty_series):\n    recomended = set(['0f9e60a8e56d', '2fc653ca75c7', '390b487231ce', '89c7daa72eee', 'a3e59c2ce3f6', 'c5d08fc3e040', 'c7b1283bb7eb' 'e11b9d69f856'] + ['0ce74d6d2106', 'efbfc4526d58', 'c3072a759efb', '655f19eabf1e'])\n    remainder = list(recomended - set(empty_series))\n    \n    \n    data = data.drop(data[data['series_id'].isin(np.array(empty_series))].index)\n    events = events.drop(events[events['series_id'].isin(np.array(empty_series))].index)\n    \n    if len(remainder) > 0:\n        data = data.drop(data[data['series_id'].isin(np.array(remainder))].index)\n        events = events.drop(events[events['series_id'].isin(np.array(remainder))].index)\n    \n    return data, events\n\ndef get_main_data():\n    events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv', usecols=['series_id', 'step', 'event'])\n    na_nums = events.isna().groupby(events['series_id'])['step'].sum()\n    prop_nums = na_nums / (events.groupby(events['series_id'])['step'].count() + na_nums)\n    sns.histplot(prop_nums)\n    empty_series = list(prop_nums[prop_nums >= 0.5].index)\n    data = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet', columns=['series_id', 'step', 'anglez', 'enmo'])\n    data, events = drop_empty_series(data, events, empty_series)\n    data, events = drop_non_symetric_events(data, events)    \n    data, events = drop_na_event(data, events)\n\n\n    return data, events","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:27:33.462086Z","iopub.execute_input":"2023-11-14T10:27:33.462361Z","iopub.status.idle":"2023-11-14T10:27:33.493202Z","shell.execute_reply.started":"2023-11-14T10:27:33.462318Z","shell.execute_reply":"2023-11-14T10:27:33.492387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data, events = get_main_data()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:27:33.494292Z","iopub.execute_input":"2023-11-14T10:27:33.494573Z","iopub.status.idle":"2023-11-14T10:29:09.226318Z","shell.execute_reply.started":"2023-11-14T10:27:33.494550Z","shell.execute_reply":"2023-11-14T10:29:09.225281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.shape)\ndata = preprocessing.preprocess(data, events)\nprint(data.shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:29:09.227648Z","iopub.execute_input":"2023-11-14T10:29:09.227946Z","iopub.status.idle":"2023-11-14T10:30:04.414456Z","shell.execute_reply.started":"2023-11-14T10:29:09.227920Z","shell.execute_reply":"2023-11-14T10:30:04.413464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_num = 10","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:04.415776Z","iopub.execute_input":"2023-11-14T10:30:04.416398Z","iopub.status.idle":"2023-11-14T10:30:04.420829Z","shell.execute_reply.started":"2023-11-14T10:30:04.416362Z","shell.execute_reply":"2023-11-14T10:30:04.419900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_median_sleep(events):\n    events = events['step'].copy()\n    steps_med = np.quantile(events.loc[1::2].to_numpy() - events.loc[::2].to_numpy(), 0.25)\n    print(steps_med // steps_in_hour, steps_med % steps_in_hour / steps_in_minute)\n    \nget_median_sleep(events)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:04.434121Z","iopub.execute_input":"2023-11-14T10:30:04.434434Z","iopub.status.idle":"2023-11-14T10:30:04.448955Z","shell.execute_reply.started":"2023-11-14T10:30:04.434406Z","shell.execute_reply":"2023-11-14T10:30:04.448053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy.lib.stride_tricks import sliding_window_view\n\nwindow_size = steps_in_hour * 24 // power\n\ndef get_pad_size(data):\n    required_padding = window_size - (data.shape[0] % window_size) if (data.shape[0] % window_size) != 0 else 0\n    return required_padding\n\ndef get_seq(data, variable):    \n    required_padding = get_pad_size(data)    \n    X = np.concatenate((data[variable], np.array(list(data[variable].iloc[-1].to_numpy()) * required_padding).reshape(-1, features_num)))\n    X = X.reshape(-1, window_size, features_num)\n    \n    return X\n\ndef pad_data(data, sub_data):\n    required_padding = get_pad_size(sub_data)\n    tail_index = int(sub_data.iloc[-1:].index[-1])\n    #LOC BEHAVIOR IS END_INCLUSIVE\n    data.index = pd.Index([index if index <= tail_index else index + required_padding for index in data.index])\n    if required_padding != 0:\n        data = pd.concat([data.loc[:tail_index-1], *([data.loc[[tail_index]]] * (required_padding+1)), data.loc[tail_index+required_padding+1:]])\n    return data\n    \n\ndef get_X(data, mode='train'):\n    data = data.copy()\n    X = np.zeros((0, window_size, features_num))\n    steps = np.zeros((0, window_size))\n    clms = data.columns[2:-1]\n    for series_id in tqdm(data['series_id'].unique()):\n        sub_data = data.query('series_id == @series_id')\n        \n        X_id = get_seq(sub_data, clms)\n        \n        step = np.squeeze(np.array([sub_data['step']]))\n        \n        step = np.pad(step, (0, window_size - (step.size % window_size) if (step.size % window_size) != 0 else 0), mode='edge').reshape(-1, window_size)\n        \n        X = np.vstack((X, X_id))\n        steps = np.vstack((steps, step))\n        \n        if mode !='train':\n            data = pad_data(data, sub_data)\n        gc.collect()\n        \n        \n    X = np.array(X)\n    \n    if mode=='train':\n        return X, steps\n    else:\n        return X, steps, data","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:04.450378Z","iopub.execute_input":"2023-11-14T10:30:04.450759Z","iopub.status.idle":"2023-11-14T10:30:04.466740Z","shell.execute_reply.started":"2023-11-14T10:30:04.450727Z","shell.execute_reply":"2023-11-14T10:30:04.465751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_val = data[data['series_id'].isin(data['series_id'].unique()[-3:])]\ndata = data[~data['series_id'].isin(data['series_id'].unique()[-3:])]","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:04.468293Z","iopub.execute_input":"2023-11-14T10:30:04.468684Z","iopub.status.idle":"2023-11-14T10:30:08.490619Z","shell.execute_reply.started":"2023-11-14T10:30:04.468647Z","shell.execute_reply":"2023-11-14T10:30:08.489574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_val = events[events['series_id']==data_val['series_id'].unique()[-1]]","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:08.491876Z","iopub.execute_input":"2023-11-14T10:30:08.492233Z","iopub.status.idle":"2023-11-14T10:30:08.530594Z","shell.execute_reply.started":"2023-11-14T10:30:08.492201Z","shell.execute_reply":"2023-11-14T10:30:08.529802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:08.531736Z","iopub.execute_input":"2023-11-14T10:30:08.532024Z","iopub.status.idle":"2023-11-14T10:30:08.537566Z","shell.execute_reply.started":"2023-11-14T10:30:08.531999Z","shell.execute_reply":"2023-11-14T10:30:08.536700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:08.538768Z","iopub.execute_input":"2023-11-14T10:30:08.539094Z","iopub.status.idle":"2023-11-14T10:30:08.605686Z","shell.execute_reply.started":"2023-11-14T10:30:08.539062Z","shell.execute_reply":"2023-11-14T10:30:08.604721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, steps_train = get_X(data)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:08.606825Z","iopub.execute_input":"2023-11-14T10:30:08.607093Z","iopub.status.idle":"2023-11-14T10:30:59.932313Z","shell.execute_reply.started":"2023-11-14T10:30:08.607069Z","shell.execute_reply":"2023-11-14T10:30:59.930127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val, steps, data_val = get_X(data_val, 'inference')","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:30:59.933848Z","iopub.execute_input":"2023-11-14T10:30:59.934149Z","iopub.status.idle":"2023-11-14T10:31:01.697198Z","shell.execute_reply.started":"2023-11-14T10:30:59.934123Z","shell.execute_reply":"2023-11-14T10:31:01.696242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_y(data):\n    y_train = np.zeros((0, window_size//(steps_in_minute// power)))\n    data = data[['series_id', 'sleeping']]\n    for series_id in tqdm(data['series_id'].unique()):\n\n        y_train_id = data.query('series_id == @series_id')['sleeping']\n        \n        y_train_id = np.pad(y_train_id, (0, window_size - (y_train_id.size % window_size) if (y_train_id.size % window_size) != 0 else 0), mode='constant', constant_values=(0, 0)).reshape(-1, window_size).astype(np.float64)\n        \n        y_train_id = y_train_id.reshape(-1, window_size//(steps_in_minute // power), steps_in_minute//power).max(axis=-1)\n        \n        y_train = np.vstack((y_train, y_train_id))\n    return y_train","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:01.698395Z","iopub.execute_input":"2023-11-14T10:31:01.698682Z","iopub.status.idle":"2023-11-14T10:31:01.706274Z","shell.execute_reply.started":"2023-11-14T10:31:01.698657Z","shell.execute_reply":"2023-11-14T10:31:01.705285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data['sleeping'] = np.convolve(np.squeeze(data['sleeping'].values), [1, 1, 1], mode='same')","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:01.707651Z","iopub.execute_input":"2023-11-14T10:31:01.708025Z","iopub.status.idle":"2023-11-14T10:31:01.716633Z","shell.execute_reply.started":"2023-11-14T10:31:01.707990Z","shell.execute_reply":"2023-11-14T10:31:01.715689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = get_y(data)\ny_val = get_y(data_val)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:01.717710Z","iopub.execute_input":"2023-11-14T10:31:01.717997Z","iopub.status.idle":"2023-11-14T10:31:33.844084Z","shell.execute_reply.started":"2023-11-14T10:31:01.717972Z","shell.execute_reply":"2023-11-14T10:31:33.843114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_sample = data.loc[:, ['series_id', 'step']]","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:33.845136Z","iopub.execute_input":"2023-11-14T10:31:33.845427Z","iopub.status.idle":"2023-11-14T10:31:34.060752Z","shell.execute_reply.started":"2023-11-14T10:31:33.845403Z","shell.execute_reply":"2023-11-14T10:31:34.059712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:34.062094Z","iopub.execute_input":"2023-11-14T10:31:34.062461Z","iopub.status.idle":"2023-11-14T10:31:34.169042Z","shell.execute_reply.started":"2023-11-14T10:31:34.062435Z","shell.execute_reply":"2023-11-14T10:31:34.168026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Conv1D, GRU, LSTM, Dense, Concatenate, Add, MaxPooling1D, Conv1DTranspose, AveragePooling1D, UpSampling1D, BatchNormalization, Concatenate, AveragePooling1D\nfrom tensorflow.keras import Input, Model, Sequential\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nimport tensorflow as tf\n\ndef get_filter_conv_block(input_):\n    conv_1 = Conv1D(kernel_size=11, filters=10, groups=features_num, padding='same')(input_)\n    conv_2 = Conv1D(kernel_size=11, filters=10, groups=10, padding='same')(conv_1)           \n    conv_3 = Conv1D(kernel_size=11, filters=100, groups=10, padding='same')(conv_2)\n    conv_4 = Conv1D(kernel_size=11, filters=100, groups=100, padding='same')(conv_3)\n    return conv_4\n\ndef conv_encoder_block(input_, conv_emb, pool_size,  activation='ReLU'):\n    conv_1 = Conv1D(kernel_size=11, filters=conv_emb,activation=activation, padding='same')(input_)\n    conv_2 = Conv1D(kernel_size=11, filters=conv_emb,activation=activation, padding='same')(conv_1)\n    batch_norm = BatchNormalization()(conv_2)\n    max_pool = MaxPooling1D(pool_size)(batch_norm)\n    return max_pool\n\ndef conv_decoder_block(input_, conv_emb, pool_size,  activation='ReLU'):\n    conv_1 = Conv1DTranspose(kernel_size=11, filters=conv_emb,activation=activation, padding='same')(input_)\n    conv_2 = Conv1DTranspose(kernel_size=11, filters=conv_emb,activation=activation, padding='same')(conv_1)\n    batch_norm = BatchNormalization()(conv_2)\n    max_pool = UpSampling1D(pool_size)(batch_norm)\n    return max_pool\n\ndef get_conv_gru_block(input_, size, padding='valid'):\n    gru_1 = GRU(size, return_sequences=True)(input_)\n    gru_2 = GRU(size, return_sequences=True)(gru_1)\n    return gru_2 \n     \npool_size = 2\nconv_emb = 24\n\ndef get_model(input_):\n    encoder_1 = conv_encoder_block(input_, conv_emb, 4)\n    encoder_2 = conv_encoder_block(encoder_1, 2*conv_emb, 6)\n    encoder_3 = conv_encoder_block(encoder_2, 3*conv_emb, 4)\n    encoder_4 = conv_encoder_block(encoder_3, 4*conv_emb, 2)\n    first_gru_block = get_conv_gru_block(encoder_4, 4*conv_emb)\n    second_gru_block =  get_conv_gru_block(first_gru_block, 4*conv_emb)\n    decoder_1 = conv_decoder_block(Concatenate()([first_gru_block, encoder_4]), 4*conv_emb, 2)\n    decoder_2 = conv_decoder_block(Concatenate()([decoder_1, encoder_3]), 3*conv_emb, 4)\n    decoder_3 = conv_decoder_block(Concatenate()([decoder_2, encoder_2]), 2*conv_emb, 6)\n    #decoder_4 = conv_decoder_block(Concatenate()([decoder_3, encoder_1]), conv_emb, 1)\n    \n    #    \n    \n    conv_1 = Conv1D(kernel_size=11, filters=conv_emb, activation='ReLU', padding='same')(Concatenate()([decoder_3, encoder_1]))\n    return conv_1\n     \n\n\ndef get_nn():\n    win = Input((window_size, features_num))\n    \n   # dense_1 =  Dense(16, activation='leaky_relu')(win)   \n    \n    #dense_2 =  Dense(24, activation='leaky_relu')(dense_1) \n    \n    filt_conv_block = get_filter_conv_block(win)\n    \n    seq = get_model(filt_conv_block)\n    \n    conv_1 = Conv1D(kernel_size=11, filters=2*conv_emb, activation='ReLU', padding='same')(seq)\n    conv_2 = Conv1D(kernel_size=11, filters=2*conv_emb, activation='ReLU', padding='same')(conv_1)\n    conv_3 = Conv1D(kernel_size=11, filters=conv_emb, activation='ReLU', padding='same')(conv_2)\n    conv_4 = Conv1D(kernel_size=11, filters=conv_emb, activation='ReLU', padding='same')(conv_3)\n    \n    \n    #attention = tf.keras.layers.MultiHeadAttention(num_heads=8, key_dim=10, attention_axes=[1])(conv_3, conv_3)\n    output = Conv1D(kernel_size=11, filters=3, activation='softmax', padding='same')(conv_4)\n    \n    model = Model(win, output)\n    \n    return model  ","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:43:22.881386Z","iopub.execute_input":"2023-11-14T10:43:22.882280Z","iopub.status.idle":"2023-11-14T10:43:22.902064Z","shell.execute_reply.started":"2023-11-14T10:43:22.882245Z","shell.execute_reply":"2023-11-14T10:43:22.901242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = get_nn()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:43:23.281781Z","iopub.execute_input":"2023-11-14T10:43:23.282737Z","iopub.status.idle":"2023-11-14T10:43:24.752108Z","shell.execute_reply.started":"2023-11-14T10:43:23.282702Z","shell.execute_reply":"2023-11-14T10:43:24.751054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.utils import class_weight\n\nclass_weight = class_weight.compute_class_weight('balanced',classes=np.unique(y_train),y=np.squeeze(y_train.reshape(-1)))\n\nclass_weight","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:43:24.753929Z","iopub.execute_input":"2023-11-14T10:43:24.754656Z","iopub.status.idle":"2023-11-14T10:43:25.507088Z","shell.execute_reply.started":"2023-11-14T10:43:24.754616Z","shell.execute_reply":"2023-11-14T10:43:25.506103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow_addons.metrics import F1Score\nfrom tensorflow.keras.losses import BinaryFocalCrossentropy\n\n\nconv_coef = [0.5] * 9 + [0.9] * 3 \n\nscce = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)\n\ndef get_compressed_cost(y_true, y_pred, bin_size):\n        y_pred_labels = tf.cast(tf.math.argmax(y_pred, axis=-1), tf.uint8)\n        \n        y_true_reshaped = tf.reshape(y_true, [y_true.shape[0], -1, bin_size])\n        y_pred_reshaped = tf.reshape(y_pred, [y_pred.shape[0], -1, bin_size, y_pred.shape[-1]])\n        y_true_60 = tf.math.reduce_max(y_true_reshaped, axis=-1)\n        \n        y_true_reshaped = tf.cast(y_true_reshaped, tf.uint8)\n        y_true_60 = tf.cast(y_true_60, tf.uint8)\n        y_true_sparse_60 = tf.one_hot(y_true_60, y_pred_reshaped.shape[-1])\n        y_true_sparse_60 = tf.expand_dims(y_true_sparse_60, axis=-2)\n        \n        y_pred_60 = tf.math.reduce_max(y_pred_reshaped * y_true_sparse_60, axis=-1)\n        y_pred_60 = tf.math.reduce_max(y_pred_60, axis=-1)\n        \n        y_pred_60 = tf.expand_dims(y_pred_60, axis=-1) * tf.one_hot(y_true_60, y_pred.shape[-1]) \n              \n        y_pred_60 = tf.where(y_pred_60==0, (tf.ones_like(y_pred_60) - tf.expand_dims(tf.reduce_max(y_pred_60, axis=-1), axis=-1)) / 2, y_pred_60)\n       \n        losses_60 = scce(y_true_60, y_pred_60)\n\n        true_sum =  tf.cast(tf.reduce_sum(y_true), tf.float32)\n        label_sum = tf.cast(tf.reduce_sum(y_pred_labels), tf.float32)     \n    \n        y_true_sparse_60_bool = tf.cast(y_true_sparse_60, tf.bool)\n        \n        losses = 0\n        if true_sum < label_sum:\n            losses =  losses_60 * (label_sum - true_sum)\n        elif  label_sum < true_sum:\n            losses =  losses_60 * (true_sum - label_sum)\n        else:\n            losses =  losses_60\n        \n        y_true_pred_foc_60 = tf.reshape(y_pred_60[tf.squeeze(y_true_sparse_60_bool)], losses.shape)  \n        \n        losses = losses * (1 - y_true_pred_foc_60) ** 2\n        return tf.reduce_mean(losses)\n\nclass Classic_Loss(tf.keras.losses.Loss):\n    def __init__(self):\n        super().__init__()\n        self.scce = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)\n    def call(self, y_true, y_pred):\n        losses = self.scce(y_true, y_pred)\n        losses = tf.where(y_true != 0, losses * 22, losses)\n\n        y_true = tf.cast(y_true, tf.uint8)\n        y_true_sparse = tf.one_hot(y_true, y_pred.shape[-1])\n        y_true_sparse = tf.cast(y_true_sparse, tf.bool)\n        \n        y_true_pred = tf.reshape(y_pred[y_true_sparse], losses.shape)  \n        \n        losses = losses * (1 - y_true_pred) ** 2\n        return tf.reduce_mean(losses)\n    \n\nclass Custom_CE_Loss(tf.keras.losses.Loss):\n    def __init__(self, bin_size, use_bin=True):\n        super().__init__()\n        self.bin_size = bin_size\n        self.use_bin = use_bin\n        \n    def call(self, y_true, y_pred):\n        losses = scce(y_true, y_pred)\n  \n        #losses = tf.where(y_true == 0, losses / 22, losses) \n        #losses = tf.where(y_true != 0, losses * 22, losses) \n        \n        y_true = tf.cast(y_true, tf.uint8)\n        y_true_sparse = tf.one_hot(y_true, y_pred.shape[-1])\n        y_true_sparse = tf.cast(y_true_sparse, tf.bool)\n        \n        y_true_pred = tf.reshape(y_pred[y_true_sparse], losses.shape)  \n        \n        losses = losses * (1 - y_true_pred) ** 2\n        \n        losses_bin = get_compressed_cost(y_true, y_pred, self.bin_size)\n        \n        if self.use_bin:\n            return losses + losses_bin\n        else:\n            losses_3 = get_compressed_cost(y_true, y_pred, 3)\n            losses_30 =get_compressed_cost(y_true, y_pred, 30)\n            losses_180 = get_compressed_cost(y_true, y_pred, 180)\n            return losses + losses_3 + losses_30 + losses_180\n\n    \nclass Crossentopy_Bin_Loss(tf.keras.metrics.Metric):\n\n    def __init__(self, bin_size, name='cross_', **kwargs):\n        super(Crossentopy_Bin_Loss, self).__init__(name=name+f\"{bin_size}\", **kwargs)\n        self.loss = self.add_weight(name='loss_'+f\"{bin_size}\", initializer='zeros')\n        self.bin_size = bin_size\n    \n    def update_state(self, y_true, y_pred, sample_weight=None):          \n        self.loss.assign(get_compressed_cost(y_true, y_pred, self.bin_size))\n        \n    def reset_state(self):\n        self.loss.assign(0)\n    \n    def result(self):\n        return self.loss    \n    \n\nclass BinaryTruePositives(tf.keras.metrics.Metric):\n\n    def __init__(self, name='binary_true_positives', **kwargs):\n        super(BinaryTruePositives, self).__init__(name=name, **kwargs)\n        self.true_positives = self.add_weight(name='tp', initializer='zeros')\n        self.f1 = F1Score(num_classes = 3, average='macro') \n    \n    def update_state(self, y_true, y_pred, sample_weight=None):  \n        y_true = tf.cast(y_true, tf.uint8)\n        y_true_sparse = tf.one_hot(y_true, y_pred.shape[-1])\n        y_true_sparse = tf.cast(y_true_sparse, tf.uint8)\n        \n        y_true_sparse = tf.reshape(y_true_sparse, [-1, y_true_sparse.shape[-1]])\n        y_pred = tf.reshape(y_pred, [-1, y_pred.shape[-1]])\n                \n        self.f1.update_state(y_true_sparse, y_pred)\n        \n        self.true_positives.assign(self.f1.result())\n        \n    def reset_state(self):\n        self.true_positives.assign(0)\n    \n    def result(self):\n        return self.true_positives","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:43:25.508848Z","iopub.execute_input":"2023-11-14T10:43:25.509223Z","iopub.status.idle":"2023-11-14T10:43:25.552391Z","shell.execute_reply.started":"2023-11-14T10:43:25.509187Z","shell.execute_reply":"2023-11-14T10:43:25.551433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:43:25.554084Z","iopub.execute_input":"2023-11-14T10:43:25.554387Z","iopub.status.idle":"2023-11-14T10:43:25.669982Z","shell.execute_reply.started":"2023-11-14T10:43:25.554358Z","shell.execute_reply":"2023-11-14T10:43:25.669038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:51.496219Z","iopub.execute_input":"2023-11-14T10:31:51.496584Z","iopub.status.idle":"2023-11-14T10:31:51.699659Z","shell.execute_reply.started":"2023-11-14T10:31:51.496550Z","shell.execute_reply":"2023-11-14T10:31:51.698689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_delta=0.0001","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:51.701023Z","iopub.execute_input":"2023-11-14T10:31:51.701412Z","iopub.status.idle":"2023-11-14T10:31:51.706118Z","shell.execute_reply.started":"2023-11-14T10:31:51.701378Z","shell.execute_reply":"2023-11-14T10:31:51.705268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:51.707309Z","iopub.execute_input":"2023-11-14T10:31:51.707640Z","iopub.status.idle":"2023-11-14T10:31:51.718715Z","shell.execute_reply.started":"2023-11-14T10:31:51.707607Z","shell.execute_reply":"2023-11-14T10:31:51.717772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(bin_size, epochs, learning_rate, use_bin=True):\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n                  loss=Custom_CE_Loss(bin_size, use_bin),\n                  metrics=['accuracy', BinaryTruePositives(), Crossentopy_Bin_Loss(bin_size)], run_eagerly=True)\n    history = model.fit(X_train, y_train, batch_size=64, callbacks=[ReduceLROnPlateau(monitor='binary_true_positives',min_delta=min_delta, mode='max', factor=0.2, patience=1), EarlyStopping(monitor='loss', min_delta=min_delta/100, patience=10)], epochs=epochs, validation_data=(X_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:51.720070Z","iopub.execute_input":"2023-11-14T10:31:51.720342Z","iopub.status.idle":"2023-11-14T10:31:51.729769Z","shell.execute_reply.started":"2023-11-14T10:31:51.720297Z","shell.execute_reply":"2023-11-14T10:31:51.728898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model(180, 10, 0.001)\ntrain_model(30, 10, 0.001)\ntrain_model(10, 10, 0.001)\ntrain_model(5, 10, 0.001)\ntrain_model(3, 20, 0.001)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:31:51.730855Z","iopub.execute_input":"2023-11-14T10:31:51.732902Z","iopub.status.idle":"2023-11-14T10:32:02.571721Z","shell.execute_reply.started":"2023-11-14T10:31:51.732877Z","shell.execute_reply":"2023-11-14T10:32:02.569600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n                  loss=Classic_Loss(),\n                  metrics=['accuracy', BinaryTruePositives(), Crossentopy_Bin_Loss(30)], run_eagerly=True)\nhistory = model.fit(X_train, y_train, batch_size=64, callbacks=[ReduceLROnPlateau(monitor='binary_true_positives',min_delta=min_delta, mode='max', factor=0.2, patience=0), EarlyStopping(monitor='loss', min_delta=min_delta/100, patience=25)], epochs=100, validation_data=(X_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:41:31.416028Z","iopub.execute_input":"2023-11-14T10:41:31.416400Z","iopub.status.idle":"2023-11-14T10:41:57.205356Z","shell.execute_reply.started":"2023-11-14T10:41:31.416371Z","shell.execute_reply":"2023-11-14T10:41:57.203692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(np.argmax(model.predict(X_train), axis=-1))","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:40:23.161016Z","iopub.execute_input":"2023-11-14T10:40:23.161987Z","iopub.status.idle":"2023-11-14T10:40:30.403307Z","shell.execute_reply.started":"2023-11-14T10:40:23.161955Z","shell.execute_reply":"2023-11-14T10:40:30.402373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(y_train)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:36:26.757483Z","iopub.execute_input":"2023-11-14T10:36:26.757776Z","iopub.status.idle":"2023-11-14T10:36:26.765094Z","shell.execute_reply.started":"2023-11-14T10:36:26.757751Z","shell.execute_reply":"2023-11-14T10:36:26.764359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(np.argmax(model.predict(X_val), axis=-1))","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:36:26.766142Z","iopub.execute_input":"2023-11-14T10:36:26.766414Z","iopub.status.idle":"2023-11-14T10:36:27.106557Z","shell.execute_reply.started":"2023-11-14T10:36:26.766390Z","shell.execute_reply":"2023-11-14T10:36:27.105647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sub_frame(X, model, data_test, steps):\n        y_pred = model.predict([X])\n\n        \n        y_pred = y_pred.reshape(y_pred.shape[0], -1, 3)\n        steps = steps.reshape(steps.shape[0], -1, steps_in_minute // power).max(axis=-1)\n        steps = steps.reshape(-1)\n        series = data_test['series_id'].loc[::steps_in_minute // power]\n        \n        prob = 0.75\n        \n        labels = np.argmax(y_pred, axis=-1)\n        y_pred = np.max(y_pred, axis=-1)\n        \n        labels = labels.reshape(-1)    \n        y_pred = y_pred.reshape(-1)    \n        \n        \n        points = np.where(labels >= 1)[0]\n        y_pred = y_pred[labels >= 1]\n        series = series[labels >= 1]\n        labels = labels[labels >= 1]   \n\n        df = pd.DataFrame({'series_id': [], 'step': [], 'event': [], 'score': []})\n        df['score'] = df['score'].astype(np.float64)\n\n                \n        for (series_id, point, pred, label) in zip(series, points, y_pred, labels):\n            if label == 1:\n                df = pd.concat((df, pd.DataFrame({'series_id': [series_id], 'step': int(steps[point]), 'event': ['onset'], 'score': [pred]})), ignore_index=True)\n                df.reset_index(drop=True)\n            elif label == 2:\n                df = pd.concat((df, pd.DataFrame({'series_id': [series_id], 'step': int(steps[point]), 'event': ['wakeup'], 'score': [pred]})), ignore_index=True)\n                df.reset_index(drop=True)\n        \n        df['step'] = df['step'].astype(np.int64)\n        df['row_id'] = df.index\n        return df","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:36:31.471794Z","iopub.execute_input":"2023-11-14T10:36:31.472501Z","iopub.status.idle":"2023-11-14T10:36:31.484427Z","shell.execute_reply.started":"2023-11-14T10:36:31.472465Z","shell.execute_reply":"2023-11-14T10:36:31.483387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#events_predict = get_sub_frame(X_l_train, X_r_train, model, data_sample, steps_train)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:36:31.778853Z","iopub.execute_input":"2023-11-14T10:36:31.779192Z","iopub.status.idle":"2023-11-14T10:36:31.783241Z","shell.execute_reply.started":"2023-11-14T10:36:31.779164Z","shell.execute_reply":"2023-11-14T10:36:31.782296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:36:32.086307Z","iopub.execute_input":"2023-11-14T10:36:32.086627Z","iopub.status.idle":"2023-11-14T10:36:32.354230Z","shell.execute_reply.started":"2023-11-14T10:36:32.086603Z","shell.execute_reply":"2023-11-14T10:36:32.353336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:36:32.409067Z","iopub.execute_input":"2023-11-14T10:36:32.409405Z","iopub.status.idle":"2023-11-14T10:36:32.415688Z","shell.execute_reply.started":"2023-11-14T10:36:32.409376Z","shell.execute_reply":"2023-11-14T10:36:32.414689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_true = events[events['series_id'].isin(data_val['series_id'].unique())]\nevents_predict_val = get_sub_frame(X_val, model, data_val, steps).drop_duplicates(['series_id', 'step'])\n    \ndisplay(events_true)\ndisplay(events_predict_val)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:36:32.954595Z","iopub.execute_input":"2023-11-14T10:36:32.955431Z","iopub.status.idle":"2023-11-14T10:36:34.079098Z","shell.execute_reply.started":"2023-11-14T10:36:32.955396Z","shell.execute_reply":"2023-11-14T10:36:34.078105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_val_predict = validation(events_true, events_predict_val)\ndata_val_predict = preprocessing.label_data_point_detection_categorical(data_val, events_val_predict)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:36:34.807061Z","iopub.execute_input":"2023-11-14T10:36:34.807700Z","iopub.status.idle":"2023-11-14T10:36:38.627069Z","shell.execute_reply.started":"2023-11-14T10:36:34.807666Z","shell.execute_reply":"2023-11-14T10:36:38.626135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Visulization:\n    def __init__(self, data, data_predict, max_step):\n        self.series_id = np.random.choice(data['series_id'].unique())\n        self.data = data[data['series_id'] == self.series_id].copy()\n        self.data = self.data[self.data['step'] <= max_step]\n        \n        self.data_predict = data_predict[data_predict['series_id'] == self.series_id].copy()\n        self.data_predict = self.data_predict[self.data_predict['step'] <= max_step]\n        \n        \n    \n    def plot_signal(self, variable, ax1, ax2, ax3):\n        data = self.data.copy()\n        data_predict = self.data_predict.copy()\n                \n        time = np.arange(data['step'].size)\n        \n        sns.lineplot(data=data, x=time, y=variable, ax=ax1)\n\n        sns.scatterplot(data=data[data['sleeping']==1], x=time[data['sleeping']==1], y=variable, ax=ax2, color='red', alpha=1) \n        sns.scatterplot(data=data[data['sleeping']==2], x=time[data['sleeping']==2], y=variable, ax=ax2, color='black', alpha=1) \n        sns.lineplot(data=data, x=time, y=variable, ax=ax2, alpha=0.75)\n        \n        sns.scatterplot(data=data_predict[data_predict['sleeping']==1], x=time[data_predict['sleeping']==1], y=variable, ax=ax3, color='red', alpha=1) \n        sns.scatterplot(data=data_predict[data_predict['sleeping']==2], x=time[data_predict['sleeping']==2], y=variable, ax=ax3, color='black', alpha=1) \n        sns.lineplot(data=data_predict, x=time, y=variable, ax=ax3, alpha=0.75)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.589654Z","iopub.status.idle":"2023-11-14T10:32:02.589999Z","shell.execute_reply.started":"2023-11-14T10:32:02.589828Z","shell.execute_reply":"2023-11-14T10:32:02.589844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_val['sleeping'].sum()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.591474Z","iopub.status.idle":"2023-11-14T10:32:02.591819Z","shell.execute_reply.started":"2023-11-14T10:32:02.591649Z","shell.execute_reply":"2023-11-14T10:32:02.591665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visulization = Visulization(data_val, data_val_predict, window_size*power*3)\n#fig, (ax1, ax2, ax3) = plt.subplots(len(data_val.columns[2:-1]), 3, figsize=(18, 9)) \n#visulization.plot_signal('enmo', ax1, ax2, ax3)\n#for column, (ax1, ax2, ax3) in  zip(data_val.columns[2:-1], axs):\n#    visulization.plot_signal(column, ax1, ax2, ax3)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.593204Z","iopub.status.idle":"2023-11-14T10:32:02.593590Z","shell.execute_reply.started":"2023-11-14T10:32:02.593408Z","shell.execute_reply":"2023-11-14T10:32:02.593426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events[events['event']=='wakeup']['step']","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.594941Z","iopub.status.idle":"2023-11-14T10:32:02.595281Z","shell.execute_reply.started":"2023-11-14T10:32:02.595114Z","shell.execute_reply":"2023-11-14T10:32:02.595130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    def label_data_point_detection_categorical(data, events):\n        data = data.copy()\n        data['sleeping'] = 0\n        \n        data.loc[data[data['step'].isin(events['step'] - (events[events['event']=='onset']['step'] % steps_in_minute))].index, 'sleeping'] = 1\n        data.loc[data[data['step'].isin(events['step'] - (events[events['event']=='wakeup']['step'] % steps_in_minute))].index, 'sleeping'] = 2\n        return data \n    \n    def label_data_point_detection_binary(data, events):\n        data = data.copy()\n        data['sleeping'] = 0\n        \n        data.loc[data[data['step'].isin(events['step'] - (events['step'] % steps_in_minute))].index, 'sleeping'] = 1\n        return data  ","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.596385Z","iopub.status.idle":"2023-11-14T10:32:02.596730Z","shell.execute_reply.started":"2023-11-14T10:32:02.596558Z","shell.execute_reply":"2023-11-14T10:32:02.596574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#area_pred = label_data_point_detection_binary(data_sample, events_predict)\n#area_pred = area_pred.where(area_pred['sleeping']!=0).dropna()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.597725Z","iopub.status.idle":"2023-11-14T10:32:02.598107Z","shell.execute_reply.started":"2023-11-14T10:32:02.597914Z","shell.execute_reply":"2023-11-14T10:32:02.597931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_sample\ndel X_train\ndel y_train\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.599624Z","iopub.status.idle":"2023-11-14T10:32:02.599989Z","shell.execute_reply.started":"2023-11-14T10:32:02.599810Z","shell.execute_reply":"2023-11-14T10:32:02.599828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id', 'step', 'anglez', 'enmo'])\nsubmission_event = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv', usecols=['series_id', 'step', 'event'])","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.603091Z","iopub.status.idle":"2023-11-14T10:32:02.603481Z","shell.execute_reply.started":"2023-11-14T10:32:02.603261Z","shell.execute_reply":"2023-11-14T10:32:02.603277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"    from numpy.lib.stride_tricks import sliding_window_view\n\n    def get_submission(test, submission_event):\n        data_test = test\n\n        data_test = preprocessing.preprocess(test, submission_event, mode='inference')\n        \n        \n        X, steps, data_test = get_X(data_test, mode='inference')\n        \n     \n        return get_sub_frame(X, model, data_test, steps).drop_duplicates(['series_id', 'step'])","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.604863Z","iopub.status.idle":"2023-11-14T10:32:02.605230Z","shell.execute_reply.started":"2023-11-14T10:32:02.605045Z","shell.execute_reply":"2023-11-14T10:32:02.605062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = get_submission(test, submission_event)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.607477Z","iopub.status.idle":"2023-11-14T10:32:02.607858Z","shell.execute_reply.started":"2023-11-14T10:32:02.607666Z","shell.execute_reply":"2023-11-14T10:32:02.607684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-11-14T10:32:02.609138Z","iopub.status.idle":"2023-11-14T10:32:02.609544Z","shell.execute_reply.started":"2023-11-14T10:32:02.609349Z","shell.execute_reply":"2023-11-14T10:32:02.609375Z"},"trusted":true},"execution_count":null,"outputs":[]}]}