{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":8706105,"sourceType":"datasetVersion","datasetId":5222143}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm import tqdm\nimport gc\nfrom sklearn.preprocessing import RobustScaler\nfrom typing import Dict, List, Tuple\nimport polars as pl\nimport yaml\nfrom types import SimpleNamespace\nimport collections\nimport os\nimport importlib\nimport pickle\nimport sys\nimport glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-16T16:32:38.760637Z","iopub.execute_input":"2024-06-16T16:32:38.761021Z","iopub.status.idle":"2024-06-16T16:32:38.767643Z","shell.execute_reply.started":"2024-06-16T16:32:38.760993Z","shell.execute_reply":"2024-06-16T16:32:38.766671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"################## PREPROCESSING SPECIFICATIONS\n# Put here features that you want to apply normalization\ncols_f = []\n\n# Put here features that you want to save\nnum_cols_f = ['step_first','hour','minute','noise_removal',\n              'enmo_std_norm','anglez_std_norm','anglez_std_norm_orig',\n              'anglez','enmo',\n              'anglez_equal1','anglez_equal2',\n              'std_q50_left','std_q50_right','fe__check_c','porc_step','group_len_c_norm',\n              'hmin_onset','hmin_wakeup']\ndef specific_process(df):\n    '''\n    Agregations here will be automatically normalize with robustscaler by series_id\n    '''\n    old_columns = df.columns\n    df = df.with_columns([\n        pl.col('anglez').diff().abs().rolling_sum(window_size=60).rolling_max(60).alias('anglez_diff_last_halfhour60'),\n        pl.col('anglez').diff().abs().rolling_sum(window_size=30).rolling_max(30).alias('anglez_diff_last_halfhour30'),\n        pl.col('anglez').diff().abs().rolling_sum(window_size=15).rolling_max(15).alias('anglez_diff_last_halfhour15'),\n        pl.col('anglez').diff().abs().rolling_sum(window_size=6).rolling_max(6).alias('anglez_diff_last_halfhour6'),\n    ])\n    new_columns = sorted(list(np.setdiff1d(df.columns, old_columns)))\n    return df, new_columns","metadata":{"execution":{"iopub.status.busy":"2024-06-16T16:32:38.769610Z","iopub.execute_input":"2024-06-16T16:32:38.769984Z","iopub.status.idle":"2024-06-16T16:32:38.778946Z","shell.execute_reply.started":"2024-06-16T16:32:38.769952Z","shell.execute_reply":"2024-06-16T16:32:38.778025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAIN_DIR_DATA = '/kaggle/input/child-mind-institute-detect-sleep-states'\nSLEEP_DATASET =  '/kaggle/input/sleep-ensemble'\nDIR_MODELS = SLEEP_DATASET\nDIR_YAMLS = f'{SLEEP_DATASET}/yaml'\nDIR_STATIC_FEATS = f'{SLEEP_DATASET}/static_feats'\nsys.path.append(\"/kaggle/input/sleep-ensemble/models\")\nsys.path.append(\"/kaggle/input/sleep-ensemble\")\n\n# Import basic functions fnoa\nmdl = importlib.import_module('util_functions_normal')\nif \"__all__\" in mdl.__dict__:\n    names = mdl.__dict__[\"__all__\"]\nelse:\n    names = [x for x in mdl.__dict__ if not x.startswith(\"_\")]\nglobals().update({k: getattr(mdl, k) for k in names})","metadata":{"execution":{"iopub.status.busy":"2024-06-16T16:32:38.780011Z","iopub.execute_input":"2024-06-16T16:32:38.780291Z","iopub.status.idle":"2024-06-16T16:32:38.790484Z","shell.execute_reply.started":"2024-06-16T16:32:38.780257Z","shell.execute_reply":"2024-06-16T16:32:38.789588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_prediction_fnoa(SER_ID, MODELS, MODEL_FOLD, EVAL, models_read, names_read):\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    INIT_HOUR = 17\n    seq_len = int(2880 * 1)\n    shift = int(2880 * 1)\n    offset = int(0)\n    target_cols = ['onset', 'wakeup']\n    DOWNSAMPLING = 6\n    bs = 16\n\n\n    # Filter the id\n    if EVAL:\n        series = pl.scan_parquet(f'{MAIN_DIR_DATA}/train_series.parquet').filter(pl.col('series_id') == SER_ID)\n    else:\n        series = pl.scan_parquet(f'{MAIN_DIR_DATA}/test_series.parquet').filter(pl.col('series_id') == SER_ID)\n\n    series = series.with_columns([pl.col('step').cast(pl.Int64),\n        pl.col('anglez').alias('anglez_orig'),\n        pl.col('anglez').abs().alias('anglez')])\n    #print('Read series DONE')\n\n    #### CREATE GOD FEATURE\n    series = series.with_columns([\n        pl.col('anglez').rolling_std(window_size=21, min_periods=21, center=True).\n        rolling_quantile(window_size=180, min_periods=180, quantile=0.1, center=True).shift(-180).over(\n            'series_id').alias('fe__check_c')\n    ])\n\n    series = (series.with_columns([ pl.col('fe__check_c') / pl.col('fe__check_c').max().over('series_id') ]).\n              with_columns([ pl.col('fe__check_c').clip(0.1, 1).alias('fe__check_c')]))\n\n    series = series.with_columns([\n        (((pl.col('fe__check_c') - pl.col('fe__check_c').shift(1)).abs() > 0).cumsum()).over('series_id').fill_null(\n            0).cast(pl.Int32).alias('group_c')\n    ])\n\n    series = series.with_columns([\n        (pl.col('step').count().over(['group_c', 'series_id'])).cast(pl.Int32).alias('group_len_c')\n    ])\n\n    series = series.with_columns([\n        pl.when((pl.col('fe__check_c') == 0.1) & (pl.col('group_len_c') > 360)).\n        then(1).when(pl.col('fe__check_c') > 0.1).then(0).otherwise(0).cast(pl.Int64).alias('fe__check_c')\n    ]).with_columns([\n        pl.col('fe__check_c').shift(120).over('series_id').alias('fe__check_c')\n    ])\n\n    series = series.with_columns([\n        pl.col('fe__check_c').rolling_max(window_size=241, min_periods=241, center=True)\n    ]).with_columns([\n        (((pl.col('fe__check_c') - pl.col('fe__check_c').shift(1)).abs() > 0.1).cumsum()).over('series_id').fill_null(\n            0).cast(pl.Int32).alias('group_c')\n    ]).with_columns([\n        pl.col('fe__check_c').cast(pl.Float32).fill_null(0).fill_nan(0)\n    ])\n\n    series = series.with_columns([\n        (pl.count('fe__check_c').over(['group_c', 'series_id'])).cast(pl.Int32).alias('group_len_c')\n    ])\n    series = series.with_columns([\n        pl.col('fe__check_c').cumcount().over(['group_c', 'series_id']).cast(pl.Int32).alias('group_step')\n    ])\n    series = series.with_columns([\n        pl.when(pl.col('fe__check_c') == 1).then(pl.col('group_step') / 8640).when(pl.col('fe__check_c') == 0).then(\n            -pl.col('group_step') / 8640).otherwise(0).alias('porc_step'),\n        (pl.col('group_len_c') / 17280).alias('group_len_c_norm')])\n\n    ### Normalize enmo and anglez globally\n    ##########################################\n\n    for col in ['anglez', 'enmo', 'anglez_orig']:\n        series = series.with_columns([\n            ((pl.col(col) - pl.col(col).mean()) / pl.col(col).std()).alias(col)\n        ])\n\n    #print('Normalize enmo and anglez globally DONE')\n\n    ### Calculating first instance at 17:00 locally (step_first)\n    ##########################################\n    series = (\n        series.with_columns([\n            pl.col('timestamp').str.slice(0, length=19).str.to_datetime(\"%Y-%m-%dT%H:%M:%S\").alias('timestamp_local')]).\n        with_columns([\n            pl.col('timestamp_local').dt.hour().cast(pl.Int32).alias('hour'),\n            pl.col('timestamp_local').dt.minute().cast(pl.Int32).alias('minute')\n        ]))\n    series = series.sort(by=['series_id', 'step'], descending=[True, False])\n\n    # Initialazing all days at 19 hours\n    df_sf = (\n        series.select(['series_id', 'step', 'hour', 'minute']).filter((pl.col('hour') == INIT_HOUR) & (pl.col('minute') == 0)).\n        sort(by=['series_id', 'step'], descending=[True, False]).group_by(['series_id']).agg(pl.col('step').first()).rename({'step':'step_first'}))\n    series = series.join(df_sf, on=['series_id'], how='left')\n\n    #print('Calculating first instance at 17:00 locally (step_first) DONE')\n    ### Calculating features related with the first step\n    ##############################################################\n    # rel_step: relative step respect to first step\n    # night: first complete day is night 0\n    # st_hour: calculated hour from steps\n    # st_step: relative step inside an st_hour\n    series = series.with_columns([\n        (pl.col('step') - pl.col('step_first')).alias('rel_step')\n    ]).with_columns([\n        (pl.col('rel_step') // 17280).alias('night')\n    ]).with_columns([\n        pl.when(pl.col('rel_step') >= 0).then((pl.col('rel_step') - pl.col('night') * 17280) // (60 * 12)).\n        otherwise((pl.col('rel_step') + 17280) // (60 * 12)).alias('st_hour')\n    ]).with_columns([\n        pl.when(pl.col('rel_step') >= 0).then(\n            (pl.col('rel_step') - pl.col('st_hour') * 60 * 12 - pl.col('night') * 17280)).\n        otherwise((pl.col('rel_step') + 17280 - pl.col('st_hour') * 60 * 12)).alias('st_step')\n    ])\n\n    #print('Calculating features related with the first step DONE')\n    ### Detecting noise through same values in same st_hour and st_step for same series\n    ##############################################################\n    # repeat: number of times that the value appears in the series\n    # noise: binary feature is 1 when repeat>1\n    # group_len: number of consecutive values with noise\n    # noise_removal: binary feat is 1 when is noise==1 and group_len>360\n\n    temp_df = (series.group_by(['series_id', 'st_hour', 'st_step', 'anglez']).\n               agg(pl.col('step').count().alias('repeat')))\n\n    series = series.join(temp_df, on=['series_id', 'st_hour', 'st_step', 'anglez'], how='left')\n\n    series = series.with_columns([\n        pl.when(pl.col('repeat') > 1).then(1).otherwise(0).cast(pl.Int32).alias('noise'),\n    ])\n\n    series = series.with_columns([\n        (((pl.col('noise') - pl.col('noise').shift(1)).abs() > 0).cumsum()).over('series_id').fill_null(0).cast(\n            pl.Int32).alias('group')\n    ])\n    series = series.with_columns([\n        pl.col('group').max().over('series_id').cast(pl.Int32).alias('group_max')\n    ])\n\n    series = series.with_columns([\n        (pl.col('step').count().over(['group', 'series_id'])).cast(pl.Int32).alias('group_len')\n    ])\n    series = series.with_columns([\n        pl.when((pl.col('noise') == 1) & (pl.col('group_len') > 360)).then(1).otherwise(0).cast(pl.Int32).alias(\n            'noise_removal')\n    ])\n\n    #print('Detecting noise DONE')\n    ### Downsampling series\n    ##########################\n    if DOWNSAMPLING > 1:\n        series = series.with_columns([\n            (((pl.col('step')) // DOWNSAMPLING)).alias('step')\n        ])\n\n        # Calculations\n        series = series.sort(by=['series_id', 'step'], descending=[True, False])\n        series = (series.group_by(['series_id', 'step']).agg([\n            pl.col('timestamp').first(),\n            pl.col('night').first(),\n            pl.col('anglez').mean().alias('anglez'),\n            pl.col('enmo').mean().alias('enmo'),\n            pl.col('anglez').std().alias('anglez_std_norm'),\n            pl.col('anglez_orig').std().alias('anglez_std_norm_orig'),\n            pl.col('enmo').std().alias('enmo_std_norm'),\n            pl.col('fe__check_c').max().alias('fe__check_c'),\n            pl.col('porc_step').mean().alias('porc_step'),\n            pl.col('group_len_c_norm').mean().alias('group_len_c_norm'),\n            (pl.col('noise').sum() / DOWNSAMPLING).alias('noise'),\n            (pl.col('noise_removal').sum() / DOWNSAMPLING).alias('noise_removal'),\n        ]))\n\n    else:\n        series = series.with_columns([\n            pl.col('anglez').alias('anglez_std_norm'),\n            pl.col('enmo').alias('enmo_std_norm'),\n        ])\n\n    series = series.sort(by=['series_id', 'step'], descending=[True, False])\n\n    #print('Downsampling series DONE')\n\n    ### Get basic time features\n    #################################\n    series = (series.with_columns([\n        pl.col('timestamp').str.slice(0, length=19).str.to_datetime(\"%Y-%m-%dT%H:%M:%S\").alias('timestamp_local')\n    ]).with_columns([\n        pl.col('timestamp_local').dt.minute().cast(pl.Int32).alias('minute'),\n        pl.col('timestamp_local').dt.second().cast(pl.Int32).alias('second'),\n        pl.col('timestamp_local').dt.hour().cast(pl.Int32).alias('hour'),\n        pl.col('timestamp_local').dt.day().cast(pl.Int32).alias('day'),\n        pl.col('timestamp_local').dt.year().cast(pl.Int32).alias('year'),\n        pl.col('timestamp_local').dt.weekday().cast(pl.Int32).alias('weekday'),\n    ]))\n    series = series.sort(by=['series_id', 'step'], descending=[True, False])\n    #print('Get basic time features DONE')\n\n    ### Get basic feature enginering\n    #################################\n    # anglez_equal1: +1 day lag features for detecting repetitive patterns\n    # anglez_equal2: -1 day lag features for detecting repetitive patterns\n    series = series.with_columns([\n        pl.when((pl.col('anglez') == pl.col('anglez').shift(int(17280 / DOWNSAMPLING)))).then(1).otherwise(0).alias(\n            'anglez_equal1'),\n        pl.when((pl.col('anglez') == pl.col('anglez').shift(int(-17280 / DOWNSAMPLING)))).then(1).otherwise(0).alias(\n            'anglez_equal2'),\n        pl.col('anglez_std_norm').rolling_quantile(window_size=15, quantile=0.5, center=False).over(\n            ['series_id']).alias(\n            'std_q50_left'),\n        pl.col('anglez_std_norm').reverse().rolling_quantile(window_size=15, quantile=0.5, center=False).reverse().over(\n            ['series_id']).alias(\n            'std_q50_right'),\n        pl.lit(0).alias('invert')\n    ])\n    series = series.sort(by=['series_id', 'step'], descending=[True, False])\n\n    #print('Get basic feature enginering DONE')\n    ### Calculating first instance at 17:00 locally (step_first)\n    ##########################################\n    if series.select(pl.count()).collect()['count'][0]>40:\n        df_sf = (\n            series.select(['series_id', 'step', 'hour', 'minute']).filter((pl.col('hour') == INIT_HOUR) & (pl.col('minute') == 0)).\n            sort(by=['series_id', 'step'], descending=[True, False]).group_by(['series_id']).agg(pl.col('step').first()).rename({'step':'step_first'}))\n        series = series.join(df_sf, on=['series_id'], how='left')\n        series = series.sort(by='step', descending=False)\n    else:\n        series = series.with_columns([pl.lit(0).alias('step_first')])\n\n    #print('Get first step DONE')\n    ### ADDING TARGET_ENCODING\n    ##########################################\n    # hminute: hour*120 + minute * 2 (hour_halfminute)\n    series = series.with_columns([\n        (pl.col('minute') * 2).alias('hminute'),\n    ]).with_columns([\n        pl.when(pl.col('second') < 30).then(0).when(pl.col('second') >= 30).then(1).otherwise(0).alias('half')\n    ]).with_columns([\n        pl.when(pl.col('half') == 1).then(pl.col('hminute') + 1).otherwise(pl.col('hminute')).alias('hminute')\n    ])\n    series = series.with_columns([\n        (pl.col('hour') * 120 + pl.col('hminute')).alias('hminute')\n    ])\n\n\n    # Frequency of onsets and wakeups at each hour-halfminute\n    minute_info = pl.scan_csv('/kaggle/input/sleep-ensemble/static_feats/minute_info.csv').with_columns([\n        pl.col('hminute').cast(pl.Int32),\n        pl.col('fold').cast(pl.Int32),\n        pl.col('hmin_onset').cast(pl.Float32),\n        pl.col('hmin_wakeup').cast(pl.Float32)\n    ])\n    minute_info = minute_info.group_by(['hminute']).agg([\n        pl.col('hmin_onset').mean().alias('hmin_onset'),\n        pl.col('hmin_wakeup').mean().alias('hmin_wakeup')\n    ])\n    series = series.join(minute_info, on=['hminute'], how='left')\n\n    #print('Add target encoding DONE')\n\n    series = series.with_columns(pl.lit(0).alias('onset'))\n    series = series.with_columns(pl.lit(0).alias('wakeup'))\n\n    all_features, id_info = preprocess_id(SER_ID, series, cols_f, num_cols_f,seq_len=seq_len,shift=shift,\n                                          offset=offset,target_cols=target_cols,specific_process=specific_process)\n\n    if models_read is None:\n        ALL_MODELS = []\n        ALL_NAMES = []\n        for name in MODELS:\n            #print(all_features)\n            _models, _names = load_models(DIR_YAMLS, DIR_MODELS, name, fold=MODEL_FOLD, cfg=None, all_features=all_features)\n            ALL_MODELS.append(_models)\n            ALL_NAMES.append(_names)\n    else:\n        ALL_MODELS = models_read\n        ALL_NAMES = names_read\n\n    num_array = np.concatenate(id_info[0], axis=0)\n    target_array = np.concatenate(id_info[1], axis=0)\n    mask_array = np.concatenate(id_info[2], axis=0)\n    pred_use_array = np.concatenate(id_info[3], axis=0)\n    time_array = np.concatenate(id_info[4], axis=0)\n    id_list = np.concatenate(id_info[5], axis=0)\n    df_id = pd.DataFrame()\n    df_id[\"series_id\"] = id_list\n\n    val_ = CustomDataset(num_array,\n                         mask_array,\n                         train=True,\n                         y=target_array)\n\n    val_loader = DataLoader(dataset=val_,\n                            batch_size=bs,\n                            shuffle=False, num_workers=1)\n\n    val_preds = np.ndarray((0, seq_len, 2))\n    #tk0 = tqdm(val_loader, total=len(val_loader))\n    \n    save_preds = {}\n    for idx1, mod in enumerate(ALL_MODELS):\n        for idx2, mod_fold in enumerate(mod):\n            save_preds.update({f'{idx1}_{idx2}':np.ndarray((0, seq_len, 2))})\n    \n    with torch.no_grad():\n        # Predicting on validation set\n        output = []\n        for d in val_loader:#tk0:\n            input_data_numerical_array = d['input_data_numerical_array'].to(device)\n            input_data_mask_array = d['input_data_mask_array'].to(device)\n            attention_mask = d['attention_mask'].to(device)\n\n            preds = []\n            for idx1, mod in enumerate(ALL_MODELS):\n\n                preds_fold = []\n                for idx2, mod_fold in enumerate(mod):\n                    \n                    tmp_preds1 = mod_fold(input_data_numerical_array,\n                                         input_data_mask_array,\n                                         attention_mask).sigmoid().detach().cpu().numpy()[:, :, :2]\n                    save_preds[f'{idx1}_{idx2}'] = np.concatenate([save_preds[f'{idx1}_{idx2}'], tmp_preds1], axis=0)\n                    \n   \n        save_preds_q = save_preds\n        num=0\n        for i, j in save_preds.items():\n            if num==0:\n                preds_mean = save_preds[i]/len(save_preds)\n            else:\n                preds_mean += save_preds[i]/len(save_preds)\n            num += 1\n                \n            save_preds_q[i][:,:,0] = np.where(save_preds[i][:,:,0]>0.1, save_preds[i][:,:,0], 0)\n            save_preds_q[i][:,:,1] = np.where(save_preds[i][:,:,1]>0.1, save_preds[i][:,:,1], 0)\n            \n            preds_mean += save_preds_q[i]\n            \n        gc.collect()\n        val_preds = np.concatenate([val_preds, preds_mean], axis=0)\n\n\n    pred_valid_index = (mask_array == 1) & (pred_use_array == 1)\n\n    # Steps\n    sol_time = time_array[pred_valid_index].ravel()\n\n    # Ids\n    val_id_array_rep = np.repeat(id_list[:, np.newaxis], num_array.shape[1], 1)\n    sol_id = val_id_array_rep[pred_valid_index].ravel()\n\n    # Predictions\n    sol_onset = val_preds[pred_valid_index][:, 0].ravel()\n    sol_wakeup = val_preds[pred_valid_index][:, 1].ravel()\n\n    # Features\n    sol_step_first = num_array[pred_valid_index][:, 0].ravel()\n    sol_hour = num_array[pred_valid_index][:, 1].ravel()\n    sol_minute = num_array[pred_valid_index][:, 2].ravel()\n\n    onset = {'series_id': sol_id, 'step': sol_time, 'event': 'onset', 'score': sol_onset,\n             'sf': sol_step_first, 'hour': sol_hour, 'minute': sol_minute}\n    onset = pd.DataFrame(onset)\n\n    wakeup = {'series_id': sol_id, 'step': sol_time, 'event': 'wakeup', 'score': sol_wakeup,\n              'sf': sol_step_first, 'hour': sol_hour, 'minute': sol_minute}\n    wakeup = pd.DataFrame(wakeup)\n\n    submission = pd.concat([onset, wakeup], axis=0)\n\n    # Create submission from raw predictions\n    submission = submission.sort_values(by='score', ascending=False)\n    submission['step'] = (submission['step'] * DOWNSAMPLING).astype(int)\n    submission['sf'] = (submission['sf'] * DOWNSAMPLING).astype(int)\n    submission['num_night'] = (1 + ((submission['step'] - submission['sf']) // 17280)).astype(int)\n    submission['hour'] = (submission['hour']).astype(int)\n    submission['minute'] = (submission['minute']).astype(int)\n    \n    return submission, ALL_MODELS, ALL_NAMES","metadata":{"execution":{"iopub.status.busy":"2024-06-16T16:32:38.792887Z","iopub.execute_input":"2024-06-16T16:32:38.793287Z","iopub.status.idle":"2024-06-16T16:32:38.869308Z","shell.execute_reply.started":"2024-06-16T16:32:38.793257Z","shell.execute_reply":"2024-06-16T16:32:38.868410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postprocess_from_god(df):\n    \n    df['c1'] = df.groupby(['series_id','event', 'num_night'])['score'].transform('max')\n    df['c2'] = np.where(df['score'] == df['c1'], df['step'], 0)\n    df['c2'] = df.groupby(['series_id','event', 'num_night'])['c2'].transform('max')\n    df['c3'] = np.where(np.abs(df['step'] - df['c2']) < 18, df['score'], df['score'] * 0.5)\n    df['score'] = df['c3']\n    \n    df = (df.pivot(index=['series_id', 'step', 'sf', 'hour', 'minute', 'num_night'], columns=\"event\",values=\"score\").sort_index(level=[1, 0]))\n    df = df.reset_index()\n    # df = pd.DataFrame(observations, columns=[\"video_id\", \"frame_id\", \"pred_challenge\", \"pred_throwin\", \"pred_play\"])\n\n    o_values = df['onset'].copy()\n    w_values = df['wakeup'].copy()\n\n\n    df['wakeup'] = df['wakeup'] - o_values\n    df['onset'] = (df['onset'] - w_values)\n    \n    val_df = df.sort_values([\"series_id\", \"step\"]).copy().reset_index(drop=True)\n\n    window_first = 5\n    # window_size = [15, 3, 5]\n    window_size = [5, 7]\n    ignore_width = [11, 11]\n\n    val_df[\"onset\"] = val_df[\"onset\"] ** 1\n    val_df[\"wakeup\"] = val_df[\"wakeup\"] ** 1\n\n    train_df = []\n    # gdf = df.loc[df['series_id']==video_id]\n    for video_id, gdf in val_df.groupby('series_id'):\n        for k, event in enumerate(['onset', 'wakeup']):\n            # Moving averages are used to smooth out the data.\n            prob_arr = gdf[f\"{event}\"].rolling(window=window_first, center=True).mean().fillna(0).rolling(\n                window=window_size[k], center=True).mean().fillna(0).values\n            gdf['rolling_prob'] = prob_arr\n            sort_arr = np.argsort(-prob_arr)\n            rank_arr = np.empty_like(sort_arr)\n            rank_arr[sort_arr] = np.arange(len(sort_arr))\n            # index list for detected action\n            idx_list = []\n            for i in range(len(prob_arr)):\n                this_idx = sort_arr[i]\n                if this_idx >= 0:\n                    # Add maximam index to index_list\n                    idx_list.append(this_idx)\n                    for parity in (-1, 1):\n                        for j in range(1, ignore_width[k] + 1):\n                            ex_idx = this_idx + j * parity\n                            if ex_idx >= 0 and ex_idx < len(prob_arr):\n                                # Exclude frames near this_idx where the action occurred.\n                                sort_arr[rank_arr[ex_idx]] = -1\n            this_df = gdf.iloc[idx_list].reset_index(drop=True).reset_index().rename(columns={'index': 'rank'})[\n                ['rank', 'series_id', 'step','num_night', 'rolling_prob']]\n            this_df['event'] = event\n\n#             this_df = this_df.loc[this_df['rolling_prob'] > np.quantile(this_df['rolling_prob'], 0.85)].reset_index(drop=True)\n\n            train_df.append(this_df)\n\n    train_df = pd.concat(train_df)\n    train_df1 = train_df.copy()\n\n    train_df1['step'] = train_df1['step'] - 0.5\n    train_df1['time'] = train_df1['step'] / 12\n    # train_df['score'] = 0.5 * train_df['rolling_prob'] + 0.5 * (1 / (train_df['rank'] + 1))\n    train_df1['score'] = train_df1['rolling_prob']\n    \n    \n    train_df1 = train_df1[train_df1['rank'] < train_df1.groupby(['series_id', 'event'])['num_night'].transform('nunique') * 20]\n    \n    train_df1 = train_df1.loc[train_df1['step']>0].reset_index(drop=True)\n    \n    return train_df1","metadata":{"execution":{"iopub.status.busy":"2024-06-16T16:32:38.870508Z","iopub.execute_input":"2024-06-16T16:32:38.870870Z","iopub.status.idle":"2024-06-16T16:32:38.889230Z","shell.execute_reply.started":"2024-06-16T16:32:38.870841Z","shell.execute_reply":"2024-06-16T16:32:38.888330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODELS = ['gru_abs','gru_abs_short','gru_abs_clean','gru_abs_new']\nMODEL_FOLD = None # Put None for taking models of all folds\nLOCAL_VAL = False # Put tu False when making submission\n\n# with open('/kaggle/input/sleep-ensemble/folds_new.pickle', 'rb') as handle:\n#     folds_div = pickle.load(handle)\n\n# Read all de Ids\nseries = pl.read_parquet(f'/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id'])\nUNIQUE_IDS = list(series['series_id'].unique())#['038441c925bb', '03d92c9f6f8a', '0402a003dae9']#folds_div[0][:5]#\nprint(UNIQUE_IDS)\n\n# Start loop making predictions\nALL_PREDS = []\nmodels_read = None\nnames_read = None\n#RAW_PREDS = []\nfor ser_id in tqdm(UNIQUE_IDS):\n    \n    #Make raw pred\n    tmp, models_read, names_read = make_prediction_fnoa(ser_id,MODELS,MODEL_FOLD,EVAL=LOCAL_VAL,models_read=models_read,names_read=names_read)\n\n    #RAW_PREDS.append(tmp)\n    # Make postprocess\n    preds_fnoa = postprocess_from_god(tmp)\n    ALL_PREDS.append(preds_fnoa[['series_id', 'step', 'event', 'score']])\n    \n# Concat\nALL_PREDS = pd.concat(ALL_PREDS)\n\n# Prepare sub format and save\nsubmission = ALL_PREDS.sort_values(by=['series_id', 'step'], ascending=True).reset_index(drop=True)\nsubmission['row_id'] = submission.index\nsubmission[['row_id', 'series_id', 'step', 'event', 'score']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-16T16:32:38.891188Z","iopub.execute_input":"2024-06-16T16:32:38.891558Z","iopub.status.idle":"2024-06-16T16:32:43.969410Z","shell.execute_reply.started":"2024-06-16T16:32:38.891527Z","shell.execute_reply":"2024-06-16T16:32:43.968414Z"},"trusted":true},"execution_count":null,"outputs":[]}]}