{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":153803982,"sourceType":"kernelVersion"}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport scipy as sp\nimport matplotlib.pyplot as plt\nimport os\nimport copy\nfrom IPython.utils import io\nfrom tqdm import tqdm\nfrom multiprocessing import Pool\nimport gc\nimport time\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import average_precision_score\nimport numba as nb\nfrom numba.core.errors import NumbaDeprecationWarning, NumbaPendingDeprecationWarning\nimport warnings\nwarnings.filterwarnings('ignore', category=pd.errors.PerformanceWarning)\nwarnings.filterwarnings('ignore', category=UserWarning)\nwarnings.filterwarnings('ignore', category=RuntimeWarning)\nwarnings.simplefilter('ignore', category=NumbaDeprecationWarning)\nwarnings.simplefilter('ignore', category=NumbaPendingDeprecationWarning)","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:07:56.877193Z","iopub.execute_input":"2023-11-20T17:07:56.878021Z","iopub.status.idle":"2023-11-20T17:08:02.364962Z","shell.execute_reply.started":"2023-11-20T17:07:56.877966Z","shell.execute_reply":"2023-11-20T17:08:02.363388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"class CFG:\n    ### data cleaning\n    remove_series=['60d31b0bec3b','e4500e7e19e1','f56824b503a0','4feda0596965',\n                '10469f6765bf','13b4d6a01d27','60e51cad2ffb']\n    remove_series_down_weight=0.\n    # target data quality check\n    target_check_is_fake_rate_thr=0.05\n    target_check_onset_valid_inactive_rate_thrs=(1,0.8) \n    target_check_wakeup_valid_inactive_rate_thrs=(0,0.99)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.367456Z","iopub.execute_input":"2023-11-20T17:08:02.367832Z","iopub.status.idle":"2023-11-20T17:08:02.375749Z","shell.execute_reply.started":"2023-11-20T17:08:02.3678Z","shell.execute_reply":"2023-11-20T17:08:02.374474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helpers","metadata":{}},{"cell_type":"code","source":"def reduce_mem(df,same=False):\n    for col,d in df.dtypes.items():\n        if d not in [np.int64,np.int32,np.float64,np.float32]:\n            continue\n        elif col in ['step','pstart','pend']:\n            df[col]=df[col].astype('int32')\n        elif same:\n            df[col]=df[col].astype('float32')\n        elif d==np.float64:\n            df[col]=df[col].astype('float32')\n        elif d==np.int64:\n            df[col]=df[col].astype('int32')\n    gc.collect()\n    return","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.377441Z","iopub.execute_input":"2023-11-20T17:08:02.377805Z","iopub.status.idle":"2023-11-20T17:08:02.391018Z","shell.execute_reply.started":"2023-11-20T17:08:02.377775Z","shell.execute_reply":"2023-11-20T17:08:02.389829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def comp_metric(target_evts,prediction,score_column='score',return_table=False,print_table=False):\n    '''\n    competition metric, faster.\n    '''\n    prediction=prediction.sort_values(score_column,ascending=False)\n    target_evts=target_evts[~target_evts.step.isna()][['series_id','event','step']].copy()\n    prediction=prediction[['series_id','event','step',score_column]].copy()\n    prediction['score']=prediction[score_column]\n\n    target_evts=target_evts.merge(\n        prediction.groupby(['series_id','event']).agg(pred_steps=('step',list)),\n        on=['series_id','event'],how='left'\n    )\n    target_evts['best_match_idx']=target_evts[['step','pred_steps']].apply(lambda x:np.argmin(np.abs(np.array(x[1])-x[0])),axis=1)\n    with_match_cond=~target_evts.pred_steps.isna()\n    target_evts.loc[with_match_cond,'best_match_step']=target_evts.loc[with_match_cond,['best_match_idx','pred_steps']].apply(lambda x:x[1][int(x[0])],axis=1)\n    target_evts.loc[with_match_cond,'best_match_gap']=np.abs(target_evts.loc[with_match_cond,'step']-target_evts.loc[with_match_cond,'best_match_step'])\n    exceed_max_gap=~(target_evts['best_match_gap']<360)\n    target_evts.loc[exceed_max_gap,'best_match_gap']=np.nan\n    target_evts['merge_key']=target_evts['best_match_step']\n    prediction['merge_key']=prediction['step']\n    prediction=prediction.merge(target_evts[~target_evts.best_match_gap.isna()][['series_id','event','merge_key','best_match_gap']],on=['series_id','event','merge_key'],how='left')\n    if (~prediction['best_match_gap'].isna()).sum()!=(~target_evts['best_match_gap'].isna()).sum():\n        print('bijective matching failed...')\n    case_scores=[]\n    eval_gaps=[12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n    score_table={'event':[],'tol':[],'score':[],'pos_recall':[]}\n    for event in ['onset','wakeup']:\n        target_event_cond=target_evts.event==event\n        pred_event_cond=prediction.event==event\n        for gap in eval_gaps:\n            unmatched_pos_num=(~(target_evts.loc[target_event_cond,'best_match_gap']<gap)).sum()\n            pred_label=(prediction.loc[pred_event_cond,'best_match_gap']<gap).values.astype('int64')\n            pred_score=prediction.loc[pred_event_cond,'score'].values\n            pos_recall=(target_event_cond.sum()-unmatched_pos_num)/target_event_cond.sum()\n            case_score=average_precision_score(pred_label,pred_score)*pos_recall\n            case_scores.append(case_score)\n            score_table['event'].append(event)\n            score_table['tol'].append(gap)\n            score_table['score'].append(case_score)\n            score_table['pos_recall'].append(pos_recall)\n    score_table=pd.DataFrame(score_table)\n    if print_table:\n        display(score_table.round(3))\n        display(score_table.groupby(['event']).mean().round(3))\n    if return_table:\n        return np.mean(case_scores),score_table\n    # more summary stats TBD\n    return np.mean(case_scores)\n\ndef keep_group_max_target(df,target_col):\n    '''\n    set score to zero except for the one closest to true target\n    '''\n    df['candidate_step']=df['step'].copy()\n    df['group_max_target']=df.groupby(['series_id','event','nearest_target_step'])[target_col].transform('max')\n    df.loc[df[target_col]!=df['group_max_target'],'candidate_step']=-1\n    df['group_candidate_max_step']=df.groupby(['series_id','event','nearest_target_step'])['candidate_step'].transform('max')\n    df.loc[(df['step']!=df['group_candidate_max_step']),target_col]=0\n    del df['group_max_target']\n    del df['candidate_step']\n    del df['group_candidate_max_step']\n    return\n\ndef DSS_CV_split(data_info,k=5,split_by_event=False,seed=7):\n    # data_info with columns: series_id,event\n    sids=data_info.series_id.unique()\n    val_batch_size=int(np.ceil(len(sids)/k))\n    np.random.seed(seed)\n    np.random.shuffle(sids)\n    if split_by_event:\n        event_conds=[data_info['event']=='onset',data_info['event']=='wakeup']\n        tasks=['onset','wakeup']\n    else:\n        event_conds=[~data_info['event'].isna()]\n        tasks=['all']\n    split_pairs=[]\n    for i,event_cond in enumerate(event_conds):\n        for j in range(k):\n            val_sids=set(sids[j*val_batch_size:(j+1)*val_batch_size])\n            sid_cond=~data_info.series_id.isin(val_sids)\n            train_index=data_info[event_cond&sid_cond].index.tolist()\n            val_index=data_info[event_cond&(~sid_cond)].index.tolist()\n            split_pairs.append((tasks[i],train_index,val_index))\n\n    return split_pairs\n\ndef get_tol_group(gap):\n    gap=np.abs(gap)\n    for tol in [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]:\n        if gap<tol:\n            return tol\n    return 10000000\n\ndef get_tol_width(tol):\n    tols=[0,12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n    for i in range(1,len(tols)):\n        if tol<tols[i]:\n            return tols[i]-tols[i-1]\n    return 1e9","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.394924Z","iopub.execute_input":"2023-11-20T17:08:02.395348Z","iopub.status.idle":"2023-11-20T17:08:02.431376Z","shell.execute_reply.started":"2023-11-20T17:08:02.395315Z","shell.execute_reply":"2023-11-20T17:08:02.430087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* interval process functions","metadata":{}},{"cell_type":"code","source":"def merge_points_to_intervals(point_idx):\n    if len(point_idx)==0:\n        return []\n    point_idx=sorted(point_idx)\n    intervals=[]\n    last=point_idx[0]\n    left=point_idx[0]\n    for i in point_idx:\n        if i-last>1:\n            right=last+1\n            intervals.append((left,right))\n            left=i\n        last=i\n    right=last+1\n    intervals.append((left,right))\n    return intervals\n\ndef filter_short_intervals(intervals,min_len=30*60//5):\n    intervals=sorted(intervals)\n    filtered_intervals=[]\n    for (i,j) in intervals:\n        if (j-i)>=min_len:\n            filtered_intervals.append((i,j))\n    return filtered_intervals\n\ndef merge_close_intervals(intervals,max_gap=30*60//5,union=False,union_max_gap=6*60*12,union_ratio=0.3):\n    if len(intervals)==0:\n        return []\n    intervals=sorted(intervals)\n    merged_intervals=[]\n    current_interval=intervals[0]\n    for (i,j) in intervals[1:]:\n        # if can merge\n        gap=i-current_interval[1]\n        merge_cond=gap<max_gap\n        if union:\n            cover_len=j-current_interval[0]\n            merge_cond=merge_cond or ((gap<union_max_gap) and (gap/(cover_len-gap)<union_ratio))\n        if merge_cond:\n            current_interval=(current_interval[0],j)\n        else:\n            merged_intervals.append(current_interval)\n            current_interval=(i,j)\n    merged_intervals.append(current_interval)\n    return merged_intervals\n\n\ndef filter_close_intervals(intervals,large_thr=5*12,allowed_gap=5*12):\n    res=[]\n    for (i,j) in intervals:\n        if len(res)==0 or (i-res[-1][1])>allowed_gap or min((j-i),(res[-1][1]-res[-1][0]))>=large_thr:\n            res.append((i,j))\n        elif (j-i)>(res[-1][1]-res[-1][0]):\n            res[-1]=(i,j)\n    return res\n\ndef merge_close_intervalsV3(intervals,max_gap=30*60//5,skip_len=5*60//5,keep_skiped=True):\n    if len(intervals)==0:\n        return []\n    intervals=sorted(intervals)\n    current_interval=None\n    merged_intervals=[]\n    for idx in range(len(intervals)):\n        if (intervals[idx][1]-intervals[idx][0])>=skip_len:\n            current_interval=intervals[idx]\n            break\n        elif keep_skiped:\n            merged_intervals.append(intervals[idx])\n    skiped_intervals=[]\n    for (i,j) in intervals[idx+1:]:\n        if (j-i)<skip_len:\n            if keep_skiped:\n                skiped_intervals.append((i,j))\n        elif (i-current_interval[1])<max_gap:\n            current_interval=(current_interval[0],j)\n            skiped_intervals=[]\n        else:\n            merged_intervals.append(current_interval)\n            merged_intervals.extend(skiped_intervals)\n            skiped_intervals=[]\n            current_interval=(i,j)\n    if current_interval is not None:\n        merged_intervals.append(current_interval)\n    merged_intervals.extend(skiped_intervals)\n    return merged_intervals\n\ndef keep_max_interval_within_day(intervals):\n    if len(intervals)==0:\n        return []\n    res=[]\n    for (i,j) in intervals:\n        if len(res)==0 or (j-res[-1][0])>720*24:\n            res.append((i,j))\n        elif (j-i)>(res[-1][1]-res[-1][0]):\n            res[-1]=(i,j)\n    return res","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.433593Z","iopub.execute_input":"2023-11-20T17:08:02.434346Z","iopub.status.idle":"2023-11-20T17:08:02.465465Z","shell.execute_reply.started":"2023-11-20T17:08:02.434303Z","shell.execute_reply":"2023-11-20T17:08:02.463944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* mainly based on description in GGIR website:https://cran.r-project.org/web/packages/GGIR/vignettes/GGIR.html#4_Inspecting_the_results","metadata":{}},{"cell_type":"code","source":"def mark_is_fake(df,exclude_inactive=False):\n    '''\n    mark suspicious fake data by interpolation\n    '''\n    detect_window_size=5*60//5\n    df['fake_stats']=df['anglez'].rolling(detect_window_size,center=True).mean().round(6)\n    df['is_fake']=df.groupby('fake_stats')['step'].transform('size')>1\n    df['fake_stats2']=df['anglez'].rolling(detect_window_size,center=True).std()\n    df['is_fake']&=df['fake_stats2']>0.5\n    ##############\n    fake_idx=np.where(df['is_fake'])[0] \n    fake_intervals=merge_points_to_intervals(fake_idx)\n    fake_intervals=filter_short_intervals(fake_intervals,min_len=3)\n    fake_intervals=merge_close_intervals(fake_intervals,max_gap=0,union=True,union_max_gap=3*60*12,union_ratio=0.8)\n    df['is_fake']=False\n    for (i,j) in fake_intervals:\n        df.loc[max(0,i-detect_window_size//2):(j-1+detect_window_size//2),'is_fake']=True\n    return\n\ndef mark_fake_extension(df,merge_close=False):\n    '''\n    mark fake region extended by inactive period\n    '''\n    if 'is_fake' not in df.columns:\n        return\n    key='anglez_abs_diff'\n    inactive_thr=5\n    inactive_idx=np.where((df[key].values<=inactive_thr)&(~df['is_fake']))[0]\n    inactive_intervals=merge_points_to_intervals(inactive_idx)\n    inactive_intervals=[(max(i,0),min(j,len(df)-1)) for (i,j) in inactive_intervals]\n    inactive_intervals=filter_short_intervals(inactive_intervals,min_len=5*12)\n    if merge_close:\n        inactive_intervals=merge_close_intervals(inactive_intervals,max_gap=12)\n    fake_detect_gap=2\n    vec=df['is_fake'].astype('float64')\n    df['fake_extension']=0\n    for i,j in inactive_intervals:\n        if vec[max(0,i-fake_detect_gap):i].sum()>0 or vec[j:(j+fake_detect_gap)].sum()>0:\n            df.loc[i:(j-1),'fake_extension']=1\n    return\n\ndef heuristic_method1(train_session,center_stats='enmo_abs_diff'):\n    '''\n    L5+/-12(not exactly)\n    '''\n    train_session['tmp']=train_session[center_stats].rolling(5*60*12,center=True,min_periods=5*60*6).mean()\n    train_session['tmp_min_in_24h']=train_session['tmp'].rolling(24*60*12,center=True,min_periods=24*60*6).min()\n    train_session['tmp_center']=(train_session['tmp']==train_session['tmp_min_in_24h']).astype('int32')\n    cond=train_session['anglez_abs_diff']<5\n    cond&=train_session['is_fake'].astype('int32')!=1\n    inactive_idx=np.where(cond)[0]\n    inactive_intervals=merge_points_to_intervals(inactive_idx)\n    inactive_intervals=[(max(i,0),min(j,len(train_session)-1)) for (i,j) in inactive_intervals]\n    inactive_intervals=filter_short_intervals(inactive_intervals,min_len=5*12)\n    inactive_intervals=merge_close_intervals(inactive_intervals,max_gap=30*12)\n    vec=train_session['tmp_center']\n    inactive_intervals=[(i,j) for (i,j) in inactive_intervals if vec[i:j].sum()>0]\n    inactive_intervals=keep_max_interval_within_day(inactive_intervals)\n    del train_session['tmp']\n    del train_session['tmp_min_in_24h']\n    del train_session['tmp_center']\n    return inactive_intervals\n\n\ndef heuristic_method2(train_session):\n    '''\n    HDCZA\n    '''\n    train_session['h2_day']=365*train_session['timestamp'].dt.year+train_session['timestamp'].dt.day_of_year\n    train_session.loc[train_session['timestamp'].dt.hour<12,'h2_day']-=1\n    train_session['aad_5min_median_smooth']=train_session['anglez_abs_diff'].rolling(5*12).median()\n    train_session['h2_day_thr']=train_session.groupby('h2_day')['aad_5min_median_smooth'].transform('quantile',0.1)*15\n    inactive_idx=np.where(train_session['aad_5min_median_smooth']<=train_session['h2_day_thr'])[0]\n    inactive_intervals=merge_points_to_intervals(inactive_idx)\n    inactive_intervals=[(max(i,0),min(j,len(train_session)-1)) for (i,j) in inactive_intervals]\n    inactive_intervals=filter_short_intervals(inactive_intervals,min_len=30*12)\n    inactive_intervals=merge_close_intervals(inactive_intervals,max_gap=60*12)\n    inactive_intervals_=[]\n    for (i,j) in inactive_intervals:\n        if len(inactive_intervals_)==0 or (j-inactive_intervals_[-1][0])>24*60*12:\n            inactive_intervals_.append((i,j))\n        elif (j-i)>(inactive_intervals_[-1][1]-inactive_intervals_[-1][0]):\n            inactive_intervals_[-1]=(i,j)\n    inactive_intervals=inactive_intervals_\n    del train_session['h2_day']\n    del train_session['h2_day_thr']\n    return inactive_intervals","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.467391Z","iopub.execute_input":"2023-11-20T17:08:02.468245Z","iopub.status.idle":"2023-11-20T17:08:02.596136Z","shell.execute_reply.started":"2023-11-20T17:08:02.468208Z","shell.execute_reply":"2023-11-20T17:08:02.594394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def summary_session_info(train_data):\n    '''\n    used to idx and loop over train_series\n    '''\n    start_df=train_data['series_id'].drop_duplicates(keep='first')\n    start_df=start_df.reset_index()\n    start_df.columns=['start_idx','series_id']\n    end_df=train_data['series_id'].drop_duplicates(keep='last')\n    end_df=end_df.reset_index()\n    end_df.columns=['end_idx','series_id']\n    session_info=start_df.merge(end_df,on='series_id',how='left')\n    session_info['start_idx']=session_info['start_idx'].astype('int64')\n    session_info['end_idx']=session_info['end_idx'].astype('int64')\n    return session_info","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.598063Z","iopub.execute_input":"2023-11-20T17:08:02.598852Z","iopub.status.idle":"2023-11-20T17:08:02.60869Z","shell.execute_reply.started":"2023-11-20T17:08:02.598805Z","shell.execute_reply":"2023-11-20T17:08:02.606896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def boundary_dummy_extending(preds):\n    '''\n    shift +/-720 as new candidates when no other candidates in 720*2 scope\n    '''\n    extended_preds=[]\n    for (sid,event),sub_df in tqdm(preds.groupby(['series_id','event'])):\n        last_step=0\n        sub_df=sub_df.sort_values('step')\n        for i,row in sub_df.iterrows():\n            step_gap=row['step']-last_step\n            if step_gap>720*2:\n                row_copy=row.copy()\n                row_copy['motivation']=2\n                row_copy['step']=last_step+720\n                extended_preds.append(row_copy.to_frame().T)\n                row_copy=row.copy()\n                row_copy['motivation']=2\n                row_copy['step']=row_copy['step']-720\n                extended_preds.append(row_copy.to_frame().T)\n            extended_preds.append(row.to_frame().T)\n            last_step=row['step']\n    extended_preds=pd.concat(extended_preds,axis=0).reset_index(drop=True)\n    for col,type_ in preds.dtypes.items():\n        extended_preds[col]=extended_preds[col].astype(type_)\n    return extended_preds","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.610919Z","iopub.execute_input":"2023-11-20T17:08:02.612667Z","iopub.status.idle":"2023-11-20T17:08:02.625733Z","shell.execute_reply.started":"2023-11-20T17:08:02.612492Z","shell.execute_reply":"2023-11-20T17:08:02.624759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def candidate_generation(train_data,session_info,train_evts=None):\n    '''\n    generate candidates based on heuristic rules\n    '''\n    preds=[]\n    for i in session_info.index:\n        sid,start_idx,end_idx=session_info.loc[i,['series_id','start_idx','end_idx']]\n        train_session=train_data.loc[start_idx:end_idx].reset_index(drop=True).copy()\n        target_session=[]\n        if train_evts is not None:\n            target_session=train_evts[(train_evts.series_id==sid)&(~train_evts.step.isna())]\n        ####################\n        # addtional stats\n        train_session['enmo_abs_diff']=train_session['enmo'].diff().abs().fillna(0).values\n        train_session['anglez_abs_diff']=train_session['anglez'].diff().abs().fillna(0).values\n        mark_is_fake(train_session)\n        mark_fake_extension(train_session)\n        ############################## RECALL METHOD ###################################################\n        # motivated by inactive interval\n        key='anglez_abs_diff'\n        inactive_thr=5\n        inactive_idx=np.where((train_session[key].values<=inactive_thr)&(~train_session['is_fake']))[0]\n        inactive_intervals=merge_points_to_intervals(inactive_idx)\n        inactive_intervals=[(max(i,0),min(j,len(train_session)-1)) for (i,j) in inactive_intervals]\n        # onset\n        inactive_intervals_onset=merge_close_intervalsV3(inactive_intervals,max_gap=6,skip_len=5*12)\n        inactive_intervals_onset=filter_short_intervals(inactive_intervals_onset,min_len=5*12)\n        # wakeup\n        inactive_intervals_wakeup=merge_close_intervalsV3(inactive_intervals,max_gap=3*12,skip_len=5*12)\n        inactive_intervals_wakeup=filter_short_intervals(inactive_intervals_wakeup,min_len=3*12)\n        inactive_intervals_wakeup=filter_close_intervals(inactive_intervals_wakeup,large_thr=5*12,allowed_gap=5*12)\n        # motivated by fake extension interval\n        fake_extension_idx=np.where(train_session['fake_extension']==1)[0]\n        fake_extension_intervals=merge_points_to_intervals(fake_extension_idx)\n        fake_extension_intervals=[(max(i,0),min(j,len(train_session)-1)) for (i,j) in fake_extension_intervals]\n        ###############################################################################################################\n        pred={'step':[],'event':[],'pstart':[],'pend':[],'motivation':[]}\n        for (i,j) in inactive_intervals_onset:\n            motivation=0\n            event='onset'\n            pred['step'].append(i)\n            pred['event'].append(event)\n            pred['pstart'].append(i)\n            pred['pend'].append(j)\n            pred['motivation'].append(motivation)\n        for (i,j) in inactive_intervals_wakeup:\n            motivation=0\n            event='wakeup'\n            # wakeup\n            pred['step'].append(j)\n            pred['event'].append(event)\n            pred['pstart'].append(i)\n            pred['pend'].append(j)\n            pred['motivation'].append(motivation)\n        for (i,j) in fake_extension_intervals:\n            # onset\n            pred['step'].append(j)\n            pred['event'].append('onset')\n            pred['pstart'].append(i)\n            pred['pend'].append(j)\n            pred['motivation'].append(1)\n            # wakeup\n            pred['step'].append(i)\n            pred['event'].append('wakeup')\n            pred['pstart'].append(i)\n            pred['pend'].append(j)\n            pred['motivation'].append(1)\n        pred=pd.DataFrame(pred)\n        for col in ['step','pstart','pend']:\n            pred[col]=pred[col].astype('int64')\n        pred['row_id']=pred.index\n        pred['series_id']=sid\n        pred['score']=1\n        preds.append(pred)\n        ####################\n    preds=pd.concat(preds)\n    preds=preds.drop_duplicates(subset=['event','series_id','step']).reset_index(drop=True)\n    preds=boundary_dummy_extending(preds)\n    preds['row_id']=preds.index\n    ##############\n    if train_evts is not None:\n        score=comp_metric(train_evts,preds)\n        print('over all:',score)\n    return preds\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.62761Z","iopub.execute_input":"2023-11-20T17:08:02.628914Z","iopub.status.idle":"2023-11-20T17:08:02.654629Z","shell.execute_reply.started":"2023-11-20T17:08:02.628872Z","shell.execute_reply":"2023-11-20T17:08:02.653744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_target_quality_info(train_data,session_info,train_evts=None):\n    if train_evts is None:\n        return\n    for i in session_info.index:\n        sid,start_idx,end_idx=session_info.loc[i,['series_id','start_idx','end_idx']]\n        target_session=train_evts[train_evts.series_id==sid]\n        target_session=target_session[~target_session.step.isna()]\n        if len(target_session)==0:\n            continue\n        train_session=train_data.loc[start_idx:end_idx].reset_index(drop=True).copy()\n        train_session['anglez_abs_diff']=train_session['anglez'].diff().abs().fillna(0).values\n        mark_is_fake(train_session)\n        mark_fake_extension(train_session)\n        #####################\n        key='anglez_abs_diff'\n        inactive_thr=5\n        inactive_idx=np.where((train_session[key].values<inactive_thr)&(~train_session['is_fake'])&(~train_session['fake_extension']))[0]\n        inactive_vec=np.zeros(len(train_session))\n        inactive_vec[inactive_idx]=1\n        is_fake_vec=(train_session['is_fake']|train_session['fake_extension']).values\n        for j in target_session.index:\n            step=target_session.loc[j,'step']\n            step=int(step)\n            train_evts.loc[j,'left_30min_inactive_rate']=inactive_vec[max(0,step-360):step].mean()\n            train_evts.loc[j,'right_30min_inactive_rate']=inactive_vec[step:(step+360)].mean()\n            train_evts.loc[j,'2h_fake_rate']=is_fake_vec[max(0,step-720):(step+720)].mean()\n    train_evts['good']=1\n    # mainly based on wrong active, wrong inactive can be unlabeled fake&extension\n    bad_cond=train_evts.step.isna()\n    ## onset\n    bad_onset_cond=bad_cond.copy()\n    bad_onset_cond|=train_evts['left_30min_inactive_rate']>CFG.target_check_onset_valid_inactive_rate_thrs[0]\n    bad_onset_cond|=train_evts['right_30min_inactive_rate']<CFG.target_check_onset_valid_inactive_rate_thrs[1]\n    bad_cond|=(bad_onset_cond)&(train_evts['event']=='onset')&(train_evts['2h_fake_rate']<CFG.target_check_is_fake_rate_thr)\n    ## wakeup\n    bad_wakeup_cond=bad_cond.copy()\n    bad_wakeup_cond|=train_evts['left_30min_inactive_rate']<CFG.target_check_wakeup_valid_inactive_rate_thrs[0]\n    bad_wakeup_cond|=train_evts['right_30min_inactive_rate']>CFG.target_check_wakeup_valid_inactive_rate_thrs[1]\n    bad_cond|=(bad_wakeup_cond)&(train_evts['event']=='wakeup')&(train_evts['2h_fake_rate']<CFG.target_check_is_fake_rate_thr)\n    train_evts.loc[bad_cond,'good']=0\n    return","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.658514Z","iopub.execute_input":"2023-11-20T17:08:02.658895Z","iopub.status.idle":"2023-11-20T17:08:02.678435Z","shell.execute_reply.started":"2023-11-20T17:08:02.658862Z","shell.execute_reply":"2023-11-20T17:08:02.67706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_target_info(preds,train_evts=None,session_info=None):\n    if train_evts is None:\n        return preds\n    train_evts=train_evts[~train_evts.step.isna()]\n    ### more information about target distribution,used to filter/weight training(not test or val!) data\n    # 1.series with unlabeled tail\n    train_evts=train_evts.merge(session_info[['series_id','start_idx','end_idx']],\n                                on='series_id',how='left'\n                               )\n    train_evts['max_step']=train_evts['end_idx']-train_evts['start_idx']\n    train_evts['last_step']=train_evts.groupby('series_id')['step'].transform(np.nanmax)\n    train_evts['first_step']=train_evts.groupby('series_id')['step'].transform(np.nanmin)\n    train_evts['empty_len']=train_evts['max_step']-train_evts['last_step']\n    preds=preds.merge(train_evts[['series_id','empty_len','last_step']].drop_duplicates('series_id'),\n                      on='series_id',how='left'\n                     )\n    preds['empty_len']=preds['empty_len'].fillna(1e9)\n    preds['last_step']=preds['last_step'].fillna(0)\n    ### add target info\n    target_steps=train_evts[~train_evts.step.isna()].groupby(['series_id','event']).agg(target_steps=('step',list)).reset_index()\n    preds=preds.merge(target_steps,on=['event','series_id'],how='left')\n    preds['step_arg_min']=preds[['step','target_steps']].apply(lambda x:np.argmin(np.abs(np.array(x[1])-x[0])),axis=1)\n    cond=~preds.target_steps.isna()\n    preds.loc[cond,'nearest_target_step']=preds.loc[cond,['step_arg_min','target_steps']].apply(lambda x:x[1][x[0]],axis=1)\n    preds['step_gap_signed']=preds['nearest_target_step']-preds['step']\n    preds['step_gap_signed']=preds['step_gap_signed'].fillna(1e12)\n    preds['step_gap']=preds['step_gap_signed'].abs()\n    del preds['target_steps']\n    gc.collect()\n    ## target quality\n    train_evts['nearest_target_step']=train_evts['step']\n    preds=preds.merge(train_evts[~train_evts.step.isna()][['series_id','event','nearest_target_step','good']],\n                      on=['series_id','event','nearest_target_step'],how='left')\n    preds['good']=preds['good'].fillna(1)\n    ## cross-entropy target\n    preds['target']=((360-preds['step_gap'])/360).clip(lower=0,upper=1).fillna(0)\n    keep_group_max_target(preds,'target')\n    best_score=comp_metric(train_evts,preds,'target',print_table=True)\n    print('best possible:',best_score.round(4),'pred/real num:',len(preds),'/',len(train_evts[~train_evts.step.isna()]))\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.685624Z","iopub.execute_input":"2023-11-20T17:08:02.68723Z","iopub.status.idle":"2023-11-20T17:08:02.720307Z","shell.execute_reply.started":"2023-11-20T17:08:02.687102Z","shell.execute_reply":"2023-11-20T17:08:02.718395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"def parse_column_map(window_stats_cfg,diff_stats_cfg):\n    input_idx_cols=[]\n    output_idx_cols=[]\n    for ms,cfg in window_stats_cfg:\n        if type(ms) is str:\n            ms=[ms]\n        for m in ms:\n            for s in cfg['stats']:\n                for w in cfg['windows']:\n                    input_idx_cols.append(f'{m}_{s}_w{w}')\n                    input_idx_cols.append(f'{m}_{s}_w{w*2}')\n                    output_idx_cols.append(f'local_{m}_{s}_w{w}')\n                    output_idx_cols.append(f'local_{m}_{s}_lw{w}')\n                    output_idx_cols.append(f'local_{m}_{s}_rw{w}')\n    for ms,cfg in diff_stats_cfg:\n        if type(ms) is str:\n            ms=[ms]\n        for m in ms:\n            for s in cfg['stats']:\n                for w in cfg['windows']:\n                    for d in cfg['diff_method']:\n                        input_idx_cols.append(f'{m}_{s}_w{w}')\n                        input_idx_cols.append(f'{m}_{s}_w{w*2}')\n                        output_idx_cols.append(f'local_{m}_{s}_{d}_w{w}')\n    input_idx_cols=sorted(list(set(input_idx_cols)))\n    output_idx_cols=sorted(list(set(output_idx_cols)))\n    input_idx_map={input_idx_cols[i]:i for i in range(len(input_idx_cols))}\n    output_idx_map={output_idx_cols[i]:i for i in range(len(output_idx_cols))}\n    return input_idx_cols,output_idx_cols,input_idx_map,output_idx_map\n\ndef assign_stats_value(window_stats_cfg,diff_stats_cfg,output_steps,input_matrix,output_matrix,input_idx_map,output_idx_map):\n    input_matrix,output_matrix\n    nrows=len(input_matrix)\n    for i in range(len(output_matrix)):\n        for ms,cfg in window_stats_cfg:\n            if type(ms) is str:\n                ms=[ms]\n            for m in ms:\n                for s in cfg['stats']:\n                    for w in cfg['windows']:\n                        step=output_steps[i]\n                        output_matrix[i,output_idx_map[f'local_{m}_{s}_w{w}']]=np.nan if (step+w)>=nrows else input_matrix[step+w,input_idx_map[f'{m}_{s}_w{w*2}']]\n                        output_matrix[i,output_idx_map[f'local_{m}_{s}_lw{w}']]=input_matrix[step,input_idx_map[f'{m}_{s}_w{w}']]\n                        output_matrix[i,output_idx_map[f'local_{m}_{s}_rw{w}']]=np.nan if (step+w)>=nrows else input_matrix[step+w,input_idx_map[f'{m}_{s}_w{w}']]\n\n        for ms,cfg in diff_stats_cfg:\n            if type(ms) is str:\n                ms=[ms]\n            for m in ms:\n                for s in cfg['stats']:\n                    for w in cfg['windows']:\n                        for d in cfg['diff_method']:\n                            if d=='explained_ratio':\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=1-\\\n                                    0.5*(output_matrix[i,output_idx_map[f'local_{m}_{s}_lw{w}']]+output_matrix[i,output_idx_map[f'local_{m}_{s}_rw{w}']])/\\\n                                    (output_matrix[i,output_idx_map[f'local_{m}_{s}_w{w}']])\n                            elif d=='side_diff':\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=\\\n                                    (output_matrix[i,output_idx_map[f'local_{m}_{s}_lw{w}']]-output_matrix[i,output_idx_map[f'local_{m}_{s}_rw{w}']])\n                            elif d=='side_diff_ratio':\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=\\\n                                    (output_matrix[i,output_idx_map[f'local_{m}_{s}_lw{w}']]-output_matrix[i,output_idx_map[f'local_{m}_{s}_rw{w}']])/\\\n                                    (output_matrix[i,output_idx_map[f'local_{m}_{s}_w{w}']])\n                            elif d=='side_diff_ratio2':\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=\\\n                                    (output_matrix[i,output_idx_map[f'local_{m}_{s}_lw{w}']]-output_matrix[i,output_idx_map[f'local_{m}_{s}_rw{w}']])/\\\n                                    (output_matrix[i,output_idx_map[f'local_{m}_{s}_lw{w}']]+output_matrix[i,output_idx_map[f'local_{m}_{s}_rw{w}']])\n                            elif d=='left_side_diff':\n                                llw=np.nan if (step-w)<0 else input_matrix[step-w,input_idx_map[f'{m}_{s}_w{w}']]\n                                lw=input_matrix[step,input_idx_map[f'{m}_{s}_w{w}']]\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=lw-llw\n                            elif d=='right_side_diff':  \n                                rw=np.nan if (step+w)>=nrows else input_matrix[step+w,input_idx_map[f'{m}_{s}_w{w}']]\n                                rrw=np.nan if (step+2*w)>=nrows else input_matrix[step+2*w,input_idx_map[f'{m}_{s}_w{w}']]\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=rrw-rw\n                            elif d=='left_side_diff_ratio2':\n                                llw=np.nan if (step-w)<0 else input_matrix[step-w,input_idx_map[f'{m}_{s}_w{w}']]\n                                lw=input_matrix[step,input_idx_map[f'{m}_{s}_w{w}']]\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=(lw-llw)/(lw+llw)\n                            elif d=='right_side_diff_ratio2':  \n                                rw=np.nan if (step+w)>=nrows else input_matrix[step+w,input_idx_map[f'{m}_{s}_w{w}']]\n                                rrw=np.nan if (step+2*w)>=nrows else input_matrix[step+2*w,input_idx_map[f'{m}_{s}_w{w}']]\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=(rrw-rw)/(rrw+rw)\n                            elif d=='hessian':  \n                                llw=np.nan if (step-w)<0 else input_matrix[step-w,input_idx_map[f'{m}_{s}_w{w}']]\n                                lw=input_matrix[step,input_idx_map[f'{m}_{s}_w{w}']]\n                                rw=np.nan if (step+w)>=nrows else input_matrix[step+w,input_idx_map[f'{m}_{s}_w{w}']]\n                                rrw=np.nan if (step+2*w)>=nrows else input_matrix[step+2*w,input_idx_map[f'{m}_{s}_w{w}']]\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=rrw-rw-(lw-llw)\n                            elif d=='side_diff_gap_1day':\n                                llw1=np.nan if (step-720*24)<0 else input_matrix[step-720*24,input_idx_map[f'{m}_{s}_w{w}']]\n                                lw1=np.nan if (step+w-720*24)<0 else input_matrix[step+w-720*24,input_idx_map[f'{m}_{s}_w{w}']]\n                                llw2=np.nan if (step+720*24)>=nrows else input_matrix[step+720*24,input_idx_map[f'{m}_{s}_w{w}']]\n                                lw2=np.nan if (step+w+720*24)>=nrows else input_matrix[step+w+720*24,input_idx_map[f'{m}_{s}_w{w}']]\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=(lw1-llw1)+(lw2-llw2)\n                            elif d=='side_diff_ratio_gap_1day':\n                                llw1=np.nan if (step-720*24)<0 else input_matrix[step-720*24,input_idx_map[f'{m}_{s}_w{w}']]\n                                lw1=np.nan if (step+w-720*24)<0 else input_matrix[step+w-720*24,input_idx_map[f'{m}_{s}_w{w}']]\n                                llw2=np.nan if (step+720*24)>=nrows else input_matrix[step+720*24,input_idx_map[f'{m}_{s}_w{w}']]\n                                lw2=np.nan if (step+w+720*24)>=nrows else input_matrix[step+w+720*24,input_idx_map[f'{m}_{s}_w{w}']]\n                                output_matrix[i,output_idx_map[f'local_{m}_{s}_{d}_w{w}']]=(lw1-llw1+lw2-llw2)/(lw1+llw1+lw2+llw2)\n\n\n                            \n    return","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.722216Z","iopub.execute_input":"2023-11-20T17:08:02.723095Z","iopub.status.idle":"2023-11-20T17:08:02.8235Z","shell.execute_reply.started":"2023-11-20T17:08:02.723051Z","shell.execute_reply":"2023-11-20T17:08:02.822029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_source_columns(df,window_stats_cfg):\n    rolling_args={'min_periods':1,'closed':'left','center':False,}\n    for ms,cfg in window_stats_cfg:\n        if type(ms) is str:\n            ms=[ms]\n        for m in ms:\n            for s in cfg['stats']:\n                for w in cfg['windows']:\n                    for mult in [1,2]:\n                        w*=mult\n                        rolling_args['min_periods']=max(w//2,1)\n                        new_col=f'{m}_{s}_w{w}'\n                        if new_col in df.columns:\n                            continue\n                        elif s=='max':\n                            df[new_col]=df[m].rolling(w,**rolling_args).max()\n                        elif s=='mean':\n                            #print(m,w,new_col)\n                            df[new_col]=df[m].rolling(w,**rolling_args).mean()\n                        elif s=='std':\n                            df[new_col]=df[m].rolling(w,**rolling_args).std(ddof=0)# ddof=0 to be consistent with past experiment\n                        elif s[0]=='q':\n                            df[new_col]=df[m].rolling(w,**rolling_args).quantile(int(s[1:])/100)\n                        elif s=='zero_rate':\n                            df[new_col]=(df[m]==0).rolling(w,**rolling_args).mean()\n                        elif s=='skew':\n                            df[new_col]=(df[m]==0).rolling(w,**rolling_args).skew()\n                        elif s=='min':\n                            df[new_col]=df[m].rolling(w,**rolling_args).min()\n                        elif s=='mad':\n                            df[new_col]=(df[m]-df[m].rolling(w,**rolling_args).median()).abs().rolling(w,**rolling_args).mean()\n                        elif s=='range':\n                            df[new_col]=df[m].rolling(w,**rolling_args).max()-df[m].rolling(w,**rolling_args).min()\n                        elif s=='iqr':\n                            df[new_col]=df[m].rolling(w,**rolling_args).quantile(0.75)-df[m].rolling(w,**rolling_args).quantile(0.25)\n\n                        df[new_col]=df[new_col].astype('float32')\n    return\n\ndef add_acum_local_max_min(df,source_name,normalize=True,window_size=1*60*60//5):\n    df[f'{source_name}_cum_w{window_size}']=df[source_name].copy()\n    if normalize:\n        df[f'{source_name}_cum_w{window_size}']/=df[f'{source_name}_cum_w{window_size}'].std()\n        df[f'{source_name}_cum_w{window_size}']-=df[f'{source_name}_cum_w{window_size}'].mean()\n    df[f'{source_name}_cum_w{window_size}']=df[f'{source_name}_cum_w{window_size}'].cumsum()\n    df[f'{source_name}_cum_window_max_w{window_size}']=df[f'{source_name}_cum_w{window_size}'].rolling(window_size,center=True,min_periods=window_size//2).max()\n    df[f'{source_name}_cum_window_min_w{window_size}']=df[f'{source_name}_cum_w{window_size}'].rolling(window_size,center=True,min_periods=window_size//2).min()\n    df[f'{source_name}_cum_window_gap_to_max_w{window_size}']=df[f'{source_name}_cum_window_max_w{window_size}']-df[f'{source_name}_cum_w{window_size}']\n    df[f'{source_name}_cum_window_gap_to_min_w{window_size}']=df[f'{source_name}_cum_window_min_w{window_size}']-df[f'{source_name}_cum_w{window_size}']\n    cols_to_add=[f'{source_name}_cum_window_gap_to_max_w{window_size}',\n                 f'{source_name}_cum_window_gap_to_min_w{window_size}',\n                ]\n    return cols_to_add\n\ndef self_product(vec,lag=1):\n    res=vec.copy()\n    res[:lag]=0\n    res[-lag:]=0\n    res[:-lag]*=vec[lag:]\n    return res\n\n@nb.jit(nopython = True, parallel = False, cache = False)\ndef inactive_interval_len(vec,thr=0):\n    res=[]\n    last_v=vec[0]\n    count=0\n    left=0\n    for i in range(len(vec)+1):\n        if i<len(vec) and np.abs(vec[i]-last_v)<=thr:\n            count+=1\n        else:\n            res.extend([count]*(i-left))\n            count=1\n            left=i\n        if i<len(vec):\n            last_v=vec[i]\n    return res","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.825624Z","iopub.execute_input":"2023-11-20T17:08:02.82643Z","iopub.status.idle":"2023-11-20T17:08:02.851091Z","shell.execute_reply.started":"2023-11-20T17:08:02.826388Z","shell.execute_reply":"2023-11-20T17:08:02.84966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(input_data):\n    sub_df,train_session,window_stats_cfg,diff_stats_cfg=input_data\n    input_idx_cols,output_idx_cols,input_idx_map,output_idx_map=\\\n    parse_column_map(window_stats_cfg,diff_stats_cfg)\n    sub_df=sub_df.reset_index(drop=True)\n    cols_to_add=[]\n    ####################### basic stats ####################################\n    max_step=train_session.step.max()\n    train_session['max_step']=max_step\n    train_session['timestamp']=pd.to_datetime(train_session['timestamp'],utc=True)\n    train_session['hour']=train_session['timestamp'].dt.hour\n    train_session['index']=train_session.index.copy()\n    train_session['enmo_diff']=train_session['enmo'].diff()\n    train_session['enmo_denoise']=(train_session['enmo']-0.02).clip(lower=0)\n    train_session['anglez_abs_diff']=train_session['anglez'].diff().abs().fillna(0)\n    train_session['anglez_abs_diff_n1']=train_session['anglez'].diff(-1).abs().fillna(0)\n    train_session['anglez_inactive_interval_len']=inactive_interval_len(train_session['anglez_abs_diff'].values,thr=5)\n    train_session['enmo_abs_diff']=train_session['enmo'].diff().abs().fillna(0)\n    train_session['anglez_abs_diff_24h_smoothed']=train_session['anglez_abs_diff']/(5+train_session['anglez_abs_diff'].rolling(720*24,center=True,min_periods=720*12).mean())\n    train_session['anglez_1min_smooth']=train_session['anglez'].rolling(12,center=True,min_periods=6).mean()\n    train_session['anglez_1min_smooth_abs_diff']=train_session['anglez_1min_smooth'].diff(12).abs().fillna(0)\n    train_session['in_bottom_zone']=train_session['anglez_abs_diff'].between(2,5).astype('int32')\n    train_session['anglez_abs_diff_pos']=train_session['anglez_abs_diff'].rank(method='min')#\n    train_session['anglez_abs_diff_pos']/=train_session['anglez_abs_diff_pos'].max()\n    train_session['anglez_in_middle']=(train_session['anglez'].abs()<45).astype('int32')\n    # mark fake region\n    mark_is_fake(train_session)\n    mark_fake_extension(train_session)    \n    train_session['is_fake']=train_session['is_fake'].astype('int32')\n    train_session['fake_and_extension']=0\n    train_session.loc[(train_session['is_fake']==1)|(train_session['fake_extension']==1),'fake_and_extension']=1\n    # sleep mark by simple rule\n    inactive_idx=np.where((train_session['anglez_abs_diff'].values<=5)&(~train_session['is_fake']))[0]\n    inactive_intervals=merge_points_to_intervals(inactive_idx)\n    inactive_intervals=[(max(i,0),min(j,len(train_session)-1)) for (i,j) in inactive_intervals]\n    inactive_intervals=filter_short_intervals(inactive_intervals,min_len=5*12)\n    inactive_intervals=merge_close_intervals(inactive_intervals,max_gap=1*12)\n    train_session['sleep_mark']=0\n    for i,j in inactive_intervals:\n        train_session.loc[i:(j-1),'sleep_mark']=1\n    # sleep mark without any filter\n    train_session['inactive']=((train_session['is_fake']==0)&(train_session['anglez_abs_diff']<5)).astype('int32')\n    train_session['signed_inactive']=2*train_session['inactive']-1\n    # sleep mark by more detailed heuristic method\n    inactive_intervals=heuristic_method1(train_session)\n    train_session['sleep_mark_h1']=0\n    train_session['onset_mark_h1']=0\n    train_session['wakeup_mark_h1']=0\n    for i,j in inactive_intervals:\n        train_session.loc[i:(j-1),'sleep_mark_h1']=1\n        train_session.loc[i,'onset_mark_h1']=1\n        train_session.loc[j,'wakeup_mark_h1']=1\n    inactive_intervals=heuristic_method2(train_session)\n    train_session['sleep_mark_h2']=0\n    for i,j in inactive_intervals:\n        train_session.loc[i:(j-1),'sleep_mark_h2']=1\n    # smoothed stats\n    for metric in ['anglez_abs_diff',]:\n        for window in [15*60//5,30*60//5,720,720*2]:\n            train_session[f'{metric}_smooth_{window}']=train_session[metric].rolling(window,center=True,min_periods=window//2,closed='left').mean()\n    # similar to LIDS\n    train_session['anglez_lids_10min']=(train_session['anglez_abs_diff']-5).clip(lower=0) \n    train_session['anglez_lids_10min']=train_session['anglez_lids_10min'].rolling(10*12,center=True,min_periods=5*12,closed='left').sum()\n    train_session['anglez_lids_10min']=1/(train_session['anglez_lids_10min']+1)\n    # self correlation\n    for metric in ['anglez_abs_diff',]:\n        train_session[f'{metric}_self_product_s5lag3']=self_product(train_session[metric].rolling(5,center=True).mean().values,lag=3)\n    ##################################### interval stats #####################################\n    # interval distribution stats\n    interval_df=sub_df[['event','step','pstart','pend',]]\n    interval_df=interval_df.sort_values(['step']).reset_index(drop=True)\n    interval_df['interval_len']=interval_df['pend']-interval_df['pstart']\n    interval_df['steps_to_last_event']=interval_df['step'].diff()\n    interval_df=interval_df.sort_values(['event','step']).reset_index(drop=True)\n    interval_df['step_in_day']=interval_df['step']%(24*3600//5)\n    interval_df['event_mean_step_in_day']=interval_df.groupby('event')['step_in_day'].transform('mean')\n    interval_df['event_std_step_in_day']=interval_df.groupby('event')['step_in_day'].transform('std')\n    interval_df['gap_to_event_mean_step_in_day']=(interval_df['step_in_day']-interval_df['event_mean_step_in_day']).abs()\n    interval_df['steps_to_last_pred']=interval_df.groupby('event')['step'].transform('diff')\n    sub_df=sub_df.merge(interval_df[['event','step','steps_to_last_pred','gap_to_event_mean_step_in_day',\n                                    'steps_to_last_event',\n                                   ]],on=['event','step',],how='left')\n    del interval_df\n    ##################################### days stats #####################################\n    train_session['day_12to12']=365*train_session['timestamp'].dt.year+train_session['timestamp'].dt.day_of_year\n    train_session.loc[train_session['hour']<12,'day_12to12']-=1\n    train_session['day_6to6']=365*train_session['timestamp'].dt.year+train_session['timestamp'].dt.day_of_year\n    train_session.loc[train_session['hour']<6,'day_6to6']-=1\n    for group_col in ['day_12to12','day_6to6']:\n        for col in ['is_fake','fake_and_extension','sleep_mark','sleep_mark_h1','sleep_mark_h2','in_bottom_zone']:\n            new_col=f'{col}_mean_over_{group_col}'\n            train_session[new_col]=train_session[[group_col,col]].groupby(group_col)[col].transform('mean')\n            cols_to_add.append(new_col)\n    ##################################### local stats #####################################\n    cols_to_add.extend(['enmo','enmo_abs_diff','anglez_abs_diff','is_fake','timestamp',\n                 'anglez_inactive_interval_len','max_step','anglez_abs_diff_n1'])#,'fake_extension','fake_and_extension'])\n    for window in [30*60//5,1*60*60//5,2*60*60//5,3*60*60//5,]:\n        cols_to_add.extend(add_acum_local_max_min(train_session,'signed_inactive',normalize=False,window_size=window))\n    # gap to mean onset/wakeup time\n    for col in ['onset_mark_h1','wakeup_mark_h1']:\n        mean_step=(train_session.loc[train_session[col]==1,'index']%(720*24)).mean()\n        train_session[f'gap_to_{col}_mean_step']=train_session['index']%(720*24)-mean_step\n        cols_to_add.append(f'gap_to_{col}_mean_step')\n    # add local stats\n    for metric in cols_to_add:\n        sub_df[metric]=train_session.loc[sub_df.step,metric].values\n    # time features\n    sub_df['weekday']=sub_df['timestamp'].dt.weekday\n    sub_df['hour']=sub_df['timestamp'].dt.hour\n    sub_df['second']=sub_df['timestamp'].dt.second\n    sub_df['minute']=sub_df['timestamp'].dt.minute\n    sub_df['minute_mod15']=sub_df['minute']%15\n    sub_df['sec_in_day']=sub_df['hour']*3600+sub_df['timestamp'].dt.minute*60+sub_df['timestamp'].dt.second\n    sub_df['within_session_location']=sub_df['step']/max_step\n    # window based features in batch\n    add_source_columns(train_session,window_stats_cfg)\n    source_matrix=train_session[input_idx_cols].values\n    feature_matrix=np.nan*np.zeros((len(sub_df),len(output_idx_cols)))\n    assign_stats_value(window_stats_cfg,diff_stats_cfg,\n                       sub_df.step.values,\n                       source_matrix,feature_matrix,\n                       input_idx_map,output_idx_map)\n    del train_session\n    del source_matrix\n    feature_matrix=pd.DataFrame(feature_matrix)\n    feature_matrix.columns=output_idx_cols\n    sub_df=pd.concat([sub_df,feature_matrix],axis=1)\n    ### compare with nearby candidates\n    block_join_gap=10*12\n    signed_score_cols=[\n        'local_sleep_mark_mean_side_diff_w1440',\n        'local_sleep_mark_mean_side_diff_w720',\n        'local_anglez_abs_diff_pos_mean_side_diff_w720',\n        'local_sleep_mark_h1_mean_side_diff_w720',\n        'local_sleep_mark_h1_mean_side_diff_w1440',\n        'local_anglez_abs_diff_pos_q95_side_diff_w360',\n        'local_anglez_abs_diff_q95_side_diff_w360',\n        'local_sleep_mark_h1_mean_side_diff_ratio2_w360',\n        'local_sleep_mark_h1_mean_side_diff_ratio2_w720',\n        'local_anglez_abs_diff_q95_side_diff_ratio2_w360',\n        'local_anglez_abs_diff_mean_side_diff_ratio2_w1440',\n        'local_enmo_abs_diff_mean_side_diff_ratio2_w1440',\n        'local_fake_extension_mean_side_diff_w720',\n        'local_fake_extension_mean_side_diff_w1440',\n        'local_fake_and_extension_mean_rw8640',\n        'local_in_bottom_zone_mean_lw2880',\n        'local_in_bottom_zone_mean_lw4320',\n        'local_in_bottom_zone_mean_lw1440',\n\n    ]\n    local_agg_df=sub_df[['event','step']+signed_score_cols].sort_values(['event','step']).reset_index(drop=True)\n    extra_features=[sub_df[['event','step']]]\n    ### shift features\n    shifts=[-1,1,-2,2,-3,3]\n    for shift in shifts:\n        shiftstr=str(shift).replace('-','n')\n        tmp=local_agg_df.iloc[:,1:]-local_agg_df.groupby('event').shift(shift)\n        cols=[col+f'_shift_{shiftstr}_gap' for col in tmp.columns]\n        tmp.columns=cols\n        extra_features.append(tmp)\n    ### compare in local block\n    blocks=[]\n    last_event=None\n    last_step=None\n    block_id=0\n    for event,step in zip(local_agg_df.event,local_agg_df.step):\n        if event!=last_event or (step-last_step)>=block_join_gap:\n            block_id+=1\n        blocks.append(block_id)\n        last_event=event\n        last_step=step\n    local_agg_df['block']=blocks\n    local_agg_df['block_size']=local_agg_df[['block','step']].groupby('block')['step'].transform('count')\n    local_agg_df['block_max_step']=local_agg_df[['block','step']].groupby('block')['step'].transform('max')\n    local_agg_df['block_min_step']=local_agg_df[['block','step']].groupby('block')['step'].transform('min')\n    local_agg_df['block_width']=local_agg_df['block_max_step']-local_agg_df['block_min_step']\n    local_agg_df['in_block_pos']=(local_agg_df['step']-local_agg_df['block_min_step'])/(1e-12+local_agg_df['block_width'])\n    extra_features.append(local_agg_df[['block_size','block_width','in_block_pos']])\n    #\n    tmp=local_agg_df[signed_score_cols]-local_agg_df.groupby('block')[signed_score_cols].transform('mean')\n    cols=[col+f'_block_{block_join_gap}_gap' for col in tmp.columns]\n    tmp.columns=cols\n    extra_features.append(tmp)\n    #\n    tmp=local_agg_df.groupby('block')[signed_score_cols].transform('rank')\n    cols=[col+f'_block_{block_join_gap}_rank' for col in tmp.columns]\n    tmp.columns=cols\n    extra_features.append(tmp)\n    #\n    extra_features=pd.concat(extra_features,axis=1)\n    sub_df=sub_df.merge(extra_features,on=['event','step'],how='left')\n    #######################\n    t2=time.time()\n    reduce_mem(sub_df)\n    return sub_df","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.853535Z","iopub.execute_input":"2023-11-20T17:08:02.854037Z","iopub.status.idle":"2023-11-20T17:08:02.922788Z","shell.execute_reply.started":"2023-11-20T17:08:02.853968Z","shell.execute_reply":"2023-11-20T17:08:02.921299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features_with_pool(train_data,preds,session_info,window_stats_cfg,diff_stats_cfg,pool_size=4):\n    def fe_eng_task_helper(train_data,preds,session_info,window_stats_cfg,diff_stats_cfg):\n        session_info=session_info.set_index('series_id')\n        for sid,sub_df in tqdm(preds.groupby('series_id')):\n            start_idx,end_idx=session_info.loc[sid,['start_idx','end_idx']]\n            train_session=train_data.loc[start_idx:end_idx].reset_index(drop=True).copy()\n            input_data=[sub_df,train_session,window_stats_cfg,diff_stats_cfg]\n            yield input_data\n    task_iter=fe_eng_task_helper(train_data,preds,session_info,window_stats_cfg,diff_stats_cfg)\n    with Pool(processes=pool_size) as pool:\n        feature_df=[x for x in pool.imap(add_features,task_iter)]\n    feature_df=pd.concat(feature_df)\n    feature_df=feature_df.reset_index(drop=True)\n    return feature_df","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.924644Z","iopub.execute_input":"2023-11-20T17:08:02.925059Z","iopub.status.idle":"2023-11-20T17:08:02.94087Z","shell.execute_reply.started":"2023-11-20T17:08:02.924998Z","shell.execute_reply":"2023-11-20T17:08:02.938845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_feature_cfg(full_cols):\n    invalid_features=['event','pstart', 'pend', 'row_id', 'series_id', 'method','score', 'step_gap', 'target','timestamp','step',\n                 'step_arg_min','nearest_target_step','step_gap_signed','max_step','empty_len','last_step','weight',\n                  'raw_timestamp','first_step','empty_len','start_idx','end_idx','tol','good','corrected_step_gap'\n                 ]\n\n    cat_features=['weekday','hour','minute_mod15']\n    features=sorted(list(set(full_cols)-set(invalid_features)))\n    print('#######',f'feature number:',len(features),'########')\n    feature_cfg={}\n    feature_cfg['step_features']=features\n    feature_cfg['step_cat_features']=cat_features\n    feature_cfg['score_features']=features\n    feature_cfg['score_cat_features']=cat_features\n    feature_cfg['step_target']='step_gap_signed'\n    feature_cfg['score_target']='corrected_target'\n    return feature_cfg","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.942563Z","iopub.execute_input":"2023-11-20T17:08:02.942947Z","iopub.status.idle":"2023-11-20T17:08:02.958413Z","shell.execute_reply.started":"2023-11-20T17:08:02.942915Z","shell.execute_reply":"2023-11-20T17:08:02.957191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postprocess(df):\n    df=df.copy()\n    ### 1.avoid mult of 12\n    fix_cond=df['step']%12==0\n    df.loc[fix_cond&(df['step']>0),'step']-=1\n    df.loc[fix_cond&(df['step']==0),'step']+=1\n    ### 2.avoid too close prediction\n    df2=[]\n    for (sid,event),sub_df in tqdm(df.groupby(['series_id','event'])):\n        filtered_idx=[]\n        sub_df=sub_df.sort_values('step').reset_index(drop=True)\n        i=0\n        while i<len(sub_df):\n            keep_score=1\n            gap=720\n            if len(filtered_idx)>0 :\n                gap=(sub_df.loc[i,'step']-sub_df.loc[filtered_idx[-1],'step'])\n                pscore1=sub_df.loc[i,'score']\n                pscore2=sub_df.loc[filtered_idx[-1],'score']\n                score1=np.exp(np.arctanh(2*pscore1-1))\n                score2=np.exp(np.arctanh(2*pscore2-1))\n                keep_score=min(gap/720,1)*np.sqrt(score1*score2/(score1**2+score2**2))\n            if len(filtered_idx)==0 or gap>=720 or keep_score>0.083:\n                filtered_idx.append(i)\n                i+=1\n            elif sub_df.loc[i,'score']>sub_df.loc[filtered_idx[-1],'score']:\n                filtered_idx.pop()\n            else:\n                i+=1\n        df2.append(sub_df.iloc[filtered_idx])\n    df2=pd.concat(df2,axis=0).reset_index(drop=True)\n    ### 3.avoid large score sum in day\n    df2['rank_group']=df2['event']+'/'+df2['series_id']+'/'+(df2['step']//(12*60*24)).astype(str)\n    df2['group_sum_score']=df2.groupby('rank_group')['score'].transform('sum')\n    cond=df2['group_sum_score']>1\n    df2.loc[cond,'score']*=1/df2.loc[cond,'group_sum_score']\n    ###\n    df2=df2[df.columns]\n    return df2","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.959895Z","iopub.execute_input":"2023-11-20T17:08:02.960278Z","iopub.status.idle":"2023-11-20T17:08:02.975324Z","shell.execute_reply.started":"2023-11-20T17:08:02.960249Z","shell.execute_reply":"2023-11-20T17:08:02.97364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_step_correct_model(feature_df,params,features,cat_features,target_column):\n    feature_df=feature_df.copy().sort_values(['event','series_id','step']).reset_index(drop=True)\n    feature_df['tol']=feature_df['step_gap_signed'].apply(get_tol_group)\n    feature_df['weight']=1.0/feature_df['tol'].apply(get_tol_width)\n    feature_df.loc[feature_df['event']=='onset','weight']=feature_df.loc[feature_df['event']=='onset','weight']**6\n    feature_df.loc[feature_df['event']=='wakeup','weight']=feature_df.loc[feature_df['event']=='wakeup','weight']**6\n    feature_df.loc[feature_df['tol']>360,'weight']=0\n    feature_df.loc[feature_df.series_id.isin(CFG.remove_series),'weight']*=CFG.remove_series_down_weight\n    feature_df.loc[feature_df['good']==0,'weight']*=0\n    feature_df.loc[(feature_df['empty_len']>12*60*24*5)&(feature_df['step']>feature_df['last_step']),'weight']=0\n    cv_splits=DSS_CV_split(feature_df,k=5,split_by_event=False,seed=233)\n    row_weight=feature_df['weight'].values\n    row_weight/=row_weight.mean()\n    models={}\n    oof_preds=[]\n    for i, (task,train_index, val_index) in enumerate(cv_splits):\n        X_train, y_train = feature_df.loc[train_index,features],feature_df.loc[train_index,target_column].values\n\n        lgb_train = lgb.Dataset(X_train, y_train,\n                              feature_name=features,\n                              weight=row_weight[train_index],\n                              categorical_feature=cat_features)\n\n        model = lgb.train(\n            params,\n            train_set = lgb_train, \n            verbose_eval=10000,\n        )\n        fold_pred=feature_df.loc[val_index,['series_id','event','step',]]\n        fold_pred['predicted_gap']=model.predict(feature_df.loc[val_index,features])\n        oof_preds.append(fold_pred)\n        models[f'{task}_{i}']=model\n    oof_preds=pd.concat(oof_preds).reset_index(drop=True)\n    return models,oof_preds","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.976964Z","iopub.execute_input":"2023-11-20T17:08:02.977531Z","iopub.status.idle":"2023-11-20T17:08:02.99545Z","shell.execute_reply.started":"2023-11-20T17:08:02.977497Z","shell.execute_reply":"2023-11-20T17:08:02.99402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_score_models(feature_df,train_evts,model_cfg={},feature_cfg=None):\n    oof_preds=[]\n    models={}\n    cv_splits=DSS_CV_split(feature_df,k=5,split_by_event=True,seed=77)\n    row_weight=(feature_df['target']>0).astype('int64')\n    row_weight[row_weight==0]=0.7\n    row_weight[(feature_df['step_gap'].values<360)&(feature_df['target'].values==0)]=0.4\n    row_weight[feature_df.series_id.isin(CFG.remove_series)]*=CFG.remove_series_down_weight\n    row_weight[feature_df['good']==0]=0\n    row_weight[(feature_df['empty_len']>12*60*24*5)&(feature_df['step']>feature_df['last_step'])]=0\n    for i, (task,train_index, val_index) in enumerate(cv_splits):\n        print('##############',f'task:{task} fold:{i}','#############')\n        ### step correct model ###\n        step_models,step_oof_preds=train_step_correct_model(\n            feature_df.loc[train_index],\n            model_cfg['step'],\n            feature_cfg['step_features'],\n            feature_cfg['step_cat_features'],\n            feature_cfg['step_target']\n        )\n        # correct step, add correct score as feature, create new target\n        feature_df=feature_df.merge(step_oof_preds,on=['series_id','event','step'],how='left')\n        feature_df.loc[val_index,'predicted_gap']=0\n        for _,model in step_models.items():\n            feature_df.loc[val_index,'predicted_gap']+=model.predict(feature_df.loc[val_index,feature_cfg['step_features']])/len(step_models)\n        feature_df['corrected_step']=feature_df['step']+feature_df['predicted_gap']\n        feature_df['corrected_step_gap']=(feature_df['nearest_target_step']-feature_df['corrected_step']).abs()\n        feature_df['corrected_target']=\\\n            (360-feature_df['corrected_step_gap']).clip(lower=0).fillna(0)/360\n        keep_group_max_target(feature_df,'corrected_target')\n        ### back to score model\n        X_train, y_train = feature_df.loc[train_index,feature_cfg['score_features']],feature_df.loc[train_index,feature_cfg['score_target']].values\n        X_val, y_val = feature_df.loc[val_index,feature_cfg['score_features']],feature_df.loc[val_index,feature_cfg['score_target']].values\n\n\n        lgb_train = lgb.Dataset(X_train, y_train,\n                              feature_name=feature_cfg['score_features'],\n                                weight=row_weight[train_index],\n                              categorical_feature=feature_cfg['score_cat_features'])\n        lgb_val = lgb.Dataset(X_val, y_val,\n                              feature_name=feature_cfg['score_features'],\n                              categorical_feature=feature_cfg['score_cat_features'])\n\n        model = lgb.train(\n            model_cfg['score'],\n            train_set = lgb_train, \n            valid_sets = [lgb_val], \n            verbose_eval=100000,\n        )\n\n        fold_pred=feature_df.loc[val_index,['row_id','series_id','event','step','timestamp','target','step_gap',\n                                            'corrected_step','corrected_target','max_step',\n                                           ]]\n        fold_pred['score']=model.predict(X_val)\n        fold_pred['fold']=i\n        oof_preds.append(fold_pred)\n        models[f'{task}_{i}']=model\n        models[f'{task}_{i}_step_correct']=step_models\n        del feature_df['corrected_step']\n        del feature_df['predicted_gap']\n        del feature_df['corrected_target']\n    oof_preds=pd.concat(oof_preds).reset_index(drop=True)\n    oof_preds.to_parquet('oof_preds_ori.pqt')\n    oof_preds['step']=oof_preds['corrected_step'].round()\n    oof_preds=postprocess(oof_preds)\n    oof_preds['step']=oof_preds['step'].clip(lower=0)\n    exceed_cond=oof_preds['step']>oof_preds['max_step']\n    oof_preds.loc[exceed_cond,'step']=oof_preds.loc[exceed_cond,'max_step']\n    oof_preds=oof_preds.sort_values('score',ascending=False).drop_duplicates(subset=['series_id','event','step'])\n    return models,oof_preds","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:02.997404Z","iopub.execute_input":"2023-11-20T17:08:02.997856Z","iopub.status.idle":"2023-11-20T17:08:03.031104Z","shell.execute_reply.started":"2023-11-20T17:08:02.997807Z","shell.execute_reply":"2023-11-20T17:08:03.029446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pipline","metadata":{}},{"cell_type":"code","source":"def pipline(train_data,train_evts,models={},pipline_mode='train',model_cfg={},\n            feature_cfg=None,series_subset=None,save=True,\n           ):\n    if pipline_mode=='train' and train_evts is None:\n        print('mode:',pipline_mode,'target info unavailable!')\n    ########################\n    session_info=summary_session_info(train_data)\n    if series_subset is not None:\n        session_info=session_info[session_info.series_id.isin(series_subset)]\n    add_target_quality_info(train_data,session_info,train_evts)\n    preds=candidate_generation(train_data,session_info,train_evts)\n    preds=add_target_info(preds,train_evts,session_info)\n    # feature eng\n    feature_df=add_features_with_pool(train_data,preds,session_info,window_stats_cfg,diff_stats_cfg)\n    if feature_cfg is None or len(feature_cfg)==0:\n        feature_cfg=get_feature_cfg(feature_df.columns)\n    if save:\n        feature_df.to_parquet('feature_df.pqt')\n        pd.to_pickle(feature_cfg,'feature_cfg.pkl')\n    # training\n    if pipline_mode=='train':\n        models,oof_preds=train_score_models(feature_df,train_evts,\n                                  model_cfg=model_cfg,\n                                  feature_cfg=feature_cfg)\n        cv_score=comp_metric(train_evts,oof_preds,score_column='score',print_table=True)\n        print('cv score:',cv_score)\n        if save:\n            pd.to_pickle(models,'lgb_models.pkl')\n            oof_preds.to_parquet('oof_preds.pqt')\n    # predict by all models\n    feature_df['score']=0\n    feature_df['predicted_gap']=0\n    task_pred_count={}\n    task_pred_count['onset']=0\n    task_pred_count['wakeup']=0\n    task_pred_count['onset_correct']=0\n    task_pred_count['wakeup_correct']=0\n    for k,v in models.items():\n        # task is defined by outter loop\n        task=k.split('_')[0]\n        # scope cond\n        if task=='all':\n            cond=~feature_df.event.isna()\n        else:\n            cond=feature_df.event==task\n        if 'step_correct' not in k:\n            if task=='all':\n                task_pred_count['onset']+=1\n                task_pred_count['wakeup']+=1\n            else:\n                task_pred_count[task]+=1\n            feature_df.loc[cond,'score']+=v.predict(feature_df.loc[cond,feature_cfg['score_features']])\n        else:\n            for k2,v2 in v.items():\n                # step correct belongs to last level\n                if task=='all':\n                    task_pred_count['onset_correct']+=1\n                    task_pred_count['wakeup_correct']+=1\n                else:\n                    task_pred_count[task+'_correct']+=1\n                feature_df.loc[cond,'predicted_gap']+=v2.predict(feature_df.loc[cond,feature_cfg['step_features']])\n    for k,v in task_pred_count.items():\n        print(k,'model number:',v)\n    feature_df.loc[feature_df.event=='onset','score']/=task_pred_count['onset']\n    feature_df.loc[feature_df.event=='wakeup','score']/=task_pred_count['wakeup']\n    feature_df.loc[feature_df.event=='onset','predicted_gap']/=task_pred_count['onset_correct']\n    feature_df.loc[feature_df.event=='wakeup','predicted_gap']/=task_pred_count['wakeup_correct']\n    feature_df['step']=(feature_df['step']+feature_df['predicted_gap']).round()\n    feature_df=postprocess(feature_df)\n    feature_df['step']=feature_df['step'].clip(lower=0)\n    exceed_cond=feature_df['step']>feature_df['max_step']\n    feature_df.loc[exceed_cond,'step']=feature_df.loc[exceed_cond,'max_step']\n    feature_df['step']=feature_df['step'].astype('int64')\n    # clean up res\n    feature_df=feature_df[['series_id','step','event','score']].reset_index(drop=True)\n    feature_df=feature_df.groupby(['event','series_id','step']).agg(\n        score=('score','max'),\n    ).reset_index()\n    feature_df['row_id']=feature_df.index\n    return feature_df,models,feature_cfg","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"model_cfg={\n    'step':{\n        'n_estimators':300,\n        'learning_rate':0.02,\n        \"objective\": 'regression_l1',\n        'metric': 'None',\n        'metric_freq':50,\n        'boosting_type': \"gbdt\",\n        #'lambda_l1': 0.001, \n        'lambda_l2': 0.5, \n        'num_leaves':20,\n        'min_data_in_leaf': 50,\n        'feature_fraction_bynode':1,\n        'feature_fraction': 0.7,\n        'bagging_fraction': 0.7,\n        'bagging_freq': 1, \n        'verbosity': -1,\n    },\n    'score':{\n        'n_estimators':2000,\n        'early_stopping_rounds':100,\n        'learning_rate':0.02,\n        \"objective\": 'cross_entropy',\n        'metric': 'cross_entropy',\n        'metric_freq':50,\n        'boosting_type': \"gbdt\",\n        #'lambda_l1': 0.001, \n        'lambda_l2': 0.5, \n        ##################\n        'num_leaves':60,\n        'min_data_in_leaf': 100,\n        'feature_fraction_bynode':1,\n        'feature_fraction': 0.7,\n        'bagging_fraction': 0.7,\n        'bagging_freq': 1, \n        'verbosity': -1,\n    },\n}","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:03.032735Z","iopub.execute_input":"2023-11-20T17:08:03.034026Z","iopub.status.idle":"2023-11-20T17:08:03.050794Z","shell.execute_reply.started":"2023-11-20T17:08:03.033965Z","shell.execute_reply":"2023-11-20T17:08:03.049335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* cfg for window based feature","metadata":{}},{"cell_type":"code","source":"window_stats_cfg=[\n    ([\n        'enmo_abs_diff','anglez_abs_diff','anglez_abs_diff_pos','anglez_1min_smooth_abs_diff',\n       'anglez_abs_diff_24h_smoothed',\n    ],\n     {\n        'stats':['mean','q50','q95'],\n        'windows':[12,36,120,180,360,720,720*2,720*4,720*6,720*8,60,240,],\n    }),\n    ('anglez_abs_diff_self_product_s5lag3',\n     {\n        'stats':['mean',],\n        'windows':[12,36,120,180,360,720,720*2,720*4,720*6,720*8,60,240,],\n    }),\n    ###############################\n    ([\n        'is_fake','fake_and_extension'\n    ],\n     {\n        'stats':['mean',],\n        'windows':[12,36,120,180,360,720,720*2,720*4,720*6,720*8,720*12,60,240,],\n    }),\n    #######################\n    ([\n        'sleep_mark','sleep_mark_h1','sleep_mark_h2','fake_extension','in_bottom_zone',\n        'anglez_in_middle',\n    ],\n     {\n        'stats':['mean',],\n        'windows':[12,36,120,180,360,720,720*2,720*4,720*6,720*8,60,240,],\n    }),\n    #########################\n    (['anglez_abs_diff_smooth_180','anglez_abs_diff_smooth_360'],\n     {\n        'stats':['max','min'],\n        'windows':[720,720*2,720*4,720*6,720*8,],\n    }),\n    ('anglez_abs_diff_smooth_720',\n     {\n        'stats':['max','min'],\n        'windows':[720*2,720*4,720*6,720*8,720*12],\n    }),\n    ('anglez_abs_diff_smooth_1440',\n     {\n        'stats':['max','min'],\n        'windows':[720*4,720*6,720*8,720*12],\n    }),\n    #####################################\n    ('anglez_lids_10min',\n     {\n        'stats':['mean','q50','q95'],\n        'windows':[12,36,120,180,360,720,720*2,720*4,720*6,720*8,60,240],\n    }),\n    ###########################\n\n\n]\ndiff_stats_cfg=[\n    ([\n        'enmo_abs_diff','anglez_abs_diff','anglez_abs_diff_pos','anglez_lids_10min',\n     ],\n     {\n        'stats':['mean','q95'],\n        'diff_method':['side_diff','side_diff_ratio','side_diff_ratio2',],\n        'windows':[12,36,120,180,360,720,720*2,720*4,720*6,720*8,60,240],\n    }),\n    ##########\n    ([\n        'enmo_abs_diff','anglez_abs_diff','anglez_abs_diff_24h_smoothed','anglez_1min_smooth_abs_diff',\n    ],\n     {\n        'stats':['mean','q95'],\n        'diff_method':['left_side_diff','right_side_diff','left_side_diff_ratio2','right_side_diff_ratio2'],\n        'windows':[12,36,120,180,360,720,720*2,60,240],\n    }),\n    #####################\n    ([\n        'sleep_mark','sleep_mark_h1','sleep_mark_h2','fake_extension','anglez_in_middle',\n    ],\n     {\n        'stats':['mean',],\n        'diff_method':['side_diff','side_diff_ratio','side_diff_ratio2'],\n        'windows':[12,36,120,180,360,720,720*2,720*4,720*6,720*8,60,240],\n    }),\n    ####################\n\n]","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:03.052645Z","iopub.execute_input":"2023-11-20T17:08:03.053483Z","iopub.status.idle":"2023-11-20T17:08:03.093536Z","shell.execute_reply.started":"2023-11-20T17:08:03.053447Z","shell.execute_reply":"2023-11-20T17:08:03.092569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"################################# train #########################################\n# pipline_mode='train'\n# models={}\n# feature_cfg={}\n# train_data=pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet')\n# train_evts=pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n# train_pred,models,feature_cfg=pipline(train_data,train_evts,\n#             models=models,pipline_mode=pipline_mode,model_cfg=model_cfg,\n#             feature_cfg=feature_cfg,\n#             series_subset=None,#train_evts.series_id.unique()[:40],\n#             save=True,\n#            )\n# score=comp_metric(train_evts,train_pred,print_table=True)\n# print('score by all models with leak:',score)\n################################# predict #########################################\npipline_mode='predict'\nmodels=pd.read_pickle('/kaggle/input/cmi-lgb-v9-train-reproduce/lgb_models.pkl')\nfeature_cfg=pd.read_pickle('/kaggle/input/cmi-lgb-v9-train-reproduce/feature_cfg.pkl')\ntest_data=pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet')\ntest_evts=None\ntest_pred,models,feature_cfg=pipline(test_data,test_evts,\n            models=models,pipline_mode=pipline_mode,model_cfg=model_cfg,\n            feature_cfg=feature_cfg,\n            save=False,\n           )\ntest_pred.to_csv('submission.csv', index=False)\ntest_pred","metadata":{"execution":{"iopub.status.busy":"2023-11-20T17:08:03.121052Z","iopub.execute_input":"2023-11-20T17:08:03.121965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if pipline_mode=='train':\n    fic_df=pd.DataFrame()\n    fic_df['features']=feature_cfg['step_features']\n    fic_df['importance(gain)']=0\n    for k,v in models.items():\n        if 'step_correct' in k and 'onset' in k:\n            for model in v.values():\n                fic_df['importance(gain)']+=model.feature_importance(importance_type='gain')\n    display(fic_df.sort_values('importance(gain)',ascending=False).head(40))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if pipline_mode=='train':\n    fic_df=pd.DataFrame()\n    fic_df['features']=feature_cfg['step_features']\n    fic_df['importance(gain)']=0\n    for k,v in models.items():\n        if 'step_correct' in k and 'wakeup' in k:\n            for model in v.values():\n                fic_df['importance(gain)']+=model.feature_importance(importance_type='gain')\n    display(fic_df.sort_values('importance(gain)',ascending=False).head(40))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if pipline_mode=='train':\n    fic_df=pd.DataFrame()\n    fic_df['features']=feature_cfg['score_features']\n    fic_df['importance(gain)']=0\n    for k,v in models.items():\n        if 'step_correct' in k:\n            continue\n        if 'onset' in k:\n            fic_df['importance(gain)']+=v.feature_importance(importance_type='gain')\n    display(fic_df.sort_values('importance(gain)',ascending=False).head(40))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if pipline_mode=='train':\n    fic_df=pd.DataFrame()\n    fic_df['features']=feature_cfg['score_features']\n    fic_df['importance(gain)']=0\n    for k,v in models.items():\n        if 'step_correct' in k:\n            continue\n        if 'wakeup' in k:\n            fic_df['importance(gain)']+=v.feature_importance(importance_type='gain')\n    display(fic_df.sort_values('importance(gain)',ascending=False).head(40))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}