{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!cp ../input/my-talibinstall/ta-lib-0.4.0-src.tar.gzh  ./ta-lib-0.4.0-src.tar.gz\n!tar -xzvf ta-lib-0.4.0-src.tar.gz > null\n!cd ta-lib && ./configure --prefix=/usr > null && make  > null && make install > null\n!cp ../input/my-talibinstall/TA-Lib-0.4.21.tar.gzh TA-Lib-0.4.21.tar.gz\n!pip install TA-Lib-0.4.21.tar.gz > null\n!pip install ../input/my-talibinstall/numpy-1.21.4-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl >null\nimport talib as ta","metadata":{"execution":{"iopub.status.busy":"2022-01-27T19:39:42.320847Z","iopub.execute_input":"2022-01-27T19:39:42.321411Z","iopub.status.idle":"2022-01-27T19:42:58.903025Z","shell.execute_reply.started":"2022-01-27T19:39:42.321279Z","shell.execute_reply":"2022-01-27T19:42:58.902007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import traceback\nimport numpy as np\nimport pandas as pd\nimport gc\nimport time\nimport os\nimport xgboost as xgb\nimport gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2022-01-30T03:39:47.506537Z","iopub.execute_input":"2022-01-30T03:39:47.507024Z","iopub.status.idle":"2022-01-30T03:39:48.627125Z","shell.execute_reply.started":"2022-01-30T03:39:47.506902Z","shell.execute_reply":"2022-01-30T03:39:48.626177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_version=58\ntuned=True #use model set new-xgbcrypto-tune\nalldata = True  #use alldata-trained version\npre_minute= 200 #df_test batch size","metadata":{"execution":{"iopub.status.busy":"2022-01-30T03:39:52.274444Z","iopub.execute_input":"2022-01-30T03:39:52.274771Z","iopub.status.idle":"2022-01-30T03:39:52.278608Z","shell.execute_reply.started":"2022-01-30T03:39:52.27472Z","shell.execute_reply":"2022-01-30T03:39:52.277848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ASSET_DETAILS_CSV = '../input/c/c/g-research-crypto-forecasting/asset_details.csv'\ndf_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")","metadata":{"papermill":{"duration":1.368073,"end_time":"2021-11-17T01:59:11.055609","exception":false,"start_time":"2021-11-17T01:59:09.687536","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-30T03:39:56.054427Z","iopub.execute_input":"2022-01-30T03:39:56.054691Z","iopub.status.idle":"2022-01-30T03:39:56.07718Z","shell.execute_reply.started":"2022-01-30T03:39:56.054663Z","shell.execute_reply":"2022-01-30T03:39:56.076368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## feature engineering\n\n- [https://mrjbq7.github.io/ta-lib/doc_index.html](https://mrjbq7.github.io/ta-lib/doc_index.html)","metadata":{"papermill":{"duration":0.071242,"end_time":"2021-11-17T01:59:53.170961","exception":false,"start_time":"2021-11-17T01:59:53.099719","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"import pickle\nwith open(f\"../input/new-xgbcrypto-tune/model_nof_{param_version}/feature_best{param_version}\", \"rb\") as f:\n    fdict=pickle.load(f)\nprint(fdict)\nfparam_str=['beta_s', 'beta_l', 'lrtn','fastk1','fastk2','adx','macd_s','macd_l','macd_sig','vol_sum','rsi','std_Crypto_Index','std_lr_15','std_Mkt_lrt_15']\n[fdict[f] for f in fparam_str]","metadata":{"execution":{"iopub.status.busy":"2022-01-30T03:40:40.475739Z","iopub.execute_input":"2022-01-30T03:40:40.476089Z","iopub.status.idle":"2022-01-30T03:40:40.487958Z","shell.execute_reply.started":"2022-01-30T03:40:40.476048Z","shell.execute_reply":"2022-01-30T03:40:40.487069Z"}}},{"cell_type":"code","source":"####mod feature params\nbeta_s, beta_l, lrtn,fastk1,fastk2,adx,macd_s,macd_l,macd_sig,vol_sum,rsi,std_Crypto_Index,std_lr_15,std_Mkt_lrt_15 = ('6h', '2d', 30, 15, 30, 40, 15, 40, 15, 15, 40, 30, 240, 10)\n\nadx,std_lr_15","metadata":{"execution":{"iopub.status.busy":"2022-01-30T04:04:33.009831Z","iopub.execute_input":"2022-01-30T04:04:33.010855Z","iopub.status.idle":"2022-01-30T04:04:33.020347Z","shell.execute_reply.started":"2022-01-30T04:04:33.010794Z","shell.execute_reply":"2022-01-30T04:04:33.0192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef beta_window(beta):\n    num, unit = int(beta[:-1]),beta[-1]\n    if unit == 'h':\n        width = 60*num\n    elif unit == 'd':\n        width = 60*24*num\n    return width\n\nbeta_sw = beta_window(beta_s)\nbeta_lw = beta_window(beta_l)\n(beta_sw,beta_lw)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-30T03:43:52.296094Z","iopub.execute_input":"2022-01-30T03:43:52.297127Z","iopub.status.idle":"2022-01-30T03:43:52.305787Z","shell.execute_reply.started":"2022-01-30T03:43:52.297078Z","shell.execute_reply":"2022-01-30T03:43:52.304674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def log_return(series, periods=5):\n    return np.log(series).diff(periods)\n\ndef lag_features(df):    \n    ####TECH indicators\n    df['slowK'], df['slowD'] = ta.STOCH(df.High, df.Low, df.Close, \n                                        fastk_period=fastk1, slowk_period=int(3*fastk1/5), slowd_period=int(3*fastk1/5),\n                                        slowk_matype=0, slowd_matype=0)\n    df['fastK'], df['fastD'] = ta.STOCHF(df.High, df.Low, df.Close,\n                                         fastk_period=fastk2, fastd_period=int(3*fastk2/5), \n                                         fastd_matype=0)\n    df[f'rsi_{rsi}'] = ta.RSI(df['Close'], timeperiod=rsi)\n    df[f'macd_{macd_s}_{macd_l}'],df[f'macd_signal_{macd_sig}'], df['macd_hist'] = \\\n                ta.MACD(df['Close'],fastperiod=macd_s, slowperiod=macd_l, signalperiod=macd_sig)\n    df[f'adx_{adx}'] = ta.ADX(df['High'], df['Low'],df['Close'], timeperiod=adx)#Average Directional Movement Index\n\n    df[f'vol_sum_{vol_sum}'] = ta.SMA(df['Volume'],vol_sum)*vol_sum\n    ####std volatility\n    #df[f'std_lr_15_{std_lr_15}'] = ta.STDDEV(df.lr_15,timeperiod=std_lr_15, nbdev=1)\n    df[f'std_Mkt_lrt_15_{std_Mkt_lrt_15}'] = ta.STDDEV(df.Mkt_lrt_15,timeperiod=std_Mkt_lrt_15, nbdev=1)\n    df[f'std_Crypto_Index_{std_Crypto_Index}'] = ta.STDDEV(df.Crypto_Index,timeperiod=std_Crypto_Index, nbdev=1)\n\n\ndef make_std(df,width):\n    df[f'std_lr_15_{width}'] = ta.STDDEV(df.lr_15,timeperiod=width, nbdev=1)\n    return df\n\ndef beta_resid(df): \n    b = ((ta.MULT(df.Mkt_lrt_15,df.lr_15).mean())/ \\\n        (ta.MULT(df.Mkt_lrt_15,df.Mkt_lrt_15).mean()))\n    if b in [np.nan,np.inf,-np.inf]:\n        b=0\n    return b \n\n\ndef get_features(df_feat):\n    pd.options.mode.chained_assignment = None  # default='warn'\n    df_feat[f\"lr_15_resid_{beta_s}\"] = ta.SUB(df_feat.lr_15, ta.MULT(df_feat[f\"beta_{beta_s}\"], df_feat.Mkt_lrt_15)).rename(f\"lr_15_resid_{beta_s}\")\n    df_feat[f\"lr_15_resid_{beta_l}\"] = ta.SUB(df_feat.lr_15, ta.MULT(df_feat[f\"beta_{beta_l}\"], df_feat.Mkt_lrt_15)).rename(f\"lr_15_resid_{beta_l}\")\n    df_feat[f\"lrtn_index_{lrtn}\"] = log_return(df_feat.Crypto_Index, lrtn)\n    lag_features(df_feat)\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-27T19:43:00.053427Z","iopub.execute_input":"2022-01-27T19:43:00.053703Z","iopub.status.idle":"2022-01-27T19:43:00.073443Z","shell.execute_reply.started":"2022-01-27T19:43:00.053673Z","shell.execute_reply":"2022-01-27T19:43:00.072621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load tuned models","metadata":{"papermill":{"duration":0.068441,"end_time":"2021-11-17T01:59:53.590549","exception":false,"start_time":"2021-11-17T01:59:53.522108","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from os.path import exists\nmodels = {}\n\ndef model_reload_train(type: str):\n    if alldata:\n        mod_suffix = \"_alldata.json\"\n    else:\n        mod_suffix = \".json\"\n    if tuned:\n        print('use model folder new-xgbcrypto-tune')\n        mod_folder = f\"../input/new-xgbcrypto-tune/model_nof_{param_version}\"\n    else:\n        print('use model folder mytrainedxgb')\n        mod_folder = f\"../input/mytrainedxgb/model_nof_{param_version}\"\n        \n    for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):            \n        model_file = mod_folder + f\"/model_{asset_id}\"+mod_suffix\n        if exists(model_file):\n            print(f\"{model_file} for {asset_name} exists\")\n            model = xgb.Booster()\n            model.load_model(model_file)\n            models[asset_id] = model\n        \n\nmodel_reload_train(type='xgb')\n","metadata":{"papermill":{"duration":12803.860437,"end_time":"2021-11-17T05:33:17.518996","exception":false,"start_time":"2021-11-17T01:59:53.658559","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-27T19:43:00.074999Z","iopub.execute_input":"2022-01-27T19:43:00.075358Z","iopub.status.idle":"2022-01-27T19:43:01.607486Z","shell.execute_reply.started":"2022-01-27T19:43:00.075328Z","shell.execute_reply":"2022-01-27T19:43:01.606741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit To Kaggle\n\nTake the contiguous pre-minutes `supplemental_train` before the API test set as the previous info for calculating the lag_features.","metadata":{"execution":{"iopub.status.busy":"2021-11-02T20:57:49.349459Z","iopub.status.idle":"2021-11-02T20:57:49.349757Z","shell.execute_reply":"2021-11-02T20:57:49.349613Z","shell.execute_reply.started":"2021-11-02T20:57:49.349596Z"},"papermill":{"duration":0.081057,"end_time":"2021-11-17T05:33:17.927359","exception":false,"start_time":"2021-11-17T05:33:17.846302","status":"completed"},"tags":[]}},{"cell_type":"code","source":"######################################################\npre_minute_beta = beta_lw + 15\nadd_weight_map = dict(zip(df_asset_details.Asset_ID, \n                        df_asset_details.Weight/df_asset_details.Weight.sum()))\n\n###load sup_train\nsup_train = pd.read_csv('../input/c/c/g-research-crypto-forecasting/supplemental_train.csv')\nsup_train = sup_train.set_index(\"timestamp\")\nind = sup_train.index.unique()\n###consistent timestamp for all 14 assets\ndef reindex(df):\n    df = df.reindex(range(ind[0],ind[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\nsup_train = sup_train.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\nsup_train = sup_train.iloc[(-14*pre_minute_beta):,:]\n#add weight\nsup_train['Weight'] = sup_train['Asset_ID'].map(add_weight_map)\nsup_train.drop('Target',axis=1, inplace=True)\nsup_train.set_index('Asset_ID',append=True, inplace=True)\n#######################################add lr_15,mkt_lr_15,crypto_index,beta_s,beta_l\nlr_15 = sup_train.groupby('Asset_ID').apply( \n        lambda x: log_return(x[['Close']],15)\n        )\nsup_train['lr_15'] = lr_15['Close']\n\nmkt_lr_15 = sup_train.groupby('timestamp').apply( \n    lambda x: x[[\"lr_15\", \"Close\"]].multiply(x[\"Weight\"], axis=\"index\").sum()\n    )\nmkt_lr_15.columns = ['Mkt_lrt_15','Crypto_Index']\nfirsts = sup_train.index.get_level_values('timestamp')\nsup_train[['Mkt_lrt_15','Crypto_Index']] = mkt_lr_15.loc[firsts].values\n#####placeholder for long window features\nsup_train[f\"beta_{beta_s}\"] = 0\nsup_train[f\"beta_{beta_l}\"] = 0\nsup_train[f\"std_lr_15_{std_lr_15}\"] = 0","metadata":{"execution":{"iopub.status.busy":"2022-01-27T19:51:42.519917Z","iopub.execute_input":"2022-01-27T19:51:42.520465Z","iopub.status.idle":"2022-01-27T19:51:52.599762Z","shell.execute_reply.started":"2022-01-27T19:51:42.52043Z","shell.execute_reply":"2022-01-27T19:51:52.598901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime \n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\npd.options.mode.chained_assignment = None  # default='warn'\n\nstart_time = datetime.now()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    num_asset_test = df_test.shape[0]\n    row_asset_id_map = dict(zip(df_test.row_id, df_test.Asset_ID))\n    test_timestamp = df_test.timestamp.values[0]\n    \n    timestamp_list = sup_train.index.get_level_values('timestamp').unique().values\n    timestamp_list = np.append(timestamp_list,test_timestamp)\n    #######################################format df_test\n    ###add weight and index\n    df_test['Weight'] = df_test['Asset_ID'].map(add_weight_map)\n    ###fillin missing assets as nan\n    df_test.set_index(['timestamp','Asset_ID'],inplace=True)\n    df_test = df_test.reindex(list(zip([test_timestamp]*14,range(14))))\n    ########################################concat to sup_train, add lr_15,mkt_lr_15,crypto_index\n    sup_train = pd.concat([sup_train,df_test.drop('row_id',axis=1)],join='outer')\n    #########################################fill in missing assets as forward\n    if num_asset_test <14:\n        #ffill in missing\n        sup_train = sup_train.groupby('Asset_ID').apply(lambda x: x.fillna(method=\"ffill\")).iloc[14:,:]\n    else:\n        sup_train = sup_train.iloc[14:,:]\n    \n    test_lr_15 = sup_train.loc[timestamp_list[[-16,-1]]].groupby('Asset_ID').apply(\n        lambda x: np.log(x[['Close']]).diff()\n    )\n    sup_train.loc[test_timestamp, 'lr_15'] = test_lr_15.loc[test_timestamp,'Close'].values\n    sup_train.loc[test_timestamp, ['Mkt_lrt_15','Crypto_Index']] = \\\n        sup_train.loc[test_timestamp, [\"lr_15\", \"Close\"]].multiply(sup_train.loc[test_timestamp,\"Weight\"], axis=\"index\").sum().values\n    ########################################beta_sl\n    beta_short = sup_train[['lr_15','Mkt_lrt_15']].iloc[-14*(beta_sw):,:].groupby('Asset_ID').apply(\n        lambda x: beta_resid(x)).rename(f\"beta_{beta_s}\")\n    beta_long = sup_train[['lr_15','Mkt_lrt_15']].iloc[-14*(beta_lw):,:].groupby('Asset_ID').apply(\n        lambda x: beta_resid(x)).rename(f\"beta_{beta_l}\")\n    sup_train.loc[test_timestamp, [f\"beta_{beta_s}\",f\"beta_{beta_l}\"]] = \\\n        pd.concat([beta_short,beta_long],axis=1).values\n    #####################################long std\n    long_std = sup_train.iloc[-14*std_lr_15:,:].groupby('Asset_ID').apply(lambda x: x.lr_15.std())\n    sup_train.loc[test_timestamp, f\"std_lr_15_{std_lr_15}\"] = long_std.values * np.sqrt((std_lr_15-1)/std_lr_15)\n    #######################################add features to test timestamp\n    sup_train2 = sup_train.iloc[(-14*pre_minute):,:].copy()\n    xx_test=sup_train2.groupby('Asset_ID').apply(\n        lambda x: get_features(x)\n    ).loc[test_timestamp]\n    #rdy for prediction\n    y_pred=df_test.apply(lambda row: models[row.name[1]].predict(\n                            xgb.DMatrix(pd.DataFrame([xx_test.loc[row.name[1],models[row.name[1]].feature_names]]))\n                                                                )[-1]\n                         ,axis =1)\n    #match with row_id\n    y_pred.reset_index('timestamp',drop=True,inplace=True)\n    df_pred['Target']= y_pred.loc[df_pred['row_id'].map(row_asset_id_map)].values\n    env.predict(df_pred)\n\ntime_elapsed = datetime.now() - start_time\nprint('Time elapsed total (hh:mm:ss.ms) {}'.format(time_elapsed))\nprint(f'time elapsed per iteration {time_elapsed/4}')\nprint(f'Submission time estimate {129600*time_elapsed/4}')","metadata":{"execution":{"iopub.status.busy":"2022-01-27T19:43:14.793629Z","iopub.execute_input":"2022-01-27T19:43:14.794456Z","iopub.status.idle":"2022-01-27T19:43:15.176073Z","shell.execute_reply.started":"2022-01-27T19:43:14.794413Z","shell.execute_reply":"2022-01-27T19:43:15.175355Z"},"trusted":true},"execution_count":null,"outputs":[]}]}