{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import IPython.display\n!cp ../input/my-talibinstall/ta-lib-0.4.0-src.tar.gzh  ./ta-lib-0.4.0-src.tar.gz\n!tar -xzvf ta-lib-0.4.0-src.tar.gz > null\n!cd ta-lib && ./configure --prefix=/usr > null && make  > null && make install > null\n!cp ../input/my-talibinstall/TA-Lib-0.4.21.tar.gzh TA-Lib-0.4.21.tar.gz\n!pip install TA-Lib-0.4.21.tar.gz > null\n!pip install ../input/my-talibinstall/numpy-1.21.4-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl >null\n\nIPython.display.clear_output()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:59:34.255174Z","iopub.execute_input":"2022-01-21T15:59:34.25569Z","iopub.status.idle":"2022-01-21T16:01:52.425957Z","shell.execute_reply.started":"2022-01-21T15:59:34.255597Z","shell.execute_reply":"2022-01-21T16:01:52.420467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import talib as ta","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gc\nimport time\nimport xgboost as xgb\nimport os\nfrom random import sample\nfrom os.path import exists\nimport json\nimport pickle\nfrom sklearn.model_selection import ParameterGrid,ParameterSampler","metadata":{"execution":{"iopub.status.busy":"2022-01-30T00:58:38.351827Z","iopub.execute_input":"2022-01-30T00:58:38.352609Z","iopub.status.idle":"2022-01-30T00:58:39.433602Z","shell.execute_reply.started":"2022-01-30T00:58:38.352463Z","shell.execute_reply":"2022-01-30T00:58:39.432575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 0 Tune settings","metadata":{}},{"cell_type":"code","source":"PRE_MOD_FOLDER = \"../input/new-xgbcrypto-tune\"#'../input/mytrainedxgb' '../input/new-xgbcrypto-tune'\nASSET_DETAILS_CSV = '../input/g-research-crypto-forecasting/asset_details.csv'\n\nprevious_version = sorted([int(s.split('_')[-1]) for s in os.listdir(PRE_MOD_FOLDER) \\\n                           if 'model_nof' in s])[-1]\nparams_version=previous_version+1\n\nprint(f\"Previous version: {previous_version}\")\nprint(f\"current saving version: {params_version}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:01:21.836149Z","iopub.execute_input":"2022-01-30T01:01:21.836494Z","iopub.status.idle":"2022-01-30T01:01:21.850173Z","shell.execute_reply.started":"2022-01-30T01:01:21.836462Z","shell.execute_reply":"2022-01-30T01:01:21.849487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make folder is not exist\nos.system(f\"mkdir -p model_nof_{params_version}\")\n#write necessary notes: check end of the notebook\nwith open(f'./model_nof_{params_version}/README{params_version}.txt', 'w') as f:\n    f.write(f'Version {params_version}\\n')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:01:24.662675Z","iopub.execute_input":"2022-01-30T01:01:24.663272Z","iopub.status.idle":"2022-01-30T01:01:24.682231Z","shell.execute_reply.started":"2022-01-30T01:01:24.663221Z","shell.execute_reply":"2022-01-30T01:01:24.680931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- xgb hyperparameters\n  - `params_xgb_prescore`\n  - `params_xgb_prebest`\n- feature parameters\n  - `feature_prescore`\n  - `feature_prebest`","metadata":{}},{"cell_type":"code","source":"tunepara = False #params_xgb\nnum_pset = 300\ntunefeatures=True #feature_params\nnum_fset = 80\njustretrain = False\n\n#################################################################xgb hyperparameters\nif exists(PRE_MOD_FOLDER+f\"/model_nof_{previous_version}/skipping_pre_next{previous_version}\"):\n    with open(PRE_MOD_FOLDER+f\"/model_nof_{previous_version}/skipping_pre_next{previous_version}\",'rb') as fp:\n        #if new feature_prebest set to False\n        skipping_pre = pickle.load(fp)\nelse:\n    skipping_pre = False\n\nwith open(PRE_MOD_FOLDER+f\"/model_nof_{previous_version}/params_xgb_best{previous_version}\", 'rb') as f: \n    params_xgb_prebest = pickle.load(f)\n\nwith open(PRE_MOD_FOLDER+f\"/model_nof_{previous_version}/params_xgb_score{previous_version}\", \"rb\") as fp:   \n    params_xgb_prescore = pickle.load(fp)\nif skipping_pre==False:\n    params_xgb_prescore = []\n\n\n#################################################################feature parameterss\nskipping_pre_feature = False #placeholder, depending on the new params_xgb\n\nwith open(PRE_MOD_FOLDER+f\"/model_nof_{previous_version}/feature_best{previous_version}\", \"rb\") as fp:\n    feature_prebest = pickle.load(fp)\n\nwith open(PRE_MOD_FOLDER+f\"/model_nof_{previous_version}/feature_score{previous_version}\", \"rb\") as fp:   \n    feature_prescore = pickle.load(fp)\npre_score = feature_prescore[-1][0]\n\n\nif justretrain:\n    feature_prescore = []\n    params_xgb_prescore = []\n    \nprint('tune settings made')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:01:30.043536Z","iopub.execute_input":"2022-01-30T01:01:30.043867Z","iopub.status.idle":"2022-01-30T01:01:30.096179Z","shell.execute_reply.started":"2022-01-30T01:01:30.043829Z","shell.execute_reply":"2022-01-30T01:01:30.095353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skipping_pre","metadata":{"execution":{"iopub.status.busy":"2022-01-28T05:35:09.377154Z","iopub.execute_input":"2022-01-28T05:35:09.377929Z","iopub.status.idle":"2022-01-28T05:35:09.383468Z","shell.execute_reply.started":"2022-01-28T05:35:09.377884Z","shell.execute_reply":"2022-01-28T05:35:09.382527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Best up-to-now\n```\n{'learning_rate': 0.41,  \n 'max_depth': 5, \n 'min_child_weight': 1.1, \n 'subsample': 0.7, \n 'colsample_bytree': 0.6, \n 'reg_lambda': 2.0, \n 'reg_alpha': 1.0}\n```","metadata":{}},{"cell_type":"code","source":"params_xgb_prebest\n","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:01:35.125589Z","iopub.execute_input":"2022-01-30T01:01:35.126378Z","iopub.status.idle":"2022-01-30T01:01:35.136520Z","shell.execute_reply.started":"2022-01-30T01:01:35.126331Z","shell.execute_reply":"2022-01-30T01:01:35.135542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Best up-to-now\n```\n{\n{'std_lr_15': 30, 'std_Mkt_lrt_15': 10, 'std_Crypto_Index': 30, 'rsi': 30, \n'adx': 50, 'macd_sig': 15, 'macd_s': 10, 'macd_l': 60, 'lrtn': 50, \n'fastk2': 10, 'fastk1': 15, 'beta_s': '6h', 'beta_l': '2d', 'vol_sum': 15}\n}\n{'std_lr_15': 30,\n 'std_Mkt_lrt_15': 10,\n 'std_Crypto_Index': 30,\n 'rsi': 30,\n 'adx': 50,\n 'macd_sig': 15,\n 'macd_s': 10,\n 'macd_l': 60,\n 'lrtn': 50,\n 'fastk2': 10,\n 'fastk1': 15,\n 'beta_s': '6h',\n 'beta_l': '2d',\n 'vol_sum': 15,\n 'adx': 40,\n 'ATR':30,\n 'willr':30\n}\n```","metadata":{}},{"cell_type":"code","source":"feature_prebest","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:01:39.056729Z","iopub.execute_input":"2022-01-30T01:01:39.057201Z","iopub.status.idle":"2022-01-30T01:01:39.065043Z","shell.execute_reply.started":"2022-01-30T01:01:39.057147Z","shell.execute_reply":"2022-01-30T01:01:39.064093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Load data `new_data.ftr`","metadata":{}},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n\n#######\nprint(\"loading new_data4.ftr from folder my-crypto-data\")\ndf_train=pd.read_feather('../input/my-crypto-data/new_data4.ftr',\n                        columns=['timestamp', 'Asset_ID', 'Count', 'Open', 'High', 'Low', 'Close',\n                               'Volume', 'Target', 'Weight', 'lr_15', 'Mkt_lrt_15','Crypto_Index'])\n\nprint('finished loading')\nprint(df_train.columns)","metadata":{"execution":{"iopub.status.busy":"2022-01-27T21:01:30.245298Z","iopub.execute_input":"2022-01-27T21:01:30.2456Z","iopub.status.idle":"2022-01-27T21:01:46.663405Z","shell.execute_reply.started":"2022-01-27T21:01:30.24557Z","shell.execute_reply":"2022-01-27T21:01:46.6621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Split to `df_retrain`, `df_test`","metadata":{}},{"cell_type":"code","source":"###timestamp endpoints\ndf_train['timestamp'].quantile(0.45), df_train['timestamp'].quantile(0.95)\npd.to_datetime([df_train['timestamp'].quantile(0.45), \n                df_train['timestamp'].quantile(0.95)],\n               unit=\"s\",infer_datetime_format=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-27T21:01:51.681933Z","iopub.execute_input":"2022-01-27T21:01:51.682258Z","iopub.status.idle":"2022-01-27T21:01:52.665202Z","shell.execute_reply.started":"2022-01-27T21:01:51.682226Z","shell.execute_reply":"2022-01-27T21:01:52.664201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**make tune/train and test dataset**","metadata":{}},{"cell_type":"code","source":"df_test = df_train[df_train['timestamp']>=df_train['timestamp'].quantile(0.95)]\ndf_retrain = df_train[(df_train['timestamp']>df_train['timestamp'].quantile(0.45)) & \\\n                      (df_train['timestamp']<df_train['timestamp'].quantile(0.95))]\n\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T16:02:25.786413Z","iopub.execute_input":"2022-01-21T16:02:25.78682Z","iopub.status.idle":"2022-01-21T16:02:27.872129Z","shell.execute_reply.started":"2022-01-21T16:02:25.786785Z","shell.execute_reply":"2022-01-21T16:02:27.871422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Feature engineering\n\n[TA-LIB doc: https://mrjbq7.github.io/ta-lib/funcs.html](https://mrjbq7.github.io/ta-lib/funcs.html)","metadata":{"papermill":{"duration":0.071242,"end_time":"2021-11-17T01:59:53.170961","exception":false,"start_time":"2021-11-17T01:59:53.099719","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def log_return(series, periods=5):\n    return np.log(series).diff(periods)\n\ndef beta_resid(df, window): \n    num, unit = int(window[:-1]),window[-1]\n    if unit == 'h':\n        width = 60*num\n    elif unit == 'd':\n        width = 60*24*num\n    b = ((ta.MULT(df.Mkt_lrt_15,df.lr_15).rolling(width).mean())/ \\\n        (ta.MULT(df.Mkt_lrt_15,df.Mkt_lrt_15).rolling(width).mean())).rename(f\"beta_{window}\")\n    b = b.replace([np.nan,np.inf,-np.inf], 0)\n    resids = ta.SUB(df.lr_15, ta.MULT(b, df.Mkt_lrt_15)).rename(f\"lr_15_resid_{window}\")\n    return pd.concat([b, resids],axis=1)\n\ndef lag_features(df,fastk1,fastk2,adx,macd_s,macd_l,macd_sig,vol_sum,rsi,std_Crypto_Index,std_lr_15,std_Mkt_lrt_15,**kwargs):    \n    ####TECH indicators\n    df['slowK'], df['slowD'] = ta.STOCH(df.High, df.Low, df.Close, \n                                        fastk_period=fastk1, slowk_period=int(3*fastk1/5), slowd_period=int(3*fastk1/5),\n                                        slowk_matype=0, slowd_matype=0)\n    df['fastK'], df['fastD'] = ta.STOCHF(df.High, df.Low, df.Close,\n                                         fastk_period=fastk2, fastd_period=int(3*fastk2/5), \n                                         fastd_matype=0)\n    df[f'rsi_{rsi}'] = ta.RSI(df['Close'], timeperiod=rsi)\n    df[f'macd_{macd_s}_{macd_l}'],df[f'macd_signal_{macd_sig}'], df['macd_hist'] = \\\n                ta.MACD(df['Close'],fastperiod=macd_s, slowperiod=macd_l, signalperiod=macd_sig)\n    df[f'adx_{adx}'] = ta.ADX(df['High'], df['Low'],df['Close'], timeperiod=adx)#Average Directional Movement Index\n    df[f'vol_sum_{vol_sum}'] = ta.SMA(df['Volume'],vol_sum)*vol_sum\n    ####std volatility\n    df[f'std_lr_15_{std_lr_15}'] = ta.STDDEV(df.lr_15,timeperiod=std_lr_15, nbdev=1)\n    df[f'std_Mkt_lrt_15_{std_Mkt_lrt_15}'] = ta.STDDEV(df.Mkt_lrt_15,timeperiod=std_Mkt_lrt_15, nbdev=1)\n    df[f'std_Crypto_Index_{std_Crypto_Index}'] = ta.STDDEV(df.Crypto_Index,timeperiod=std_Crypto_Index, nbdev=1)\n    #####new after mod 49\n    #df[f\"ATR_{kwargs['ATR']}\"] = ta.ATR(df['High'], df['Low'],df['Close'], timeperiod=kwargs['ATR'])\n    #df['TRENDLINE'] =ta.HT_TRENDLINE(df['Open'])\n    #df[f\"willr_{kwargs['willr']}\"] = ta.WILLR(df['High'], df['Low'],df['Close'], timeperiod=kwargs['willr'])\n    \n\n\n\ndef get_features(df_feat, fpara_dict):\n    pd.options.mode.chained_assignment = None  # default='warn'\n    df_feat[[f\"beta_{fpara_dict['beta_s']}\",f\"lr_15_resid_{fpara_dict['beta_s']}\"]] = beta_resid(df_feat, window = fpara_dict['beta_s'])\n    df_feat[[f\"beta_{fpara_dict['beta_l']}\",f\"lr_15_resid_{fpara_dict['beta_l']}\"]] = beta_resid(df_feat, window = fpara_dict['beta_l'])\n    df_feat[f\"lrtn_index_{fpara_dict['lrtn']}\"] = log_return(df_feat.Crypto_Index, fpara_dict['lrtn'])\n    lag_features(df_feat, **fpara_dict)\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:42:47.073198Z","iopub.execute_input":"2022-01-26T17:42:47.073712Z","iopub.status.idle":"2022-01-26T17:42:47.091185Z","shell.execute_reply.started":"2022-01-26T17:42:47.073665Z","shell.execute_reply":"2022-01-26T17:42:47.090536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. hypeparameters and model configure\n\n[xgboost parameters: https://xgboost.readthedocs.io/en/stable/parameter.html](https://xgboost.readthedocs.io/en/stable/parameter.html)","metadata":{}},{"cell_type":"code","source":"####################################################################################parameters placeholder\n#https://xgboost.readthedocs.io/en/stable/parameter.html#general-parameters\nparams_general ={'booster': 'gbtree', 'verbosity':0, 'validate_parameters': 1}\n\n#https://xgboost.readthedocs.io/en/stable/parameter.html#parameters-for-tree-booster\nparams_booster ={\n    'learning_rate': 0.3,#check\n    'min_split_loss': 0, #gamma. check\n    'max_depth': 6,#check\n    'min_child_weight': 1, #instance weight (hessian). check\n    'subsample': 0.8,#check\n    'colsample_bytree': 1,#check\n    'reg_lambda': 1,#L2 regularization term on weights\n    'reg_alpha': 0, #L1 regularization term on weights\n    'max_delta_step': 0,\n    'scale_pos_weight': 1,\n    'tree_method': 'gpu_hist', #hist\n    'predictor': 'gpu_predictor',\n    'num_parallel_tree': 1\n}\n#https://xgboost.readthedocs.io/en/stable/parameter.html#learning-task-parameters\nparams_learning={\n    'objective': 'reg:squarederror', 'eval_metric': 'rmse',\n    'base_score': 0.5, 'seed': 2021\n}\n\n#https://xgboost.readthedocs.io/en/stable/parameter.html#command-line-parameters\nparams_train={\n    'num_boost_round':500, #alias as 'n_estimators' in sklearn api\n    'early_stopping_rounds':50, 'verbose_eval':False\n}\n\nparams_xgb = {**params_general, **params_booster, **params_learning}\n\nprint(\"finish paramx_xgb initialization\")","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:42:56.941034Z","iopub.execute_input":"2022-01-26T17:42:56.941331Z","iopub.status.idle":"2022-01-26T17:42:56.949935Z","shell.execute_reply.started":"2022-01-26T17:42:56.941297Z","shell.execute_reply":"2022-01-26T17:42:56.949023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Hyperparameters selection(xgb.cv)\n\n- parameters list: [xgboost-parameter](https://xgboost.readthedocs.io/en/stable/parameter.html)\n- One asset benchmark(ID=1, Bitcoin)","metadata":{}},{"cell_type":"code","source":"print(f\"skipping params_xgb_pre: {skipping_pre}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:43:00.458463Z","iopub.execute_input":"2022-01-26T17:43:00.458771Z","iopub.status.idle":"2022-01-26T17:43:00.464477Z","shell.execute_reply.started":"2022-01-26T17:43:00.458736Z","shell.execute_reply":"2022-01-26T17:43:00.463693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##core tuning cv function\ndef tune_para(search_params, dtrain, skip_pre=False):\n    c=0\n    for psets in search_params:\n        params_xgb.update(psets)\n        c+=1\n        if skip_pre:\n            if psets in [s[2] for s in params_xgb_prescore]:\n                print(f'skipping this cv batch {c}/{len(search_params)}')\n                continue\n        result=xgb.cv(params_xgb, dtrain, \n            num_boost_round=params_train['num_boost_round'], \n            nfold=5,\n            metrics=['rmse'], \n            seed=0,verbose_eval=False,as_pandas=True,\n            shuffle=False,\n            early_stopping_rounds=params_train['early_stopping_rounds'])\n        #best score among num_boost_round\n        psets_score = (round(result['test-rmse-mean'].min(),7),result.shape[0], psets)\n        params_xgb_prescore.append(psets_score)\n        print(f\"finished {c}/{len(search_params)}\")\n    return params_xgb_prescore","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:43:14.841157Z","iopub.execute_input":"2022-01-26T17:43:14.841922Z","iopub.status.idle":"2022-01-26T17:43:14.849371Z","shell.execute_reply.started":"2022-01-26T17:43:14.841866Z","shell.execute_reply":"2022-01-26T17:43:14.848735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gridsearch_params = {\n    'learning_rate': [i/100 for i in range(30,51,10)],\n    'max_depth': [6,10],\n    'subsample' : [i/100 for i in range(70,81,10)],\n    'colsample_bytree' : [i/100 for i in range(50,81,10)],\n    'min_child_weight' : [i/100 for i in range(100,201,10)],\n    'reg_lambda' : [i/100 for i in range(100,301,20)] ,\n    'reg_alpha' : [i/100 for i in range(40,201,20)]\n}\nsearch_params = list(ParameterSampler(param_distributions = gridsearch_params, n_iter = num_pset))\n\nif params_xgb_prebest in search_params:\n    pass\nelse:\n    search_params.append(params_xgb_prebest)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:43:21.668647Z","iopub.execute_input":"2022-01-26T17:43:21.669168Z","iopub.status.idle":"2022-01-26T17:43:21.688664Z","shell.execute_reply.started":"2022-01-26T17:43:21.669135Z","shell.execute_reply":"2022-01-26T17:43:21.687755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make tune data\ndf = df_retrain[df_retrain[\"Asset_ID\"] == 1]\npd.options.mode.chained_assignment = None  # default='warn'\ndf = get_features(df, feature_prebest)\ndf.dropna(axis = 0, inplace= True)#for lag_features missing rows:<100\ndtrain=xgb.DMatrix(df.drop(['timestamp', 'Asset_ID','Target','Weight'],axis=1),label= df['Target'])\n\ndel df\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##############################start tuning and update params_xgb, logging ,setup skipping_pre_feature\nif tunepara:\n    print('Start tuning')\n    params_xgb_score=tune_para(search_params,dtrain,skip_pre = skipping_pre)\n    del dtrain\n    gc.collect()\n    ##score ranking rules min: rmse,min_child_weight,learning_rate,reg_alpha\n    params_xgb_score.sort(key=lambda x: (x[0],x[2]['min_child_weight'],x[2]['learning_rate'],x[2]['reg_alpha']))\n    params_xgb.update(params_xgb_score[0][2])\n    \n    with open(f\"./model_nof_{params_version}/params_xgb_score{params_version}\", \"wb\") as fp:   \n        pickle.dump(params_xgb_score, fp)\n    with open(f\"./model_nof_{params_version}/params_xgb_best{params_version}\", 'wb') as f: \n        pickle.dump(params_xgb_score[0][2],f)\n    if params_xgb_score[0][2] == params_xgb_prebest:\n        print('best params_xgb does not change!')\n        skipping_pre_feature = True\n    else:\n        print('best params_xgb differs!')\n        skipping_pre_feature = False\n        feature_prescore = []\nelse:\n    params_xgb.update(params_xgb_prebest)\n    \n    with open(f\"./model_nof_{params_version}/params_xgb_score{params_version}\", \"wb\") as fp:   \n        pickle.dump(params_xgb_prescore, fp)\n    with open(f\"./model_nof_{params_version}/params_xgb_best{params_version}\", 'wb') as f: \n        pickle.dump(params_xgb_prebest,f)\n    skipping_pre_feature = True","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:45:57.74308Z","iopub.execute_input":"2022-01-26T17:45:57.743737Z","iopub.status.idle":"2022-01-26T17:45:57.75473Z","shell.execute_reply.started":"2022-01-26T17:45:57.743702Z","shell.execute_reply":"2022-01-26T17:45:57.753907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_xgb","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:46:00.747156Z","iopub.execute_input":"2022-01-26T17:46:00.747418Z","iopub.status.idle":"2022-01-26T17:46:00.754209Z","shell.execute_reply.started":"2022-01-26T17:46:00.74739Z","shell.execute_reply":"2022-01-26T17:46:00.753486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Feature Selection","metadata":{}},{"cell_type":"code","source":"print(f\"skipping feature_pre: {skipping_pre_feature}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:46:04.238042Z","iopub.execute_input":"2022-01-26T17:46:04.238318Z","iopub.status.idle":"2022-01-26T17:46:04.242659Z","shell.execute_reply.started":"2022-01-26T17:46:04.238289Z","shell.execute_reply":"2022-01-26T17:46:04.241939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model_for_asset(df_train,asset_id, fpara_dict):\n    pd.options.mode.chained_assignment = None\n    dftrain = df_train[df_train[\"Asset_ID\"] == asset_id].copy()\n    dftrain = get_features(df_feat=dftrain, fpara_dict=fpara_dict)\n    dftrain.dropna(axis = 0, inplace= True)\n    dmat_train=xgb.DMatrix(data = dftrain.drop(['timestamp', 'Asset_ID','Target','Weight'],\n                                        axis=1),\n                    label= dftrain['Target'])\n    del dftrain\n    gc.collect()\n    model = xgb.train(params_xgb, dtrain=dmat_train, \n                    evals=[(dmat_train,'train')],\n                    **params_train)\n    return model\n\ndef model_reload_train(df_train, fpara_dict):\n    models={}\n    for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n        models[asset_id] = train_model_for_asset(df_train,asset_id, fpara_dict)\n    return models\n\ndef make_testset(df, fpara_dict):\n    ###consistent timestamp for all 14 assets\n    df2 = df.copy()\n    # df2 = df.set_index(\"timestamp\").copy()\n    # ind = df2.index.unique()\n    # def reindex(df):\n    #     df = df.reindex(range(ind[0],ind[-1]+60,60),method='nearest')\n    #     df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    #     return df\n    # df2 = df2.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\n    ###add features\n    df2 = df2.groupby('Asset_ID').apply(lambda x: get_features(x, fpara_dict))\n    return df2.dropna(axis = 0)#.reset_index()\n\ndef weighted_correlation(a, b, weights):\n  w = np.ravel(weights)\n  a = np.ravel(a)\n  b = np.ravel(b)\n  sum_w = np.sum(w)\n  mean_a = np.sum(a * w) / sum_w\n  mean_b = np.sum(b * w) / sum_w\n  var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n  var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n  cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n  corr = cov / np.sqrt(var_a * var_b)\n  return corr","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:46:21.91404Z","iopub.execute_input":"2022-01-26T17:46:21.914693Z","iopub.status.idle":"2022-01-26T17:46:21.931102Z","shell.execute_reply.started":"2022-01-26T17:46:21.914639Z","shell.execute_reply":"2022-01-26T17:46:21.930521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_prebest","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:05:14.245353Z","iopub.execute_input":"2022-01-30T01:05:14.245672Z","iopub.status.idle":"2022-01-30T01:05:14.252661Z","shell.execute_reply.started":"2022-01-30T01:05:14.245639Z","shell.execute_reply":"2022-01-30T01:05:14.251833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_grid = {\n'std_lr_15': [30,40,60,120,240],\n 'std_Mkt_lrt_15': [10,15,30,50,60,120,240],\n 'std_Crypto_Index': [30,40,60,120,240],\n 'macd_sig': [15,20],\n 'macd_s': [10,15,30],\n 'macd_l': [60,40],\n 'lrtn': [50,30,15],\n 'fastk2': [10,30,50],\n 'fastk1': [15,20,40],\n 'beta_s': ['6h','12h'],\n 'beta_l': ['2d'],\n 'vol_sum': [15,30],\n 'adx': [40,50,60],\n 'rsi':[30,40,50,60]\n}\n\np_grid = list(ParameterSampler(param_distributions = param_grid, n_iter = num_fset))\n\nif feature_prebest in p_grid:\n    pass\nelse:\n    p_grid.append(feature_prebest)\n\nlen(p_grid)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:02:56.199875Z","iopub.execute_input":"2022-01-30T01:02:56.200212Z","iopub.status.idle":"2022-01-30T01:02:56.215228Z","shell.execute_reply.started":"2022-01-30T01:02:56.200177Z","shell.execute_reply":"2022-01-30T01:02:56.214427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if tunefeatures:\n    c = 0\n    for psets in p_grid:\n        c += 1\n        if skipping_pre_feature:\n            if psets in [s[1] for s in feature_prescore]:\n                print(f\"skipping scoring for feature parameters set{c} /{len(p_grid)}\")\n                continue\n        #############################################train model\n        models = model_reload_train(df_train=df_retrain, fpara_dict = psets)\n        #############################################score model with the features' params\n        print(f\"scoring for feature parameters set{c} /{len(p_grid)}\")\n        df_test2 = make_testset(df = df_test, fpara_dict = psets)\n        result_frame = []\n        for id in range(0,14):\n            model = models[id]\n            x = df_test2[df_test2['Asset_ID']==id]\n            x['Pred'] = model.predict(xgb.DMatrix(x[model.feature_names]))\n            result_frame.append(x[['timestamp','Asset_ID','Weight','Target','Pred']])\n        result = pd.concat(result_frame, axis=0)\n        score = weighted_correlation(a=result['Target'],b=result['Pred'],weights=result['Weight'])\n        feature_prescore.append((score,psets))##add to feature_score logs\n    ##########################\n    feature_prescore.sort(key=lambda y: y[0])\n    final_score, final_fpara_dict = feature_prescore[-1]\nelse:\n    print(\"make score for new params_xgb\")\n    models = model_reload_train(df_train=df_retrain, fpara_dict = feature_prebest)\n    df_test2 = make_testset(df = df_test, fpara_dict = feature_prebest)\n    result_frame = []\n    for id in range(0,14):\n        model = models[id]\n        x = df_test2[df_test2['Asset_ID']==id]\n        x['Pred'] = model.predict(xgb.DMatrix(x[model.feature_names]))\n        result_frame.append(x[['timestamp','Asset_ID','Weight','Target','Pred']])\n    result = pd.concat(result_frame, axis=0)\n    score = weighted_correlation(a=result['Target'],b=result['Pred'],weights=result['Weight'])\n    feature_prescore.append((score, feature_prebest))\n    final_score, final_fpara_dict = score, feature_prebest\n    \n###logging feature tune\nwith open(f\"./model_nof_{params_version}/feature_score{params_version}\", \"wb\") as fp:   \n    pickle.dump(feature_prescore, fp)\nwith open(f\"./model_nof_{params_version}/feature_best{params_version}\", \"wb\") as fp:   \n    pickle.dump(final_fpara_dict, fp)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f\"./model_nof_{params_version}/skipping_pre_next{params_version}\", \"wb\") as fp:   \n    if final_fpara_dict == feature_prebest:\n        #no change to the feature params, we can skip scored params_xgb next time\n        print('best feature params does not change!')\n        pickle.dump(True, fp)\n    else:\n        print('best feature params differs!')\n        pickle.dump(False, fp)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Train models with optimal hyperparameters and features","metadata":{}},{"cell_type":"code","source":"final_score, final_fpara_dict, pre_score","metadata":{"execution":{"iopub.status.busy":"2022-01-26T17:53:10.831991Z","iopub.execute_input":"2022-01-26T17:53:10.832479Z","iopub.status.idle":"2022-01-26T17:53:10.947653Z","shell.execute_reply.started":"2022-01-26T17:53:10.832446Z","shell.execute_reply":"2022-01-26T17:53:10.946551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_retrain(df_train, fpara_dict, alldata):\n    models={}\n    for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n        \n        models[asset_id] = train_model_for_asset(df_train, asset_id, fpara_dict)\n        if alldata:\n            print(f\"df_retrain model fitting for asset_id {asset_id}\",end='\\r')\n            models[asset_id].save_model(f'./model_nof_{params_version}/model_{asset_id}_alldata.json')\n        else:\n            print(f\"all data model fitting for asset_id {asset_id}\", end = '\\r')\n            models[asset_id].save_model(f'./model_nof_{params_version}/model_{asset_id}.json')\n    return models\n\nif final_score < pre_score:\n    print('No improvement, No train!')\nelse:    \n    models = model_retrain(df_train= df_retrain, \n                           fpara_dict= final_fpara_dict,alldata=False)\n    models_alldata = model_retrain(df_train= pd.concat([df_retrain,df_test], join='outer'), \n                                   fpara_dict= final_fpara_dict,alldata=True)\n    print(models[0].save_config())\n    print(models[0].feature_names)\n    xgb.plot_importance(models[1],height=0.8, max_num_features=20)","metadata":{"execution":{"iopub.status.busy":"2022-01-20T04:14:52.44656Z","iopub.execute_input":"2022-01-20T04:14:52.447009Z","iopub.status.idle":"2022-01-20T04:17:23.170583Z","shell.execute_reply.started":"2022-01-20T04:14:52.446976Z","shell.execute_reply":"2022-01-20T04:17:23.169769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#write readable notes\nwith open(f'./model_nof_{params_version}/README{params_version}.txt', 'a') as f:\n    notes = f\"\"\"\n    Use feature para selection: {tunefeatures}\n    Features params: {final_fpara_dict}\n    Use auto-tuned hyperpara: {tunepara}\n    Xgb-hyperparams: {params_xgb}\n    On-test-score: {final_score}\n    \"\"\"\n    f.write(f'{notes}\\n')\n# if tunepara <1:\n#     import shutil\n#     shutil.make_archive(f'./model_nof_{params_version}', 'zip', f'./model_nof_{params_version}')","metadata":{"execution":{"iopub.status.busy":"2022-01-20T04:17:23.561244Z","iopub.execute_input":"2022-01-20T04:17:23.561676Z","iopub.status.idle":"2022-01-20T04:17:27.216319Z","shell.execute_reply.started":"2022-01-20T04:17:23.561637Z","shell.execute_reply":"2022-01-20T04:17:27.215561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if final_score > pre_score:\n    print(\"Improved!!!\")\n    print(f\"{final_score} greater than {pre_score}\")\nelse:\n    print(f\"Not improved!!!\")\n    ##recover prebest params\n    with open(f\"./model_nof_{params_version}/params_xgb_best{params_version}\", 'wb') as f: \n        pickle.dump(params_xgb_prebest,f)\n    with open(f\"./model_nof_{params_version}/feature_best{params_version}\", \"wb\") as fp:   \n        pickle.dump(feature_prebest, fp)\n    with open(f\"./model_nof_{params_version}/skipping_pre_next{params_version}\", \"wb\") as fp:   \n        pickle.dump(True, fp)","metadata":{},"execution_count":null,"outputs":[]}]}