{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Crypto Forecasting - Basic NN + Feature importance\n\nA basic NN exemple, optimising the competition target, calibrated on folds from on-line Feature engineering (see: https://www.kaggle.com/lucasmorin/on-line-feature-engineering). The online-Feature engineering allows for submission with features. For the corresponding lgbm baseline see: https://www.kaggle.com/lucasmorin/online-fe-lgbm-feval-importances (no custom loss - not nubmitting). \n\nThe notebook also includes:\n- some changes to standard architecture to use the custom weighted loss in the model.\n- the architecture of the model is rather simple and regularized thanks to ton of noise. \n- for the moment the convergence is rather poor: +/-1% inside a fold. Very different results across folds. ensembling seems to improve the overall LB result a bit. \n- a feature importance solution (from @cdeotte work in the Google Ventilator Pressure Prediction Challenge) based on shuffling features. The solution can be rather slow so it is possible to deactivate it easily. a similar feature importance trough SHAP approximation of Shapley values. A feature importance plot for a nice visualisation of importance across folds (from @nyanp currently winning solution from Optiver volatility forecasting competition).\n- some practical tests: normalisation of targets, post processing (removing the market average), removing the worst models from fold 4. Not really working.  ","metadata":{"papermill":{"duration":0.008211,"end_time":"2021-11-11T23:04:25.614714","exception":false,"start_time":"2021-11-11T23:04:25.606503","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Import and options","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\n\nenv = gresearch_crypto.make_env()\n\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport gc\nimport pickle\n\nimport time\nfrom datetime import datetime\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import RobustScaler\n\nimport tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nfrom tensorflow.keras import backend as K\n\nseed = 2021\n\nDEBUG = False\nTARGET_NORM = False\nSHAP = True\nEXPLAIN = False\nLAST_FOLD = False\nMarket_Correction = False\nFilter_Bad_Models = False\n\nn_fold = 5","metadata":{"papermill":{"duration":7.371323,"end_time":"2021-11-11T23:04:32.993946","exception":false,"start_time":"2021-11-11T23:04:25.622623","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-08T23:06:48.179563Z","iopub.execute_input":"2022-01-08T23:06:48.179878Z","iopub.status.idle":"2022-01-08T23:06:53.478718Z","shell.execute_reply.started":"2022-01-08T23:06:48.179793Z","shell.execute_reply":"2022-01-08T23:06:53.477878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some definitions","metadata":{}},{"cell_type":"code","source":"fold = 4\npath = '../input/on-line-feature-engineering/'\ntrain = pd.read_parquet(path+'train_fold_'+str(fold)+'.parquet')\n\nmean = pd.read_parquet(path+'mean_fold_'+str(fold)+'.parquet')\nstd = pd.read_parquet(path+'std_fold_'+str(fold)+'.parquet')\n\nnumerical_columns = [col for col in train.columns if col not in {'timestamp', 'Target', 'Target_M','weights','Asset_ID'}]\ncategorical_columns = ['Asset_ID']\ntarget_columns = ['Target']\ncols = numerical_columns + categorical_columns\n\nasset_nunique = train['Asset_ID'].nunique()\nprint('asset_nunique:',asset_nunique)\n\nasset_details = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\n\n#create dictionnary of weights\ndict_weights = {}\nfor i in range(asset_details.shape[0]):\n    dict_weights[asset_details.iloc[i,0]] = asset_details.iloc[i,1]","metadata":{"papermill":{"duration":7.424399,"end_time":"2021-11-11T23:04:40.451533","exception":false,"start_time":"2021-11-11T23:04:33.027134","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-08T23:06:53.480677Z","iopub.execute_input":"2022-01-08T23:06:53.480935Z","iopub.status.idle":"2022-01-08T23:06:57.717417Z","shell.execute_reply.started":"2022-01-08T23:06:53.480903Z","shell.execute_reply":"2022-01-08T23:06:57.716698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model definition","metadata":{}},{"cell_type":"code","source":"hidden_units = (256,64,16,4)\nhidden_noise = np.array((5,2,1,0.5,0.25))\n\nfrom functools import partial\n\ndef corr_loss(y_true, y_pred):\n    x = tf.cast(y_true, tf.float32)\n    y = tf.cast(y_pred, tf.float32)\n    mx = K.mean(x)\n    my = K.mean(y)\n    xm, ym = x-mx, y-my\n    r_num = K.sum(tf.multiply(xm,ym))\n    r_den = K.sqrt(tf.multiply(K.sum(K.square(xm)), K.sum(K.square(ym))))\n    r = r_num / r_den\n    r = K.maximum(K.minimum(r, 1.0), -1.0)\n    return - r\n\ndef wcorr_loss(y_true, y_pred, w):\n    x = tf.cast(y_true, tf.float32)\n    y = tf.cast(y_pred, tf.float32)\n    w = tf.cast(w, tf.float32)\n    wmx = K.sum(tf.multiply(x, w)) / K.sum(w)\n    wmy = K.sum(tf.multiply(y, w)) / K.sum(w)\n    xm, ym = x-wmx, y-wmy\n    tfwcovxy = K.sum( tf.multiply(tf.multiply(xm, w),  ym)) / K.sum(w)\n    tfwcovxx = K.sum( tf.multiply(tf.multiply(xm, w),  xm)) / K.sum(w)\n    tfwcovyy = K.sum( tf.multiply(tf.multiply(ym, w),  ym)) / K.sum(w)\n    r = tfwcovxy / K.sqrt(tf.multiply(tfwcovxx , tfwcovyy))\n    r = K.maximum(K.minimum(r, 1.0), -1.0)\n    return - r\n\ndef wcorr_fn(w):\n    def wcorr_eval(y_true, y_pred):\n        return wcorr_loss(y_true, y_pred, w)\n\ndef base_model():\n    \n    # Each instance will consist of two inputs: a single user id, and a single movie id\n    #stock_id_input = keras.Input(shape=(1,), name='stock_id')\n    weights_input = keras.Input(shape=(1,), name='weigths')\n    y_true = keras.Input(shape=(1,), name='true')\n    num_input = keras.Input(shape=(len(numerical_columns),), name='num_data')\n    \n    out = keras.layers.BatchNormalization()(num_input)\n    out = keras.layers.GaussianNoise(stddev=hidden_noise[0])(out)\n\n    # Add one or more hidden layers\n    for n in range(len(hidden_units)):\n        out = keras.layers.Dense(hidden_units[n], activation='swish')(out) \n        #out = keras.layers.Dropout(0.1)(out)\n        out = keras.layers.BatchNormalization()(out)\n        out = keras.layers.GaussianNoise(stddev=hidden_noise[n])(out)\n      \n    # A single output: our predicted rating\n    out = keras.layers.Dense(1, activation='linear', name='prediction')(out)\n    \n    model = keras.Model(\n    inputs = [weights_input, y_true, num_input], #[stock_id_input, weights_input, y_true, num_input],\n    outputs = out,\n    )\n    \n    model.add_loss(wcorr_loss(y_true, out, weights_input))\n    \n    model.compile(\n        keras.optimizers.Adam(learning_rate=0.00005),\n        loss = None,#'MSE',\n        metrics=[wcorr_fn(weights_input)],\n    )\n    \n    return model","metadata":{"papermill":{"duration":0.05438,"end_time":"2021-11-11T23:04:40.521954","exception":false,"start_time":"2021-11-11T23:04:40.467574","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-09T20:45:36.701504Z","iopub.execute_input":"2022-01-09T20:45:36.702349Z","iopub.status.idle":"2022-01-09T20:45:36.813061Z","shell.execute_reply.started":"2022-01-09T20:45:36.702166Z","shell.execute_reply":"2022-01-09T20:45:36.811842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model calibration","metadata":{}},{"cell_type":"code","source":"models = {}\nimportances = []\ndf_scores = []\n\nn_seed = 2 if DEBUG else 5\nn_epochs = 10 if DEBUG else 300\nfolds = [4] if LAST_FOLD else range(n_fold-1, -1, -1)\n\nfor fold in folds:\n    print('Fold: '+str(fold))\n\n    train = pd.read_parquet(path+'train_fold_'+str(fold)+'.parquet')\n    test = pd.read_parquet(path+'test_fold_'+str(fold)+'.parquet')\n    \n    mean = pd.read_parquet(path+'mean_fold_'+str(fold)+'.parquet')\n    std = pd.read_parquet(path+'std_fold_'+str(fold)+'.parquet')\n    \n    train[numerical_columns] = (train[numerical_columns]-mean.transpose()[numerical_columns].iloc[0].values.squeeze()).div(std.transpose()[numerical_columns].iloc[0]).fillna(0)\n    test[numerical_columns] = (test[numerical_columns]-mean.transpose()[numerical_columns].iloc[0].values.squeeze()).div(std.transpose()[numerical_columns].iloc[0]).fillna(0)\n    \n    if DEBUG:\n        timestamp_sample_train = train.timestamp.unique()[:np.int(len(train.timestamp.unique())*0.05)]\n        timestamp_sample_test = test.timestamp.unique()[:np.int(len(test.timestamp.unique())*0.05)]\n        train = train[train.timestamp.isin(timestamp_sample_train)]\n        test = test[test.timestamp.isin(timestamp_sample_test)]\n         \n    train = train[~train.Target.isna()]\n    test = test[~test.Target.isna()]\n\n    train['weights'] = train.Asset_ID.map(dict_weights).astype('float32')\n    test['weights'] = test.Asset_ID.map(dict_weights).astype('float32')\n\n    if TARGET_NORM:\n        target_mean = np.mean(train['Target'])\n        target_std = np.std(train['Target'])\n        train['Target'] = (train['Target']-target_mean)/target_std\n        test['Target'] = (test['Target']-target_mean)/target_std\n        \n    weights_train = train[['weights']]\n    weights_test = test[['weights']]\n    \n    for seed in range(n_seed):\n        \n        tf.random.set_seed(seed)\n        \n        print('Fold: '+str(fold)+ ' - seed: '+str(seed))\n    \n        model = base_model()\n\n        es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-05, patience=25, verbose=1,mode='min',restore_best_weights=True)\n\n        plateau = tf.keras.callbacks.ReduceLROnPlateau(\n            monitor='val_loss', factor=0.5, patience=10, verbose=1,\n            mode='min')\n\n        hist = model.fit([weights_train, train['Target'],train[numerical_columns]], \n                  train['Target'],\n                  batch_size=1024,\n                  epochs=n_epochs,\n                  validation_data=([weights_test, test['Target'], test[numerical_columns]], test['Target']),\n                  callbacks=[plateau,es],\n                  shuffle=True,verbose = 0)\n        \n        plt.plot(hist.history['val_loss'], label= 'fold '+str(fold)+' seed '+str(seed))\n        df_scores.append((fold, seed, -min(hist.history['val_loss'])))\n        \n        final_model = keras.Model(model.input[2],model.output)\n        \n        final_model.save('model_'+str(fold)+ '_seed_'+str(seed))\n        \n        models[(seed,fold)] = final_model\n    \n    plt.legend(loc=\"upper left\", bbox_to_anchor=(1, 0.5))\n    plt.show()\n\n    #only explain one seed per fold...\n    if EXPLAIN:\n        #print(' Computing NN feature importance...')\n        results = []\n        # COMPUTE BASELINE (NO SHUFFLE)\n        oof_preds = model.predict([weights_test, test['Target'],test[numerical_columns]], verbose=0).squeeze() \n        baseline_loss = wcorr_loss(test['Target'], oof_preds, test['weights'])        \n\n        for k in cols:\n            # print(k)\n            # SHUFFLE FEATURE K\n            save_col = test[k].copy()\n            np.random.shuffle(test[k].values)\n\n            # COMPUTE OOF MAE WITH FEATURE K SHUFFLED\n            oof_preds = model.predict([weights_test, test['Target'],test[numerical_columns]], verbose=0).squeeze() \n            loss  = wcorr_loss(test['Target'], oof_preds, test['weights'])  \n            results.append(loss - baseline_loss )\n            test[k] = save_col\n\n            del save_col, oof_preds\n            gc.collect()\n\n        importances.append(results)","metadata":{"papermill":{"duration":628.31564,"end_time":"2021-11-11T23:15:08.853707","exception":false,"start_time":"2021-11-11T23:04:40.538067","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-08T23:06:57.738515Z","iopub.execute_input":"2022-01-08T23:06:57.739186Z","iopub.status.idle":"2022-01-08T23:07:14.293047Z","shell.execute_reply.started":"2022-01-08T23:06:57.739151Z","shell.execute_reply":"2022-01-08T23:07:14.292336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results","metadata":{}},{"cell_type":"code","source":"df_results = pd.DataFrame(df_scores,columns=['fold','seed','score']).pivot(index='fold',columns='seed',values='score')\n\ndf_results.loc['seed_mean']= df_results.mean(numeric_only=True, axis=0)\ndf_results.loc[:,'fold_mean'] = df_results.mean(numeric_only=True, axis=1)\ndf_results","metadata":{"execution":{"iopub.status.busy":"2022-01-08T23:07:14.29531Z","iopub.execute_input":"2022-01-08T23:07:14.295711Z","iopub.status.idle":"2022-01-08T23:07:14.320801Z","shell.execute_reply.started":"2022-01-08T23:07:14.295673Z","shell.execute_reply":"2022-01-08T23:07:14.320066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SHAP - explainability 2","metadata":{}},{"cell_type":"code","source":"import shap\n\nif SHAP:\n    SHAP_values = []\n    for j in folds:\n        for i in range(n_seed):\n            explainer = shap.GradientExplainer(models[(i,j)], train[numerical_columns].iloc[:1000].values)\n            shap_values = explainer.shap_values(test[numerical_columns].iloc[:1000].values)\n            SHAP_values.append(np.mean(np.abs(shap_values[0]),axis=0))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T23:50:03.484494Z","iopub.execute_input":"2022-01-08T23:50:03.484776Z","iopub.status.idle":"2022-01-08T23:50:05.734833Z","shell.execute_reply.started":"2022-01-08T23:50:03.484746Z","shell.execute_reply":"2022-01-08T23:50:05.733918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"from nyanp's Optiver solution.","metadata":{"papermill":{"duration":0.279084,"end_time":"2021-11-11T23:15:09.942078","exception":false,"start_time":"2021-11-11T23:15:09.662994","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_importance(importances, features_names = cols, PLOT_TOP_N = 20, figsize=(10, 10)):\n    importance_df = pd.DataFrame(data=importances, columns=features_names)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    #ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    sns.boxplot(data=sorted_importance_df[plot_cols],\n                orient='h',\n                ax=ax)\n    plt.show()\n\nif EXPLAIN:\n    plot_importance(np.array(importances),cols, PLOT_TOP_N = 20, figsize=(10, 20))\n    \nif SHAP:\n    plot_importance(np.array(SHAP_values),numerical_columns, PLOT_TOP_N = 20, figsize=(10, 20))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T23:17:21.196074Z","iopub.execute_input":"2022-01-08T23:17:21.196346Z","iopub.status.idle":"2022-01-08T23:17:21.982567Z","shell.execute_reply.started":"2022-01-08T23:17:21.196312Z","shell.execute_reply":"2022-01-08T23:17:21.981884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pickle.dump(models, open('NN_models.pkl', 'wb'))\npickle.dump(importances, open('importances.pkl', 'wb'))\npickle.dump(cols, open('featrures.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T23:17:21.984304Z","iopub.execute_input":"2022-01-08T23:17:21.984568Z","iopub.status.idle":"2022-01-08T23:17:21.990665Z","shell.execute_reply.started":"2022-01-08T23:17:21.984534Z","shell.execute_reply":"2022-01-08T23:17:21.989905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"dict_score = {}\n\nfor k in df_scores:\n    dict_score[(k[1],k[0])]=k[2]","metadata":{"execution":{"iopub.status.busy":"2022-01-08T23:07:19.999502Z","iopub.status.idle":"2022-01-08T23:07:20.000114Z","shell.execute_reply.started":"2022-01-08T23:07:19.999861Z","shell.execute_reply":"2022-01-08T23:07:19.999885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RunningMean:\n    def __init__(self, WIN_SIZE=20, n_size = 1):\n        self.n = 0\n        self.mean = np.zeros(n_size)\n        self.cum_sum = 0\n        self.past_value = 0\n        self.WIN_SIZE = WIN_SIZE\n        self.windows = collections.deque(maxlen=WIN_SIZE+1)\n        \n    def clear(self):\n        self.n = 0\n        self.windows.clear()\n\n    def push(self, x):\n        #currently fillna with past value, might want to change that\n        x = fillna_npwhere(x, self.past_value)\n        self.past_value = x\n        \n        self.windows.append(x)\n        self.cum_sum += x\n        \n        if self.n < self.WIN_SIZE:\n            self.n += 1\n            self.mean = self.cum_sum / float(self.n)\n            \n        else:\n            self.cum_sum -= self.windows.popleft()\n            self.mean = self.cum_sum / float(self.WIN_SIZE)\n\n    def get_mean(self):\n        return self.mean if self.n else np.zeros(n_size)\n\n    def __str__(self):\n        return \"Current window values: {}\".format(list(self.windows))\n\n# Temporary removing njit as it cause many bugs down the line\n# Problems mainly due to data types, I have to find where I need to constraint types so as not to make njit angry\n#@njit\ndef fillna_npwhere(array, values):\n    if np.isnan(array.sum()):\n        array = np.where(np.isnan(array), values, array)\n    return array","metadata":{"execution":{"iopub.status.busy":"2022-01-08T23:07:20.001203Z","iopub.status.idle":"2022-01-08T23:07:20.001756Z","shell.execute_reply.started":"2022-01-08T23:07:20.001521Z","shell.execute_reply":"2022-01-08T23:07:20.001545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Market_Correction = False\nFilter_Bad_Models = True\nlevel = 0.045\n\niter_test = env.iter_test()\n\ndict_RM = pickle.load(open('../input/on-line-feature-engineering/dict_RM_4.pkl', 'rb'))\ndict_RM_M = pickle.load(open('../input/on-line-feature-engineering/dict_RM_M_4.pkl', 'rb'))\ndict_MM = pickle.load(open('../input/on-line-feature-engineering/dict_MM_4.pkl', 'rb'))\ndict_Mr = pickle.load(open('../input/on-line-feature-engineering/dict_MR_4.pkl', 'rb'))\n\n\nimport os\nfrom random import random\n\nsampling = 0.05\n\nMA_lags = [2,5,15,30,60,120,300,1800,3750,2*3750,7*24*60]\nbeta_lags = [15,30,60,120,300,600,1800,3750,2*3750,7*24*60]\n\nFeatures_names = ['log_ret','log_ret_H','log_ret_L','log_ret_VWAP','GK_vol','RS_vol','log_Count','log_Volume','log_Dollars','log_Volume_per_trade','log_Dollars_per_trade']\nMarket_Features_names = [s+'_M' for s in Features_names]\nTime_Features_names = ['sin_month','cos_month','sin_day','cos_day','sin_hour','cos_hour','sin_minute','cos_minute']\nMA_Features_names = [s+'_'+str(lag) for lag in MA_lags for s in Features_names ]\nMA_Features_M_names = [s+'_'+str(lag) for lag in MA_lags for s in Market_Features_names]\nbetas_names = ['betas_'+str(lag) for lag in beta_lags]\n\nAll_names = Features_names + Market_Features_names + Time_Features_names + MA_Features_names + MA_Features_M_names + betas_names\n#df_values = pd.DataFrame(values, columns = All_names)\n\n#not building the weights each loops\nasset_details = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\ndict_weights = {}\nfor i in range(asset_details.shape[0]):\n    dict_weights[asset_details.iloc[i,0]] = asset_details.iloc[i,1]\nweigths = np.array([dict_weights[i] for i in range(14)])\n\n# only needed when saving ?\ndtype={'Asset_ID': 'int8', 'Count': 'int32', 'row_id': 'int32', 'Count': 'int32',\n       'Open': 'float32', 'High': 'float32', 'Low': 'float32', 'Close': 'float32',\n       'Volume': 'float32', 'VWAP': 'float32'}\n#test_df = test_df.astype(dtype)\n\n#refactoring functions:\n\ndef timestamp_to_date(timestamp):\n    return(datetime.fromtimestamp(timestamp))\n\ndef Clean_df(x):\n    Asset_ID = x[:,1]\n    timestamp = x[0,0]\n    if len(Asset_ID)<14:\n        missing_ID = [i for i in range(14) if i not in Asset_ID]\n        for i in missing_ID:\n            row = np.array((timestamp,i,np.nan,np.nan,np.nan,np.nan,np.nan,np.nan,np.nan,np.nan))\n            x = np.concatenate((x,np.expand_dims(row,axis=0)))\n    x = x[np.argsort(x[:,1])]\n    return (x[:,i] for i in range(x.shape[1]))\n\ndef Base_Feature_fn(timestamp,Asset_ID,Count,O,H,L,C,Volume,VWAP):\n    VWAP = np.where(np.isinf(VWAP),(C+O)/2,VWAP)\n    base = C\n    O = O/base\n    H = H/base\n    L = L/base\n    C = C/base\n    VWAP = VWAP/base\n    Price = base\n\n    Dollars = Volume * Price\n    Volume_per_trade = Volume/Count\n    Dollars_per_trade = Dollars/Count\n\n    log_ret = np.log(C/O)\n    log_ret_H = np.log(H/C)\n    log_ret_L = np.log(C/L)\n    log_ret_VWAP = np.log(C/VWAP)\n    \n    GK_vol = (1 / 2 * np.log(H/L) ** 2 - (2 * np.log(2) - 1) * np.log(C/O) ** 2)\n    RS_vol = np.log(H/C)*np.log(H/O) + np.log(L/C)*np.log(L/O)\n\n    #return(np.transpose(np.array([Count,O,H,L,C,Price,Volume,VWAP,Dollars,Volume_per_trade,Dollars_per_trade,log_ret,GK_vol,RS_vol])))\n    \n    log_Count,log_Volume,log_Dollars,log_Volume_per_trade,log_Dollars_per_trade = np.log([Count,Volume,Dollars,Volume_per_trade,Dollars_per_trade])\n\n    return(np.transpose(np.array([log_ret,log_ret_H,log_ret_L,log_ret_VWAP,GK_vol,RS_vol,log_Count,log_Volume,log_Dollars,log_Volume_per_trade,log_Dollars_per_trade])))\n\ndef Time_Feature_fn(timestamp):\n    \n    sin_month = (np.sin(2 * np.pi * timestamp.month/12))\n    cos_month = (np.cos(2 * np.pi * timestamp.month/12))\n    sin_day = (np.sin(2 * np.pi * timestamp.day/31))\n    cos_day = (np.cos(2 * np.pi * timestamp.day/31))\n    sin_hour = (np.sin(2 * np.pi * timestamp.hour/24))\n    cos_hour = (np.cos(2 * np.pi * timestamp.hour/24))\n    sin_minute = (np.sin(2 * np.pi * timestamp.minute/60))\n    cos_minute = (np.cos(2 * np.pi * timestamp.minute/60))\n\n    return(np.array((sin_month,cos_month,sin_day,cos_day,sin_hour,cos_hour,sin_minute,cos_minute)))\n\n# to do : preprocessing per fold\n\nmean = pd.read_parquet(path+'mean_fold_'+str(fold)+'.parquet')\nstd = pd.read_parquet(path+'std_fold_'+str(fold)+'.parquet')\n\nmean = mean.transpose()[numerical_columns].values\nstd = std.transpose()[numerical_columns].values\n\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    #df = Clean_df(pd.DataFrame(x,columns=f))\n    #timestamp,Asset_ID,Count,O,H,L,C,Volume,VWAP,row_id = (test_df[col].values for col in ['timestamp','Asset_ID','Count','Open','High','Low','Close','Volume','VWAP','row_id'])\n    timestamp,Asset_ID,Count,O,H,L,C,Volume,VWAP,row_id = Clean_df(test_df.values)\n    \n    # np.array([Count,O,H,L,C,Price,Volume,VWAP,Dollars,Volume_per_trade,Dollars_per_trade,log_ret,GK_vol,RS_vol])\n    Features = Base_Feature_fn(timestamp,Asset_ID,Count,O,H,L,C,Volume,VWAP)\n\n    #removing wieghts when data is missing so that they don't appears in market\n    weigths_curr = np.where(np.isnan(O),O,weigths)\n    Market_Features = np.nansum(Features*np.expand_dims(weigths_curr,axis=1)/np.nansum(weigths_curr),axis=0)\n    #Market_Features = np.tile(Market_Features,(14,1))\n\n    #np.array((sin_month,cos_month,sin_day,cos_day,sin_hour,cos_hour,sin_minute,cos_minute))\n    time = timestamp_to_date(timestamp[0])\n    Time_Features = Time_Feature_fn(time)\n    #Time_Features = np.tile(Time_Features,(14,1))\n\n    MA_Features = []\n    MA_Features_M  = [] \n\n    for lag in MA_lags:\n        dict_RM[lag].push(Features.copy())\n        dict_RM_M[lag].push(Market_Features.copy())\n\n        MA_Features.append(dict_RM[lag].get_mean())\n        MA_Features_M.append(dict_RM_M[lag].get_mean())\n\n    #standardise w/ 3750 lag\n    ref = 3750\n\n    for i in range(len(MA_lags)):\n        if MA_lags[i] == ref:\n            MA_ref = dict_RM[MA_lags[i]].get_mean().copy()\n            MA_M_ref = dict_RM_M[MA_lags[i]].get_mean().copy()\n\n    Features[:,-6:] = (Features[:,-6:] - MA_ref[:,-6:]).copy()\n    Market_Features[-6:] = (Market_Features[-6:] - MA_M_ref[-6:]).copy()\n\n    for i in range(len(MA_lags)):\n        MA_Features[i][:,-6:] = (MA_Features[i][:,-6:] - MA_ref[:,-6:]).copy()\n        MA_Features_M[i][-6:] = (MA_Features_M[i][-6:] - MA_M_ref[-6:]).copy()\n\n    MA_Features_agg = np.concatenate(MA_Features,axis=1)\n    MA_Features_M_agg = np.concatenate(MA_Features_M)\n\n    betas = []\n\n    for lag in beta_lags:\n        dict_MM[lag].push(Market_Features[0]**2)\n        dict_Mr[lag].push(Market_Features[0]*Features[:,0])\n        betas.append(np.expand_dims(dict_Mr[lag].get_mean()/dict_MM[lag].get_mean(),axis=1))\n\n    betas = np.concatenate(betas,axis=1)\n    betas = np.nan_to_num(betas, nan=0., posinf=0., neginf=0.) \n\n    values = np.concatenate((Features,np.tile(Market_Features,(14,1)),np.tile(Time_Features,(14,1)),MA_Features_agg,np.tile(MA_Features_M_agg,(14,1)),betas),axis=1)\n    values = np.nan_to_num((values - mean)/std)\n    \n    if Filter_Bad_Models:\n        preds = np.median(np.array([models[(i,j)].predict(values) for i in range(n_seed) for j in folds if dict_score[(i,j)]>level]),axis=0).flatten()\n    else:\n        preds = np.median(np.array([models[(i,j)].predict(values) for i in range(n_seed) for j in folds]),axis=0).flatten()\n    \n    if Market_Correction:\n        market_pred = np.sum(preds * weigths_curr)/np.sum(weigths_curr).copy()\n        preds -= market_pred\n    \n    sample_prediction_df['Target'] = [preds[(row_id == rid)][0] for rid in sample_prediction_df.row_id.values]\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T23:07:20.002906Z","iopub.status.idle":"2022-01-08T23:07:20.003493Z","shell.execute_reply.started":"2022-01-08T23:07:20.003253Z","shell.execute_reply":"2022-01-08T23:07:20.003277Z"},"trusted":true},"execution_count":null,"outputs":[]}]}