{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Load the training set","metadata":{"id":"OevdFjmtar0j"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datetime import datetime","metadata":{"id":"90fqnpNQ2wd8","execution":{"iopub.status.busy":"2022-01-12T19:23:23.810957Z","iopub.execute_input":"2022-01-12T19:23:23.811264Z","iopub.status.idle":"2022-01-12T19:23:23.836770Z","shell.execute_reply.started":"2022-01-12T19:23:23.811183Z","shell.execute_reply":"2022-01-12T19:23:23.836127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"../input/g-research-crypto-forecasting/\"","metadata":{"id":"Zw_oi4NETkKv","outputId":"a95925cc-58a0-4eb6-9a9e-799ee7c79abb","execution":{"iopub.status.busy":"2022-01-12T19:23:27.325919Z","iopub.execute_input":"2022-01-12T19:23:27.326627Z","iopub.status.idle":"2022-01-12T19:23:27.330378Z","shell.execute_reply.started":"2022-01-12T19:23:27.326592Z","shell.execute_reply":"2022-01-12T19:23:27.329609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df = pd.read_csv(data_folder + 'train.csv')\nasset_details = pd.read_csv(data_folder + 'asset_details.csv')","metadata":{"id":"La0-ivRAUopA","scrolled":true,"execution":{"iopub.status.busy":"2022-01-12T19:23:29.558261Z","iopub.execute_input":"2022-01-12T19:23:29.558939Z","iopub.status.idle":"2022-01-12T19:24:25.157042Z","shell.execute_reply.started":"2022-01-12T19:23:29.558903Z","shell.execute_reply":"2022-01-12T19:24:25.156298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.head(10)","metadata":{"id":"14ZSPE6daadW","outputId":"8b8f2299-8787-49e8-c16d-ae8ff4a90fb7","execution":{"iopub.status.busy":"2022-01-12T19:24:25.158722Z","iopub.execute_input":"2022-01-12T19:24:25.158966Z","iopub.status.idle":"2022-01-12T19:24:25.184501Z","shell.execute_reply.started":"2022-01-12T19:24:25.158933Z","shell.execute_reply":"2022-01-12T19:24:25.183734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_assets = []\nfor idx in range(len(asset_details)):\n  crypto_assets.append(crypto_df[crypto_df[\"Asset_ID\"]==idx].set_index(\"timestamp\"))  ","metadata":{"id":"mwvYdwGdNQz4","execution":{"iopub.status.busy":"2022-01-12T19:24:30.036549Z","iopub.execute_input":"2022-01-12T19:24:30.037136Z","iopub.status.idle":"2022-01-12T19:24:33.744283Z","shell.execute_reply.started":"2022-01-12T19:24:30.037098Z","shell.execute_reply":"2022-01-12T19:24:33.743476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{"id":"S-oceNYiivUi"}},{"cell_type":"markdown","source":"## Dealing with missing data\n","metadata":{"id":"K9ce7eWddHfM"}},{"cell_type":"code","source":"for idx in range(len(crypto_assets)):\n  print(crypto_assets[idx].isna().sum())","metadata":{"id":"0hVTUf1LflPM","outputId":"9b748ee8-495c-4ade-a884-8c2ddc64ca85","execution":{"iopub.status.busy":"2022-01-12T19:24:34.159745Z","iopub.execute_input":"2022-01-12T19:24:34.159986Z","iopub.status.idle":"2022-01-12T19:24:34.541758Z","shell.execute_reply.started":"2022-01-12T19:24:34.159959Z","shell.execute_reply":"2022-01-12T19:24:34.541009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx in range(len(crypto_assets)):\n  print('Assets', idx ,'data goes from ', crypto_assets[idx].index[0].astype('datetime64[s]'), 'to ', crypto_assets[idx].index[1].astype('datetime64[s]'))","metadata":{"id":"YrVaTbdtZ6Qv","outputId":"8888890a-9089-44e2-e81e-3a4d6cf3d830","execution":{"iopub.status.busy":"2022-01-12T19:24:39.695069Z","iopub.execute_input":"2022-01-12T19:24:39.695369Z","iopub.status.idle":"2022-01-12T19:24:39.711130Z","shell.execute_reply.started":"2022-01-12T19:24:39.695315Z","shell.execute_reply":"2022-01-12T19:24:39.710374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx in range(len(crypto_assets)):\n  crypto_assets[idx] = crypto_assets[idx].reindex(range(crypto_assets[idx].index[0],crypto_assets[idx].index[-1]+60,60),method='pad')\n  crypto_assets[idx] = crypto_assets[idx].fillna(method='ffill')","metadata":{"id":"8h29S-1Hhp3Y","execution":{"iopub.status.busy":"2022-01-12T19:24:42.286097Z","iopub.execute_input":"2022-01-12T19:24:42.286371Z","iopub.status.idle":"2022-01-12T19:24:46.329238Z","shell.execute_reply.started":"2022-01-12T19:24:42.286335Z","shell.execute_reply":"2022-01-12T19:24:46.328433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(crypto_assets[1].index[1:]-crypto_assets[1].index[:-1]).value_counts().head()","metadata":{"id":"UDUygkngiBKT","outputId":"52a9cf65-c0cc-463d-cf73-0b2eb19ab2d4","execution":{"iopub.status.busy":"2022-01-12T19:24:47.549586Z","iopub.execute_input":"2022-01-12T19:24:47.549834Z","iopub.status.idle":"2022-01-12T19:24:47.584875Z","shell.execute_reply.started":"2022-01-12T19:24:47.549807Z","shell.execute_reply":"2022-01-12T19:24:47.584205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)","metadata":{"id":"DBg185h-SOwI","execution":{"iopub.status.busy":"2022-01-12T19:24:49.437475Z","iopub.execute_input":"2022-01-12T19:24:49.437929Z","iopub.status.idle":"2022-01-12T19:24:49.441745Z","shell.execute_reply.started":"2022-01-12T19:24:49.437887Z","shell.execute_reply":"2022-01-12T19:24:49.441097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building your prediction model","metadata":{"id":"EaIGlwUB9kPy"}},{"cell_type":"code","source":"# Select some input features from the trading data: \n# 5 min log return, abs(5 min log return), upper shadow, and lower shadow.\nupper_shadow = lambda asset: asset.High - np.maximum(asset.Close,asset.Open)\nlower_shadow = lambda asset: np.minimum(asset.Close,asset.Open)- asset.Low\n\nX_assets = []\ny_target = []\nfor idx in range(len(crypto_assets)):\n  X_assets.append(pd.concat([log_return(crypto_assets[idx].VWAP,periods=5), \n                             log_return(crypto_assets[idx].VWAP,periods=1).abs(), \n                             upper_shadow(crypto_assets[idx]), lower_shadow(crypto_assets[idx])], axis=1))\n  y_target.append(crypto_assets[idx].Target)","metadata":{"id":"2kQz1YDeqsch","outputId":"5bc50697-df54-41a4-a53a-26b3e0eb24c1","execution":{"iopub.status.busy":"2022-01-12T20:04:18.804428Z","iopub.execute_input":"2022-01-12T20:04:18.804821Z","iopub.status.idle":"2022-01-12T20:04:19.918554Z","shell.execute_reply.started":"2022-01-12T20:04:18.804786Z","shell.execute_reply":"2022-01-12T20:04:19.917789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\ntrain_window = [totimestamp(\"01/09/2021\"), totimestamp(\"15/09/2021\")]\nval_window = [totimestamp(\"16/09/2021\"), totimestamp(\"18/09/2021\")]\ntest_window = [totimestamp(\"19/09/2021\"), totimestamp(\"21/09/2021\")]\n\nX_train_raw = []\ny_train_raw = []\n\nX_val_raw = []\ny_val_raw = []\n\nX_test_raw = []\ny_test_raw = []\n\nfor idx in range(len(X_assets)):\n  X_train_raw.append(X_assets[idx].loc[train_window[0]:train_window[1]].fillna(0).to_numpy())  # filling NaN's with zeros\n  y_train_raw.append(y_target[idx].loc[train_window[0]:train_window[1]].fillna(0).to_numpy())\n\n  X_val_raw.append(X_assets[idx].loc[val_window[0]:val_window[1]].fillna(0).to_numpy())\n  y_val_raw.append(y_target[idx].loc[val_window[0]:val_window[1]].fillna(0).to_numpy())\n  \n  X_test_raw.append(X_assets[idx].loc[test_window[0]:test_window[1]].fillna(0).to_numpy())\n  y_test_raw.append(y_target[idx].loc[test_window[0]:test_window[1]].fillna(0).to_numpy())","metadata":{"id":"Ud_lz31L7S1x","execution":{"iopub.status.busy":"2022-01-12T19:24:59.301341Z","iopub.execute_input":"2022-01-12T19:24:59.301986Z","iopub.status.idle":"2022-01-12T19:24:59.333682Z","shell.execute_reply.started":"2022-01-12T19:24:59.301950Z","shell.execute_reply":"2022-01-12T19:24:59.333003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_raw[1]","metadata":{"id":"LmN1kbvQa_5W","outputId":"e5597c6f-9520-4359-94c3-78e74dee7a8c","execution":{"iopub.status.busy":"2022-01-12T17:21:23.027448Z","iopub.execute_input":"2022-01-12T17:21:23.028378Z","iopub.status.idle":"2022-01-12T17:21:23.035875Z","shell.execute_reply.started":"2022-01-12T17:21:23.028344Z","shell.execute_reply":"2022-01-12T17:21:23.034878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We now standardize the input data. Standardization is the process of putting different variables on the same scale. In regression analysis, it is often crucial to standardize your independent variables or you may risk obtaining misleading results.","metadata":{"id":"iWGNGmX16l28"}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n# simple preprocessing of the data \nscaler = StandardScaler()\n\nX_train_scaled = []\nX_val_scaled = []\nX_test_scaled = []\nfor idx in range(len(X_train_raw)):\n  X_train_scaled.append(scaler.fit_transform(X_train_raw[idx]))\n  X_val_scaled.append(scaler.fit_transform(X_val_raw[idx]))\n  X_test_scaled.append(scaler.fit_transform(X_test_raw[idx]))","metadata":{"id":"90LRr1HC8vSH","execution":{"iopub.status.busy":"2022-01-12T19:25:04.152568Z","iopub.execute_input":"2022-01-12T19:25:04.153367Z","iopub.status.idle":"2022-01-12T19:25:04.948767Z","shell.execute_reply.started":"2022-01-12T19:25:04.153305Z","shell.execute_reply":"2022-01-12T19:25:04.948016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def windowingData(X_list, y_list, start, end, window_size, dataset, label):\n    data, labels = [], []\n    idx = start\n    while (idx + window_size) < end:\n        # adding sequence of length window_size\n        data.append(dataset[idx: idx+window_size])\n        # adding the close price of next point\n        labels.append(dataset[idx+window_size, 0])\n        idx += 1\n    X_list.append(np.array(data)), y_list.append(np.array(labels))\n\nX_train_all = []\ny_train_all = []\n\nX_val_all = []\ny_val_all = []\n\nX_test_all = []\ny_test_all = []\n\nfor idx in range(len(X_train_scaled)):\n  windowingData(\n      X_train_all,\n      y_train_all,\n      start = 0,\n      end = len(X_train_scaled[idx]),\n      window_size = 14,\n      dataset = X_train_scaled[idx],\n      label = y_train_raw[idx]\n  )\n\n\nfor idx in range(len(X_val_scaled)):\n  windowingData(\n      X_val_all,\n      y_val_all,\n      start = 0,\n      end = len(X_val_scaled[idx]),\n      window_size = 14,\n      dataset = X_val_scaled[idx],\n      label = y_val_raw[idx]\n  )\n\n\nfor idx in range(len(X_test_scaled)):\n  windowingData(\n      X_test_all,\n      y_test_all,\n      start = 0,\n      end = len(X_test_scaled[idx]),\n      window_size = 14,\n      dataset = X_test_scaled[idx],\n      label = y_test_raw[idx]\n  )","metadata":{"id":"CJEghxM13kX3","execution":{"iopub.status.busy":"2022-01-12T19:25:07.846610Z","iopub.execute_input":"2022-01-12T19:25:07.847234Z","iopub.status.idle":"2022-01-12T19:25:08.534625Z","shell.execute_reply.started":"2022-01-12T19:25:07.847196Z","shell.execute_reply":"2022-01-12T19:25:08.533820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train_all[1].shape)\nprint(y_train_all[1].shape)\nprint(X_val_all[1].shape)\nprint(y_val_all[1].shape)\nprint(X_test_all[1].shape)\nprint(y_test_all[1].shape)","metadata":{"id":"IY8kRxr73kX3","outputId":"48acbc2a-ff43-451f-d2d4-7cb250dad7c6","execution":{"iopub.status.busy":"2022-01-12T19:25:13.112123Z","iopub.execute_input":"2022-01-12T19:25:13.112395Z","iopub.status.idle":"2022-01-12T19:25:13.120821Z","shell.execute_reply.started":"2022-01-12T19:25:13.112364Z","shell.execute_reply":"2022-01-12T19:25:13.119979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X_train_all)","metadata":{"execution":{"iopub.status.busy":"2022-01-12T19:25:15.384648Z","iopub.execute_input":"2022-01-12T19:25:15.385183Z","iopub.status.idle":"2022-01-12T19:25:15.390968Z","shell.execute_reply.started":"2022-01-12T19:25:15.385144Z","shell.execute_reply":"2022-01-12T19:25:15.390143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GRU","metadata":{"id":"HCQlF28h3kX4"}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom keras.models import Sequential\nfrom tensorflow.keras.utils import to_categorical\n# from keras.layers import Conv1D, GRU, LSTM, Dropout, Dense, CuDNNGRU\nfrom keras.layers import Activation, Dense, Dropout, LSTM, BatchNormalization, CuDNNLSTM, CuDNNGRU\n\nfrom keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.layers import Bidirectional","metadata":{"id":"enzsxJrt3kX4","execution":{"iopub.status.busy":"2022-01-12T19:25:17.652074Z","iopub.execute_input":"2022-01-12T19:25:17.652563Z","iopub.status.idle":"2022-01-12T19:25:22.327596Z","shell.execute_reply.started":"2022-01-12T19:25:17.652525Z","shell.execute_reply":"2022-01-12T19:25:22.326798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Fitting to the training set\n# learning_rate = 0.0001\n# earlyStop = keras.callbacks.EarlyStopping(min_delta=0.01, monitor='val_loss', patience=5, restore_best_weights=True)\n\n# regressors = []\n# hist = []\n# for idx in range(len(X_train_all)):\n#     regressors.append(tf.keras.models.Sequential([\n#         tf.keras.layers.Input(shape=X_train_all[0].shape[-2:]),\n#         tf.compat.v1.keras.layers.CuDNNLSTM(128, return_sequences=True),\n#         tf.keras.layers.Dropout(0.1),\n#         tf.compat.v1.keras.layers.CuDNNLSTM(60, return_sequences=False),\n#         tf.keras.layers.Dropout(0.1),\n#         tf.keras.layers.Dense(32, activation='relu'),\n#         tf.keras.layers.Dense(1)\n#     ]))\n#     regressors[idx].summary()\n#     # Compiling the RNN\n#     regressors[idx].compile(optimizer=tf.optimizers.Adam(learning_rate),loss='mae', metrics=['mse'])\n#     xva, yva = X_val_all[idx],y_val_all[idx]\n#     history = regressors[idx].fit(\n#         X_train_all[idx],\n#         y_train_all[idx],\n#         epochs=100,\n#         callbacks=[earlyStop],\n#         batch_size=256,\n#         validation_data=(xva,yva)\n#     )\n#     hist.append(history)","metadata":{"execution":{"iopub.status.busy":"2022-01-12T18:36:37.303194Z","iopub.execute_input":"2022-01-12T18:36:37.303535Z","iopub.status.idle":"2022-01-12T18:39:32.893880Z","shell.execute_reply.started":"2022-01-12T18:36:37.303502Z","shell.execute_reply":"2022-01-12T18:39:32.892731Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fitting to the training set\nlearning_rate = 0.0001\nearlyStop = keras.callbacks.EarlyStopping(min_delta=0.01, monitor='val_loss', patience=5, restore_best_weights=True)\n\nregressors = []\nhist = []\nfor idx in range(len(X_train_all)):\n    regressors.append(tf.keras.models.Sequential([\n        tf.keras.layers.Input(shape=X_train_all[0].shape[-2:]),\n        tf.compat.v1.keras.layers.CuDNNGRU(128, return_sequences=True),\n        tf.keras.layers.Dropout(0.1),\n        tf.compat.v1.keras.layers.CuDNNGRU(60, return_sequences=False),\n        tf.keras.layers.Dropout(0.1),\n        tf.keras.layers.Dense(32, activation='relu'),\n        tf.keras.layers.Dense(1)\n    ]))\n    regressors[idx].summary()\n    # Compiling the RNN\n    regressors[idx].compile(optimizer=tf.optimizers.Adam(learning_rate),loss='mae')\n    xva, yva = X_val_all[idx],y_val_all[idx]\n    history = regressors[idx].fit(\n        X_train_all[idx],\n        y_train_all[idx],\n        epochs=100,\n        callbacks=[earlyStop],\n        batch_size=256,\n        validation_data=(xva,yva)\n    )\n    hist.append(history)","metadata":{"id":"WOF1YCd73kX5","execution":{"iopub.status.busy":"2022-01-12T19:25:46.742807Z","iopub.execute_input":"2022-01-12T19:25:46.743085Z","iopub.status.idle":"2022-01-12T19:27:43.974620Z","shell.execute_reply.started":"2022-01-12T19:25:46.743055Z","shell.execute_reply":"2022-01-12T19:27:43.973905Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import mean_squared_error\nimport math\npredict_list = []\nmse = 0\nmape = 0\nfor idx in range(len(X_test_all)):\n    predict_list.append(regressors[idx].predict(X_test_all[idx]))\n    mse += mean_squared_error(predict_list[idx], y_test_all[idx])\n    mape+= np.mean(np.abs((y_test_all[idx] - predict_list[idx])/y_test_all[idx]))*100\n\nprint(\"mse: %s\" % math.sqrt(mse/len(X_test_all)))\nprint(\"mape: %s\" % (mape/len(X_test_all)))\n","metadata":{"id":"EadA63eM3kX5","execution":{"iopub.status.busy":"2022-01-12T19:28:00.377081Z","iopub.execute_input":"2022-01-12T19:28:00.377355Z","iopub.status.idle":"2022-01-12T19:28:27.108001Z","shell.execute_reply.started":"2022-01-12T19:28:00.377305Z","shell.execute_reply":"2022-01-12T19:28:27.107218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfor idx_plot in range(len(regressors)):\n    loss = hist[idx_plot].history['loss']\n    val_loss = hist[idx_plot].history['val_loss']\n    epochs_plot = range(1, len(loss) + 1)\n    plt.plot(epochs_plot, loss, 'bo', label='Training loss')\n    plt.plot(epochs_plot, val_loss, 'b', label='Validation loss')\n    plt.title('Training and validation loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-12T18:45:07.594189Z","iopub.execute_input":"2022-01-12T18:45:07.594524Z","iopub.status.idle":"2022-01-12T18:45:11.505920Z","shell.execute_reply.started":"2022-01-12T18:45:07.594475Z","shell.execute_reply":"2022-01-12T18:45:11.504766Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndef line_plot(line1, line2, idx, label1=None, label2=None, title='', lw=2):\n    fig, ax = plt.subplots(1, figsize=(13,7))\n    ax.plot(line1, label=label1, linewidth=lw)\n    ax.plot(line2, label=label2, linewidth=lw)\n    ax.set_ylabel('price diff ' + str(idx), fontsize=14)\n    ax.set_title(title, fontsize=16)\n    ax.legend(loc='best', fontsize=16)\n\nfor idx in range(len(predict_list)):\n  line_plot(y_test_all[idx], predict_list[idx], idx, 'actual', 'prediction', lw=3)","metadata":{"id":"JXnFWDB_3kX7","outputId":"b492340d-d153-45b5-b452-e0aaa2444910","execution":{"iopub.status.busy":"2022-01-12T18:45:52.556441Z","iopub.execute_input":"2022-01-12T18:45:52.556880Z","iopub.status.idle":"2022-01-12T18:45:57.354684Z","shell.execute_reply.started":"2022-01-12T18:45:52.556842Z","shell.execute_reply":"2022-01-12T18:45:57.353671Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_all[0][0]","metadata":{"id":"CsUTMZYEnlaH","outputId":"495306d5-d913-46c3-c773-3571c4ceed93","execution":{"iopub.status.busy":"2022-01-12T17:09:50.870513Z","iopub.execute_input":"2022-01-12T17:09:50.871194Z","iopub.status.idle":"2022-01-12T17:09:50.877118Z","shell.execute_reply.started":"2022-01-12T17:09:50.871158Z","shell.execute_reply":"2022-01-12T17:09:50.876356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(row):\n  asset_id = row['Asset_ID'].values[0]\n  X_train_ref[asset_id] = X_train_ref[asset_id].append(test_row.set_index(\"timestamp\"))\n  raw_feature = pd.concat([log_return(X_train_ref[asset_id].VWAP,periods=5),\n                           log_return(X_train_ref[asset_id].VWAP,periods=1).abs(),\n                           upper_shadow(X_train_ref[asset_id]),\n                           lower_shadow(X_train_ref[asset_id])], axis=1).to_numpy()\n  # scaled_feature = scaler.fit_transform([raw_feature[-1]])\n  reduce_set = np.delete(last_feature_set[asset_id], (0), axis=0)\n  last_feature_set[asset_id] = np.append(reduce_set, [raw_feature[-1]], axis=0)\n  return last_feature_set[asset_id]","metadata":{"id":"cJvnO8jynYov","execution":{"iopub.status.busy":"2022-01-12T19:30:46.024143Z","iopub.execute_input":"2022-01-12T19:30:46.024510Z","iopub.status.idle":"2022-01-12T19:30:46.031114Z","shell.execute_reply.started":"2022-01-12T19:30:46.024469Z","shell.execute_reply":"2022-01-12T19:30:46.030407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(data_folder + 'example_test.csv')\ndf_test\nfor idx in range(len(df_test)):\n    print(df_test[idx])\n#     regressors[df_test[idx].Asset_ID].predict(df_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-12T19:41:22.936212Z","iopub.execute_input":"2022-01-12T19:41:22.936691Z","iopub.status.idle":"2022-01-12T19:41:23.010896Z","shell.execute_reply.started":"2022-01-12T19:41:22.936654Z","shell.execute_reply":"2022-01-12T19:41:23.009703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        if regressors[row['Asset_ID']] is not None:\n            try:\n                model = regressors[row['Asset_ID']]\n                X_test = get_features(row)\n                y_pred = regressors[row['Asset_ID']].predict(X_test)\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n            except:\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n                traceback.print_exc()\n        else: \n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n        \n    env.predict(df_pred)","metadata":{"id":"atpaHfQtiPy5","execution":{"iopub.status.busy":"2022-01-12T19:30:48.446523Z","iopub.execute_input":"2022-01-12T19:30:48.447107Z","iopub.status.idle":"2022-01-12T19:30:48.482446Z","shell.execute_reply.started":"2022-01-12T19:30:48.447072Z","shell.execute_reply":"2022-01-12T19:30:48.481367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import math\n# from sklearn.metrics import mean_squared_error\n# # print('Test score for LSTM: BTC', np.corrcoef(predicted_btc, y_test.reshape(len(y_test),1))[0,1])\n# rmse = math.sqrt(mean_squared_error(y_test, predicted_btc.flatten()))\n# print(rmse)","metadata":{"id":"MNK44GYJ3kX8","outputId":"72773e17-8fa8-479f-9ea4-7a319ed5fc25"},"execution_count":null,"outputs":[]}]}