{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**G-Research Crypto Forecasting**\n* Load Data\n* Add Feature Cols\n* Scale\n* Build & Fit NN\n* Evaluate Results\n* Submit results via G-Research API","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-10T14:51:30.278663Z","iopub.execute_input":"2021-12-10T14:51:30.279049Z","iopub.status.idle":"2021-12-10T14:51:30.550822Z","shell.execute_reply.started":"2021-12-10T14:51:30.279007Z","shell.execute_reply":"2021-12-10T14:51:30.549673Z"}}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pandas.tseries.offsets import DateOffset\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\nimport warnings\nimport datetime as dt\nfrom statsmodels.graphics.tsaplots import plot_pacf\nimport copy\nfrom IPython.display import Image\nwarnings.simplefilter(action = 'ignore', category = FutureWarning)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:30:34.004005Z","iopub.execute_input":"2021-12-19T23:30:34.00444Z","iopub.status.idle":"2021-12-19T23:30:35.031938Z","shell.execute_reply.started":"2021-12-19T23:30:34.004333Z","shell.execute_reply":"2021-12-19T23:30:35.03076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:30:35.033999Z","iopub.execute_input":"2021-12-19T23:30:35.034322Z","iopub.status.idle":"2021-12-19T23:30:35.0444Z","shell.execute_reply.started":"2021-12-19T23:30:35.03428Z","shell.execute_reply":"2021-12-19T23:30:35.043796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load Data & Explore Structure**\n- load training data set\n- load supplemental train (this will be replaced with Sept - Dec once the comp starts)\n- merge","metadata":{}},{"cell_type":"code","source":"def loadData(file):\n    df = pd.read_csv(file)\n    df['timestamp'] = pd.to_datetime(df['timestamp'], unit = 's')\n    df.set_index(['timestamp', 'Asset_ID'], inplace = True) # make multi-index\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:30:35.04543Z","iopub.execute_input":"2021-12-19T23:30:35.045996Z","iopub.status.idle":"2021-12-19T23:30:35.053081Z","shell.execute_reply.started":"2021-12-19T23:30:35.045959Z","shell.execute_reply":"2021-12-19T23:30:35.05246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read data, format, filter time\ndata = loadData('/kaggle/input/g-research-crypto-forecasting/train.csv')\ndata = data[data.index.get_level_values('timestamp') > '2020-12-30']\nprint(data.info(show_counts = True))\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:30:35.05483Z","iopub.execute_input":"2021-12-19T23:30:35.055134Z","iopub.status.idle":"2021-12-19T23:31:37.265512Z","shell.execute_reply.started":"2021-12-19T23:30:35.055092Z","shell.execute_reply":"2021-12-19T23:31:37.264666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get supp train data\nsuppData = loadData('../input/g-research-crypto-forecasting/supplemental_train.csv')\nprint(suppData.info(show_counts = True))\nsuppData.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:31:37.267781Z","iopub.execute_input":"2021-12-19T23:31:37.267979Z","iopub.status.idle":"2021-12-19T23:31:41.332384Z","shell.execute_reply.started":"2021-12-19T23:31:37.267955Z","shell.execute_reply":"2021-12-19T23:31:41.331539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stack dataframes without overlapping index\n\noverlapDate = suppData.index.get_level_values('timestamp').min() # returns earliest time from suppTrain\ndata = data[data.index.get_level_values('timestamp') < overlapDate] # filter original DF so there's no overlap\n\nstacked = pd.concat([data, suppData], ignore_index = False, levels = 'timestamp')\n\ndouplicateRows = stacked.shape[0] - data.shape[0] - suppData.shape[0]\nprint(f\"There are {douplicateRows} missing rows\")","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:31:41.333906Z","iopub.execute_input":"2021-12-19T23:31:41.334411Z","iopub.status.idle":"2021-12-19T23:31:41.904085Z","shell.execute_reply.started":"2021-12-19T23:31:41.33438Z","shell.execute_reply":"2021-12-19T23:31:41.903207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the asset details into dictionaries\n\nfile = '../input/g-research-crypto-forecasting/asset_details.csv'\n\nassetDetails = (pd.read_csv(file)).sort_values(by = ['Asset_ID']).reset_index(drop = True)\n\nnames = {}\nweights = {}\n\nfor row in assetDetails.index:\n    assetID = assetDetails.at[row, 'Asset_ID'] \n    names[assetID] = assetDetails.at[row, 'Asset_Name']\n    weights[assetID] = assetDetails.at[row, 'Weight']\n\nprint(names)\nprint(weights)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:31:41.905723Z","iopub.execute_input":"2021-12-19T23:31:41.906254Z","iopub.status.idle":"2021-12-19T23:31:41.919286Z","shell.execute_reply.started":"2021-12-19T23:31:41.906196Z","shell.execute_reply":"2021-12-19T23:31:41.91843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Define feature cols to be added**","metadata":{}},{"cell_type":"code","source":"# create functions to add in feature cols\n    \ndef FeatureCols(df):\n    df['hlDiff'] = df['High'] - df['Low'] # high - low to measure volitility\n    df['avgSize'] = df['Volume'] // df['Count'] # average size of each trade as int\n    \n    # shadows\n    df['uShadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n    df['bShadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n    \n    # encode minute from timestamp\n    df['minute'] = df.index.get_level_values('timestamp').minute\n    df['minSin'] = np.sin(df.minute*(2.*np.pi/60))\n    df['minCos'] = np.cos(df.minute*(2.*np.pi/60))\n    \n    # encode day of month from timestamp\n    df['mDay'] = df.index.get_level_values('timestamp').day\n    df['daySin'] = np.sin(df.mDay*(2.*np.pi/31))\n    df['dayCos'] = np.cos(df.mDay*(2.*np.pi/31))\n    \n    df.drop(columns = ['minute', 'mDay'], axis = 1, inplace = True) # clear progress columns\n    \n    return (df)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:31:41.920763Z","iopub.execute_input":"2021-12-19T23:31:41.921266Z","iopub.status.idle":"2021-12-19T23:31:41.932533Z","shell.execute_reply.started":"2021-12-19T23:31:41.921223Z","shell.execute_reply":"2021-12-19T23:31:41.931771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final = FeatureCols(stacked) # Apply feature cols to the entire dataset\nfinal = final[ [ col for col in final.columns if col != 'Target' ] + ['Target'] ] # move target to end\nfinal.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:31:41.93388Z","iopub.execute_input":"2021-12-19T23:31:41.934274Z","iopub.status.idle":"2021-12-19T23:31:44.907515Z","shell.execute_reply.started":"2021-12-19T23:31:41.934244Z","shell.execute_reply":"2021-12-19T23:31:44.906547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final.info(show_counts = True) # check dataset after feature cols were added","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:31:44.908587Z","iopub.execute_input":"2021-12-19T23:31:44.908816Z","iopub.status.idle":"2021-12-19T23:31:45.244425Z","shell.execute_reply.started":"2021-12-19T23:31:44.908789Z","shell.execute_reply":"2021-12-19T23:31:45.243537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Build pipeline to scale data**\n- Select cols to be used as features\n- Save fit model to scale data for use in testing","metadata":{}},{"cell_type":"code","source":"# get libraries for preprocessing \nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, FunctionTransformer","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:31:45.245884Z","iopub.execute_input":"2021-12-19T23:31:45.246283Z","iopub.status.idle":"2021-12-19T23:31:45.366821Z","shell.execute_reply.started":"2021-12-19T23:31:45.246225Z","shell.execute_reply":"2021-12-19T23:31:45.365676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select cols\nnoScale_features = ['minSin', 'minCos', 'daySin', 'dayCos']\nhighVol_features = ['Volume']\ncont_features = ['Close', 'avgSize', 'uShadow', 'bShadow']\nfeatureCols = noScale_features + highVol_features + cont_features\n\n# set up pipeline for different data types\ndef ScaleData(inputDF, noScale_features = noScale_features, highVol_features = highVol_features, cont_features = cont_features):\n\n    noScale_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'most_frequent'))])\n\n    highVol_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'constant', fill_value = 1)),\n        ('encoder', RobustScaler(quantile_range = (20.0, 80.0)))])\n\n    cont_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'mean')),\n        ('encoder', MinMaxScaler())])\n\n    # process\n    preprosessor = ColumnTransformer(transformers = [\n        ('noScale', noScale_transformer, noScale_features),\n        ('negPos', highVol_transformer, highVol_features),\n        ('cont', cont_transformer, cont_features)])\n    \n    cols = noScale_features + highVol_features + cont_features # get cols we want to transform\n    df_to_scale = inputDF[cols] # select these cols from input df\n    fitScaler = preprosessor.fit(df_to_scale) # fit scaler\n    scaled = fitScaler.transform(df_to_scale) # scale\n    \n    return [(pd.DataFrame(scaled, columns = cols).set_index(inputDF.index)), fitScaler] # df with scaled data & fit model to be used later","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:31:45.368314Z","iopub.execute_input":"2021-12-19T23:31:45.368737Z","iopub.status.idle":"2021-12-19T23:31:45.38035Z","shell.execute_reply.started":"2021-12-19T23:31:45.368689Z","shell.execute_reply":"2021-12-19T23:31:45.379387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Split assets into individual tables**\n- Missing timestamps are filled using 'pad'\n- Data is scaled using the pipeline built above\n- Pairplot and Autocorrelations are plotted for the scaled data","metadata":{}},{"cell_type":"code","source":"# Split each table into a df, fill missing values, create feature cols, scale\n\nassets = []\nassetScalers = {}\n\nfor asset in names.keys():\n    df = final.xs(asset, level = 'Asset_ID')\n    \n    timeStamps = df.index\n    \n    # set index so there's no missing times\n    minDate = timeStamps.min()\n    maxDate = timeStamps.max()\n    df = df.reindex(index = list(pd.date_range(minDate, maxDate, freq = 'min')), method = 'pad')\n\n    # scale data\n    result = ScaleData(df)\n    scaledDF, fitScaler = result[0], result[1] \n    \n    # fill na's for target\n    scaledDF['Target'] = df['Target'].fillna(0)\n    \n    assets.append(scaledDF) # save transformed df\n    assetScalers[asset] = fitScaler # save scaler \n    \n    # visualize data\n    print(names[asset])\n    \n    plt.figure(figsize = (10, 10))\n    sns.pairplot(scaledDF.sample(10000, random_state = 10, ignore_index = True))\n    plt.show()\n    \n    plot_pacf(scaledDF['Target'].to_list(), lags = 50)\n    plt.show()\n    \ndel data # we no longer need the table. Free up memory.","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-12-19T23:31:45.381559Z","iopub.execute_input":"2021-12-19T23:31:45.3818Z","iopub.status.idle":"2021-12-19T23:39:26.004555Z","shell.execute_reply.started":"2021-12-19T23:31:45.381772Z","shell.execute_reply":"2021-12-19T23:39:26.003643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prepare data for input into model**\n- Class created to store attributes for each asset\n- Create the number of lags to feed into the model\n- Train/test split is set\n- Check shape of all assets inputs","metadata":{}},{"cell_type":"code","source":"# create class to store data\nclass Asset():\n    def __init__(self, xTrain, xTest, yTrain, yTest, builtModel = None, fitModel = None):\n        self.xTrain = xTrain\n        self.xTest = xTest\n        self.yTrain = yTrain\n        self.yTest = yTest\n        self.builtModel = builtModel\n        self.fitModel = fitModel","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:39:26.009083Z","iopub.execute_input":"2021-12-19T23:39:26.009928Z","iopub.status.idle":"2021-12-19T23:39:26.01618Z","shell.execute_reply.started":"2021-12-19T23:39:26.009878Z","shell.execute_reply":"2021-12-19T23:39:26.015188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numLags = 15 # set the number of lags we want to feed to each array (i.e. were going to feed in the current minute + 15 previous minutes)\nassetNames = names.keys()\ntrainPct = 0.8\n\npreppedData = [] # store a instance for each asset\nassetShapes = [] # store the shape of each assets df\n\n# go through each asset and save details\nfor i, asset in enumerate(assets):\n    y = np.array(asset['Target'].values.tolist())[numLags:]\n    asset.drop('Target', axis = 1, inplace = True)\n    \n    x = [] #store the data with lags\n    \n    for time in asset.index[numLags:]:\n        refTime = time - dt.timedelta(minutes = numLags) # go back number of lags required\n        refRows = asset.loc[refTime : time] # get df of rows\n        x.append(refRows.to_numpy())\n    x = np.array(x) \n    \n    # append a class instance with the training and testing data\n    trainIndex = int(len(x) * trainPct)\n    preppedData.append(Asset(x[:trainIndex], x[trainIndex:], y[:trainIndex], y[trainIndex:]))\n    \n    # append shapes\n    shapes = {}\n    shapes['xTrain_shape'] = np.shape(preppedData[i].xTrain)\n    shapes['xTest_shape'] = np.shape(preppedData[i].xTest)\n    shapes['yTrain_shape'] = np.shape(preppedData[i].yTrain)\n    shapes['yTest_shape'] = np.shape(preppedData[i].yTest)\n    assetShapes.append(shapes)\n    \nshapes = pd.DataFrame(assetShapes, index = assetNames)\nshapes # i.e. asset 0 has a xTrain shape of 305268 timestamps, each with 16 times, each with 9 columns","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:39:26.017404Z","iopub.execute_input":"2021-12-19T23:39:26.017715Z","iopub.status.idle":"2021-12-19T23:49:38.163836Z","shell.execute_reply.started":"2021-12-19T23:39:26.017673Z","shell.execute_reply":"2021-12-19T23:49:38.162953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Build Model and Fit**\n- Use Keras functional API (add layers, set activation functions, early stopping, etc..)\n- Save the model and fit model for each asset to the class instance\n","metadata":{}},{"cell_type":"code","source":"# get libraries for the model\nimport tensorflow as tf\nfrom tensorflow import keras \nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2021-12-19T23:49:38.165264Z","iopub.execute_input":"2021-12-19T23:49:38.16567Z","iopub.status.idle":"2021-12-19T23:49:42.717528Z","shell.execute_reply.started":"2021-12-19T23:49:38.16563Z","shell.execute_reply":"2021-12-19T23:49:42.71677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"K.clear_session() ","metadata":{"execution":{"iopub.status.busy":"2021-12-20T00:39:19.841015Z","iopub.execute_input":"2021-12-20T00:39:19.841384Z","iopub.status.idle":"2021-12-20T00:39:20.367192Z","shell.execute_reply.started":"2021-12-20T00:39:19.841346Z","shell.execute_reply":"2021-12-20T00:39:20.366562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# timeSteps, features = shapes.at[0, 'xTrain_shape'][1], shapes.at[0, 'xTrain_shape'][2] # set shape\n\n# # variables to adjust\n# act = 'tanh'\n# init = tf.keras.initializers.he_uniform()\n\n# # define layers\n# X = keras.Input(shape = (timeSteps, features), name = 'X_Inputs')\n# L1 = keras.layers.LSTM(64, activation = act, kernel_initializer = init, return_sequences = True, \n#                        name = 'Layer_1')(inputs = X)\n# L2 = keras.layers.BatchNormalization(name = 'Layer_2')(inputs = L1)\n# L3 = keras.layers.LSTM(64, activation = act, kernel_initializer = init, return_sequences = True, \n#                        name = 'Layer_3')(inputs = L2)\n# L4 = keras.layers.BatchNormalization(name = 'Layer_4')(inputs = L3)\n# L5 = keras.layers.LSTM(32, activation = act, kernel_initializer = init, return_sequences = False, \n#                        name = 'Layer_5')(inputs = L4)\n# L6 = keras.layers.BatchNormalization(name = 'Layer_6')(inputs = L5)\n# y_proba = keras.layers.Dense(1, name = 'Y_Outputs')(inputs = L6) ","metadata":{"scrolled":true,"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-20T00:42:39.983298Z","iopub.execute_input":"2021-12-20T00:42:39.983647Z","iopub.status.idle":"2021-12-20T00:42:40.712372Z","shell.execute_reply.started":"2021-12-20T00:42:39.983596Z","shell.execute_reply":"2021-12-20T00:42:40.711405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# speed testing\ntimeSteps, features = shapes.at[0, 'xTrain_shape'][1], shapes.at[0, 'xTrain_shape'][2] # set shape\n\n# variables to adjust\nact = 'tanh'\ninit = tf.keras.initializers.he_uniform()\n\n# define layers\nX = keras.Input(shape = (timeSteps, features), name = 'X_Inputs')\nL1 = keras.layers.LSTM(64, activation = act, kernel_initializer = init, return_sequences = False, \n                       name = 'Layer_1')(inputs = X)\ny_proba = keras.layers.Dense(1, name = 'Y_Outputs')(inputs = L1) ","metadata":{"execution":{"iopub.status.busy":"2021-12-20T00:43:27.695839Z","iopub.execute_input":"2021-12-20T00:43:27.696175Z","iopub.status.idle":"2021-12-20T00:43:27.933376Z","shell.execute_reply.started":"2021-12-20T00:43:27.696123Z","shell.execute_reply":"2021-12-20T00:43:27.932179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# variables to adjust\nesPatience = 4\noptLr = 0.02\nnumEpochs = 1\nbatchSize = 1000\n\n# fit models\nes = EarlyStopping(monitor = 'loss', mode = 'min', patience = esPatience, restore_best_weights = True, verbose = 1)\n\nfor i, asset in enumerate(preppedData):\n    # assemble\n    model = tf.keras.Model(inputs = [X], outputs = [y_proba], name = 'Individual_Asset_Model')\n    \n    # define optimizer and compile\n    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = optLr), \n                  loss = 'mean_squared_error',metrics = ['accuracy'])\n    if i == 0:\n        print(model.summary())\n        \n    print(\"Fitting to\", names[i])\n    asset.builtModel = model # save\n    asset.fitModel = asset.builtModel.fit(asset.xTrain, asset.yTrain, validation_split = 0.1, epochs = numEpochs, \n                                           batch_size = batchSize, verbose = 1, callbacks = [es]) # fit","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-12-20T00:43:30.089676Z","iopub.execute_input":"2021-12-20T00:43:30.08999Z","iopub.status.idle":"2021-12-20T00:43:42.31517Z","shell.execute_reply.started":"2021-12-20T00:43:30.089956Z","shell.execute_reply":"2021-12-20T00:43:42.313448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Explore Results**\n- Plot Model Loss Values\n- Create y_pred for each asset and evaluate","metadata":{}},{"cell_type":"code","source":"# plot loss values\n\n#list of all plot locations\nrows, cols = 3, 5\nplotList = [] \nfor row in list(range(rows)):\n    for col in list(range(cols)):\n        plotList.append([row, col])\n\n# loss values by epoch\nfig, axis = plt.subplots(rows, cols, figsize = (10, 10))\n\nfor i, asset in enumerate(preppedData):\n    modelHist = asset.fitModel\n    pltRow, pltCol = plotList[i][0], plotList[i][1]\n    axis[pltRow, pltCol].plot(modelHist.history['loss'], label = 'tL')\n    axis[pltRow, pltCol].plot(modelHist.history['val_loss'], label = 'vL')\n    axis[pltRow, pltCol].legend(loc = 1) \n    axis[pltRow, pltCol].set_title(names[i], y = 1.05)\n    axis[pltRow, pltCol].ticklabel_format(style = 'sci')\n\nfig.suptitle('Loss Values by Epoch')\nfig.tight_layout() ","metadata":{"execution":{"iopub.status.busy":"2021-12-20T00:05:20.229042Z","iopub.execute_input":"2021-12-20T00:05:20.22964Z","iopub.status.idle":"2021-12-20T00:05:22.563668Z","shell.execute_reply.started":"2021-12-20T00:05:20.229587Z","shell.execute_reply":"2021-12-20T00:05:22.562989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot metrics for each models performance\n\n#list of all plot locations\nrows, cols = len(names), 3\nplotList = [] \nfor row in list(range(rows)):\n    for col in list(range(cols)):\n        plotList.append([row, col])\n        \nfig, axis = plt.subplots(rows, cols, figsize = (10, 40))\n\ncorrelations = {}\n\nfor i, asset in enumerate(preppedData):\n    assetName = names[i]\n    ypred = asset.builtModel.predict(asset.xTest).flatten()\n    inputs = asset.yTest\n    \n    result = pd.DataFrame(list(zip(ypred, inputs)), columns = ['ypred', 'inputs']) # df of results\n    result['diff'] = result['ypred'] - result['inputs']\n    \n    correlations[assetName] = result['ypred'].corr(result['inputs']) # add correlations \n    \n    axis[i, 0].scatter(result.ypred, result.inputs)\n    axis[i, 0].set_title(\"yPred vs yTest: \" + assetName)\n    axis[i, 1].plot(result['diff'])\n    axis[i, 1].set_title(\"yPred - yTest: \" + assetName)\n    sample = result.sample(100, random_state = 99).sort_index() # just plotting a portion of the dataset\n    axis[i, 2].plot(sample['ypred'], alpha = 0.5, label = 'yP') # yP = y_pred\n    axis[i, 2].plot(sample['inputs'], alpha = 0.5, label = 'yT') # yT = y_test\n    axis[i, 2].set_title(\"yPred vs yTest: \" + assetName)\n    axis[i, 2].legend(loc = 1)\n\nfig.suptitle('Metrics by Asset')\nfig.tight_layout() \n\npd.DataFrame.from_dict(correlations, orient = 'index', columns = ['Correlation'])","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-12-20T00:05:22.564783Z","iopub.execute_input":"2021-12-20T00:05:22.565329Z","iopub.status.idle":"2021-12-20T00:05:37.369389Z","shell.execute_reply.started":"2021-12-20T00:05:22.565293Z","shell.execute_reply":"2021-12-20T00:05:37.36808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Submit Predictions**\n- Uses G-Research's API developed for the competition","metadata":{}},{"cell_type":"code","source":"lagData = {}\n\nfor i, asset in enumerate(assets): # get historical data from last dataset\n    lagData[i] = asset.iloc[-numLags:]","metadata":{"execution":{"iopub.status.busy":"2021-12-20T00:27:34.831063Z","iopub.execute_input":"2021-12-20T00:27:34.831913Z","iopub.status.idle":"2021-12-20T00:27:34.837825Z","shell.execute_reply.started":"2021-12-20T00:27:34.831854Z","shell.execute_reply":"2021-12-20T00:27:34.836823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2021-12-20T00:05:40.805433Z","iopub.execute_input":"2021-12-20T00:05:40.805868Z","iopub.status.idle":"2021-12-20T00:05:40.828569Z","shell.execute_reply.started":"2021-12-20T00:05:40.805828Z","shell.execute_reply":"2021-12-20T00:05:40.827925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()","metadata":{"execution":{"iopub.status.busy":"2021-12-20T00:05:42.563606Z","iopub.execute_input":"2021-12-20T00:05:42.564128Z","iopub.status.idle":"2021-12-20T00:05:42.567707Z","shell.execute_reply.started":"2021-12-20T00:05:42.56409Z","shell.execute_reply":"2021-12-20T00:05:42.567129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2021-12-20T00:05:44.147343Z","iopub.execute_input":"2021-12-20T00:05:44.147645Z","iopub.status.idle":"2021-12-20T00:05:44.151515Z","shell.execute_reply.started":"2021-12-20T00:05:44.147598Z","shell.execute_reply":"2021-12-20T00:05:44.150689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create function to predict results for each row\ndef Predict(x):\n    \n    time = x.name[1] # current time\n    refTime = time - dt.timedelta(minutes = numLags) # get the timestamp of the earliest lag\n    \n    asset = x.name[0] # get asset name\n    \n    df = pd.DataFrame(x[featureCols].values, index = featureCols).T # select these cols from input df\n    df = assetScalers[asset].transform(df) # scale with model used for the individual asset\n    preppedRow = pd.DataFrame(df, columns = featureCols) # make DF with scaled data, set columns\n    preppedRow['timestamp'] = time\n    preppedRow.set_index('timestamp', drop = True, inplace = True)\n    \n    workingDF = lagData[asset] \n        \n    # add row to lagData\n    workingDF = pd.concat([workingDF, preppedRow]).sort_index() # add new row\n        \n    # prep data for predictions & predict\n    lagData[asset] = workingDF.reindex(index = list(pd.date_range(refTime, time, freq = 'min')), method = 'nearest') # reindex for input\n        \n    x = lagData[asset].to_numpy()\n        \n    if np.count_nonzero(x==0) > (0.25*(len(x.flatten()))): # i.e. if more than half of the elemets in the prediction are 0\n        y_pred = 0\n        \n    else:\n        x = np.expand_dims(x, axis = 0)\n        y_pred = preppedData[asset].builtModel.predict(x)[0][0]\n\n    return np.float16(y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-20T00:27:47.6918Z","iopub.execute_input":"2021-12-20T00:27:47.692538Z","iopub.status.idle":"2021-12-20T00:27:47.704597Z","shell.execute_reply.started":"2021-12-20T00:27:47.692502Z","shell.execute_reply":"2021-12-20T00:27:47.703614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    \n    # clean input df, set index\n    test_df['timestamp'] = pd.to_datetime(test_df['timestamp'], unit = 's')\n    test_df.set_index(['Asset_ID', 'timestamp'], inplace = True)\n\n    indexValues = test_df.index.get_level_values('timestamp')\n    \n    test_df.fillna(0) # fill na's\n    test_df = FeatureCols(test_df) # add in feature cols\n    \n    test_df['Target'] = test_df.apply(lambda x: Predict(x), axis = 1) # predict results\n    \n    prediction_df = test_df[['row_id', 'Target']].reset_index(drop = True) # get row and target\n    \n    prediction_df['Target'].clip(-0.4, 0.4, inplace = True) # remove outliers\n\n    env.predict(prediction_df) # submit","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}