{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is prep for the G-Research Crypto Forecasting Kaggle Competition. The data has explored, prepped, and run through a basic model. The model must be tuned first to speed up predictions then to increase accuracy.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pandas.tseries.offsets import DateOffset\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\nimport warnings\nimport datetime as dt\nimport os\nfrom statsmodels.graphics.tsaplots import plot_pacf\nwarnings.simplefilter(action = 'ignore', category = FutureWarning)\npd.options.mode.chained_assignment = None","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:05.207765Z","iopub.execute_input":"2022-01-29T14:29:05.208189Z","iopub.status.idle":"2022-01-29T14:29:06.526801Z","shell.execute_reply.started":"2022-01-29T14:29:05.208038Z","shell.execute_reply":"2022-01-29T14:29:06.525996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:06.528550Z","iopub.execute_input":"2022-01-29T14:29:06.528824Z","iopub.status.idle":"2022-01-29T14:29:06.546859Z","shell.execute_reply.started":"2022-01-29T14:29:06.528791Z","shell.execute_reply":"2022-01-29T14:29:06.546124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the asset details into dictionaries\n\nfile = '../input/g-research-crypto-forecasting/asset_details.csv'\n\nassetDetails = (pd.read_csv(file)).sort_values(by = ['Asset_ID']).reset_index(drop = True)\n\nnames = {}\nweights = {}\n\nfor row in assetDetails.index:\n    assetID = assetDetails.at[row, 'Asset_ID'] \n    names[assetID] = assetDetails.at[row, 'Asset_Name']\n    weights[assetID] = assetDetails.at[row, 'Weight']\n\nprint(names)\nprint(weights)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:06.548920Z","iopub.execute_input":"2022-01-29T14:29:06.550013Z","iopub.status.idle":"2022-01-29T14:29:06.580315Z","shell.execute_reply.started":"2022-01-29T14:29:06.549973Z","shell.execute_reply":"2022-01-29T14:29:06.579208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Load Data**\n* Get training data\n* Get supplemental data\n* Combine","metadata":{}},{"cell_type":"code","source":"def loadData(file):\n    df = pd.read_csv(file)\n    df['timestamp'] = pd.to_datetime(df['timestamp'], unit = 's')\n    df.set_index(['timestamp', 'Asset_ID'], inplace = True) # make multi-index\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:06.582594Z","iopub.execute_input":"2022-01-29T14:29:06.582857Z","iopub.status.idle":"2022-01-29T14:29:06.588081Z","shell.execute_reply.started":"2022-01-29T14:29:06.582826Z","shell.execute_reply":"2022-01-29T14:29:06.587311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read data, format, filter time\ndata = loadData('/kaggle/input/g-research-crypto-forecasting/train.csv')\ndata = data[data.index.get_level_values('timestamp') > '2020-12-30'] \nprint(data.info(show_counts = True))\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:06.589376Z","iopub.execute_input":"2022-01-29T14:29:06.589807Z","iopub.status.idle":"2022-01-29T14:29:27.519212Z","shell.execute_reply.started":"2022-01-29T14:29:06.589744Z","shell.execute_reply":"2022-01-29T14:29:27.516716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get supp train data\nsuppData = loadData('../input/g-research-crypto-forecasting/supplemental_train.csv')\nprint(suppData.info(show_counts = True))\nsuppData.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.520191Z","iopub.status.idle":"2022-01-29T14:29:27.520588Z","shell.execute_reply.started":"2022-01-29T14:29:27.520376Z","shell.execute_reply":"2022-01-29T14:29:27.520397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stack dataframes without overlapping index\n\noverlapDate = suppData.index.get_level_values('timestamp').min() # returns earliest time from suppTrain\ndata = data[data.index.get_level_values('timestamp') < overlapDate] # filter original DF so there's no overlap\n\nstacked = pd.concat([data, suppData], ignore_index = False, levels = 'timestamp')\n\ndouplicateRows = stacked.shape[0] - data.shape[0] - suppData.shape[0]\nprint(f\"There are {douplicateRows} missing rows\")","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.522074Z","iopub.status.idle":"2022-01-29T14:29:27.522439Z","shell.execute_reply.started":"2022-01-29T14:29:27.522250Z","shell.execute_reply":"2022-01-29T14:29:27.522271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prep Data and Explore**\n* Add in feature columns to create a 'final' dataframe\n* Review data types, correlations, and the target's distribution\n* Build pipeline to impute missing values & scale","metadata":{}},{"cell_type":"code","source":"# create functions to add in feature cols\n    \ndef FeatureCols(df):\n    df['hlDiff'] = df['High'] - df['Low'] # high - low to measure volitility\n    \n    # shadows\n    df['uShadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n    df['bShadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n    \n    # encode minute from timestamp\n    df['minute'] = df.index.get_level_values('timestamp').minute\n    df['minSin'] = np.sin(df.minute*(2.*np.pi/60))\n    df['minCos'] = np.cos(df.minute*(2.*np.pi/60))\n    \n    df.drop(columns = ['minute'], axis = 1, inplace = True) # clear progress columns\n    \n    return (df)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.523564Z","iopub.status.idle":"2022-01-29T14:29:27.524096Z","shell.execute_reply.started":"2022-01-29T14:29:27.523886Z","shell.execute_reply":"2022-01-29T14:29:27.523908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final = FeatureCols(stacked) # Apply feature cols to the entire dataset\nfinal = final[ [ col for col in final.columns if col != 'Target' ] + ['Target'] ] # move target to end\nfinal.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.525225Z","iopub.status.idle":"2022-01-29T14:29:27.525563Z","shell.execute_reply.started":"2022-01-29T14:29:27.525374Z","shell.execute_reply":"2022-01-29T14:29:27.525392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make correlation matrix\ndef corrMatrix(columns):\n    corr = columns.corr()\n    cmap = sns.diverging_palette(230, 20, as_cmap = True) # colour palette to match correlation matrix\n    mask = np.triu(np.ones_like(corr, dtype = bool)) # hide top half\n    sns.heatmap(corr, mask = mask, cmap = cmap, square = True, linewidths = 0.5, \n                        center = 0, cbar_kws = {\"shrink\": .5})\n    plt.title('Correlation Matrix')\n    plt.show()  ","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.526768Z","iopub.status.idle":"2022-01-29T14:29:27.527303Z","shell.execute_reply.started":"2022-01-29T14:29:27.527111Z","shell.execute_reply":"2022-01-29T14:29:27.527131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final.info(show_counts = True) # check dataset after feature cols were added\n  \ncorrMatrix(final) # correlation matrix of all assets\n\n#plot distribution of targets \nfor asset in names.keys():\n    subset = final.xs(asset, level = 'Asset_ID') # individual asset\n    subset = subset.loc[(subset['Target'] > subset['Target'].quantile(0.025)) & \n                        (subset['Target'] < subset['Target'].quantile(0.975))] # remove outliers\n    sns.distplot(subset['Target'], hist = False, kde = True, kde_kws = {'linewidth': 3})\n\nplt.title('Density Plot with All Assets')\nplt.xlabel('Target')\nplt.ylabel('Density')\nplt.legend(labels = names.values())\nplt.show()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-29T14:29:27.528429Z","iopub.status.idle":"2022-01-29T14:29:27.528807Z","shell.execute_reply.started":"2022-01-29T14:29:27.528593Z","shell.execute_reply":"2022-01-29T14:29:27.528614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get libraries for preprocessing \nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, FunctionTransformer","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.529954Z","iopub.status.idle":"2022-01-29T14:29:27.530296Z","shell.execute_reply.started":"2022-01-29T14:29:27.530115Z","shell.execute_reply":"2022-01-29T14:29:27.530135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add in rsi\n\nwindowLen = 30\n\n# https://www.alpharithms.com/relative-strength-index-rsi-in-python-470209/\ndef addRSI(closePrices):\n\n    # initalize variables\n    gains = [] # Initialize containers for avg. gains and losses\n    losses = []\n\n    window = [] # Create a container for current lookback prices\n\n    prev_avg_gain = None # Keeps track of previous average values\n    prev_avg_loss = None\n\n    output = [] # Create a container for our final output\n    \n    # caclulate price differences\n    for i, price in enumerate(closePrices): # keep track of the price for the first period but don't calculate a difference value.\n        \n        if i == 0:\n            window.append(price)\n            output.append(0)\n            continue\n    \n        difference = round(closePrices[i] - closePrices[i - 1], 2) # calculate the difference between price and previous price as a rounded value\n    \n        # Calculate Gains & Losses\n        if difference > 0: # Record positive differences as gains\n            gain = difference\n            loss = 0\n\n        elif difference < 0: # Record negative differences as losses\n            gain = 0\n            loss = abs(difference)\n\n        else: # Record no movements as neutral\n            gain = 0\n            loss = 0\n    \n        gains.append(gain) # Save gains/losses\n        losses.append(loss)\n\n        if i < windowLen: # Continue to iterate until enough gains/losses data is available to calculate the initial RS value\n            window.append(price)\n            output.append(0)\n            continue\n        \n        # Calculate Average Gains & Losses\n    \n        if i == windowLen: # Calculate SMA for first gain\n            avg_gain = sum(gains) / len(gains)\n            avg_loss = sum(losses) / len(losses)\n    \n        else: # Use WSM after initial window-length period\n            avg_gain = (prev_avg_gain * (windowLen - 1) + gain) / windowLen\n            avg_loss = (prev_avg_loss * (windowLen - 1) + loss) / windowLen\n        \n        prev_avg_gain = avg_gain # Keep in memory\n        prev_avg_loss = avg_loss\n        \n        if avg_loss == 0:\n            rsi = 0\n        \n        else:\n            # Calculate the RS Value\n            rs = np.float16(avg_gain) /  np.float16(avg_loss)\n    \n            # Calculate the RSI Value\n            rsi = np.float16(100 - (100 / (1 + rs)))\n    \n        # Remove oldest values\n        window.append(price)\n        window.pop(0)\n        gains.pop(0)\n        losses.pop(0)\n\n        output.append(rsi)\n    \n    return output","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.532090Z","iopub.status.idle":"2022-01-29T14:29:27.532801Z","shell.execute_reply.started":"2022-01-29T14:29:27.532592Z","shell.execute_reply":"2022-01-29T14:29:27.532613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select cols\nnoScale_features = ['minSin', 'minCos']\nhighVol_features = ['Volume']\ncont_features = ['Close', 'uShadow', 'bShadow', 'rsi']\nfeatureCols = noScale_features + highVol_features + cont_features\n\n# set up pipeline for different data types\ndef ScaleData(inputDF, noScale_features = noScale_features, highVol_features = highVol_features, cont_features = cont_features):\n\n    noScale_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'most_frequent'))])\n\n    highVol_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'constant', fill_value = 1)),\n        ('encoder', RobustScaler(quantile_range = (20.0, 80.0)))])\n\n    cont_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'mean')),\n        ('encoder', MinMaxScaler(feature_range = (0, 1)))])\n\n    # process\n    preprosessor = ColumnTransformer(transformers = [\n        ('noScale', noScale_transformer, noScale_features),\n        ('negPos', highVol_transformer, highVol_features),\n        ('cont', cont_transformer, cont_features)])\n    \n    cols = noScale_features + highVol_features + cont_features # get cols we want to transform\n    df_to_scale = inputDF[cols] # select these cols from input df\n    fitScaler = preprosessor.fit(df_to_scale) # fit scaler\n    scaled = fitScaler.transform(df_to_scale) # scale\n    \n    return [(pd.DataFrame(scaled, columns = cols).set_index(inputDF.index)), fitScaler] # df with scaled data & fit model to be used later","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.533765Z","iopub.status.idle":"2022-01-29T14:29:27.534451Z","shell.execute_reply.started":"2022-01-29T14:29:27.534256Z","shell.execute_reply":"2022-01-29T14:29:27.534276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Prepare Data for Training**\n* Split each asset into its own table and review individual asset features\n* Split into training and testing data","metadata":{}},{"cell_type":"code","source":"# Split each table into a df, fill missing values, create feature cols, scale\n\nassets = []\nassetScalers = {} # save the scaler\nclosingPrices = {}\n\nfor asset in names.keys():\n    df = final.xs(asset, level = 'Asset_ID')\n    \n    timeStamps = df.index\n    \n    # set index so there's no missing times\n    minDate = timeStamps.min()\n    maxDate = timeStamps.max()\n    df = df.reindex(index = list(pd.date_range(minDate, maxDate, freq = 'min')), method = 'pad')\n    \n    # add in rsi\n    df['rsi'] = addRSI(df['Close'].to_list())\n    df = df.iloc[windowLen:, :] # remove first rows with nan\n    closingPrices[asset] = df['Close'].to_list()[-windowLen:] # save closing prices for predictions\n    \n    # remove last rows with missing target\n    df = df.iloc[:-250, :]\n    \n    # scale data\n    result = ScaleData(df)\n    scaledDF, fitScaler = result[0], result[1] \n    \n    # fill na's for target\n    scaledDF['Target'] = df['Target'].fillna(0)\n    \n    assets.append(scaledDF) # save transformed df\n    assetScalers[asset] = fitScaler # save scaler \n    \n    # visualize data\n    print(names[asset])\n    sampleData = scaledDF.sample(10000, random_state = 10, ignore_index = True)\n    \n    plt.figure(figsize = (7, 7))\n    sns.pairplot(sampleData)\n    plt.show()\n    \n    plot_pacf(scaledDF['Target'].to_list(), lags = 50)\n    plt.show()\n    \ndel data # we no longer need the table. Free up memory.","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-29T14:29:27.535451Z","iopub.status.idle":"2022-01-29T14:29:27.536197Z","shell.execute_reply.started":"2022-01-29T14:29:27.535979Z","shell.execute_reply":"2022-01-29T14:29:27.536002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create class to store data\nclass Asset():\n    def __init__(self, xTrain, xTest, yTrain, yTest, builtModel = None):\n        self.xTrain = xTrain\n        self.xTest = xTest\n        self.yTrain = yTrain\n        self.yTest = yTest\n        self.builtModel = builtModel","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.537108Z","iopub.status.idle":"2022-01-29T14:29:27.537845Z","shell.execute_reply.started":"2022-01-29T14:29:27.537590Z","shell.execute_reply":"2022-01-29T14:29:27.537611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assetNames = names.keys()\ntrainPct = 0.8\n\npreppedData = [] # store a instance for each asset\nassetShapes = [] # store the shape of each assets df\n\nfor i, asset in enumerate(assets):\n    \n    minOutlier = asset['Target'].quantile(0.025)\n    maxOutlier = asset['Target'].quantile(0.985)\n    \n    asset = asset.loc[(asset['Target'] > minOutlier) & (asset['Target'] < maxOutlier)] # remove outliers\n    \n    y = np.array(asset['Target'].values)\n    asset.drop('Target', axis = 1, inplace = True)\n    \n    x = np.array(asset)\n    \n    # append a class instance with the training and testing data\n    trainIndex = int(len(x) * trainPct)\n    preppedData.append(Asset(x[:trainIndex], x[trainIndex:], y[:trainIndex], y[trainIndex:]))\n    \n    # append shapes\n    shapes = {}\n    shapes['xTrain_shape'] = np.shape(preppedData[i].xTrain)\n    shapes['xTest_shape'] = np.shape(preppedData[i].xTest)\n    shapes['yTrain_shape'] = np.shape(preppedData[i].yTrain)\n    shapes['yTest_shape'] = np.shape(preppedData[i].yTest)\n    assetShapes.append(shapes)\n    \nshapes = pd.DataFrame(assetShapes, index = assetNames)\nshapes","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.538903Z","iopub.status.idle":"2022-01-29T14:29:27.539622Z","shell.execute_reply.started":"2022-01-29T14:29:27.539411Z","shell.execute_reply":"2022-01-29T14:29:27.539433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Run Model - XGBoost Regression**\n* Define Paramaters\n* Fit\n* Save fit model and evaluate","metadata":{}},{"cell_type":"code","source":"# get libraries\nimport xgboost as xgb\nfrom sklearn.metrics import mean_squared_error as MSE","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.540818Z","iopub.status.idle":"2022-01-29T14:29:27.541156Z","shell.execute_reply.started":"2022-01-29T14:29:27.540971Z","shell.execute_reply":"2022-01-29T14:29:27.540989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set params\n\nparam = {'booster' : 'gbtree',\n         'objective' : 'reg:squaredlogerror',\n         'eta' : 0.05,\n         'max_depth' : 12,\n         'eval_metric' : 'rmsle',\n         'min_child_weight' : 1,\n         'tree_method' : 'hist'} \n\nepochs = 200\n\n# save metrics\nscores = {}\npredictions = {}\n\n# fit\nfor i, asset in enumerate(preppedData):\n    \n    print(names[i])\n    dtrain = xgb.DMatrix(asset.xTrain, feature_names = featureCols, label = asset.yTrain)\n    dtest = xgb.DMatrix(asset.xTest, feature_names = featureCols, label = asset.yTest) \n    \n    asset.builtModel = xgb.train(param, dtrain, epochs, evals = [(dtest, \"Test\")], early_stopping_rounds = 10)\n    prediction = asset.builtModel.predict(dtest) # predict\n    \n    predictions[i] = prediction # save predictions\n    accScore = np.sqrt(MSE(asset.yTest, prediction)) # get RMSE\n    scores[names[i]] = \"{:.2%}\".format(accScore) # append\n    \npd.DataFrame.from_dict(scores, orient = 'index', columns = ['RMSE'])","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-29T14:29:27.542196Z","iopub.status.idle":"2022-01-29T14:29:27.542930Z","shell.execute_reply.started":"2022-01-29T14:29:27.542718Z","shell.execute_reply":"2022-01-29T14:29:27.542742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot metrics for each models performance\n\n#list of all plot locations\nrows, cols = len(names), 3\nplotList = [] \nfor row in list(range(rows)):\n    for col in list(range(cols)):\n        plotList.append([row, col])\n        \nfig, axis = plt.subplots(rows, cols, figsize = (10, 40))\n\ncorrelations = {}\n\nfor i, asset in enumerate(preppedData):\n    assetName = names[i]\n    ypred = predictions[i]\n    inputs = asset.yTest\n    \n    result = pd.DataFrame(list(zip(ypred, inputs)), columns = ['ypred', 'inputs']).dropna() # df of results\n    result['diff'] = result['ypred'] - result['inputs']\n    \n    correlation = result['ypred'].corr(result['inputs'])\n    correlations[assetName] =  \"{:.2%}\".format(correlation)# add correlations \n    \n    axis[i, 0].scatter(result.ypred, result.inputs)\n    axis[i, 0].set_title(\"yPred vs yTest: \" + assetName)\n    axis[i, 1].plot(result['diff'])\n    axis[i, 1].set_title(\"yPred - yTest: \" + assetName)\n    sample = result.sample(100, random_state = 99).sort_index() # just plotting a portion of the dataset\n    axis[i, 2].plot(sample['ypred'], alpha = 0.5, label = 'yP') # yP = y_pred\n    axis[i, 2].plot(sample['inputs'], alpha = 0.5, label = 'yT') # yT = y_test\n    axis[i, 2].set_title(\"yPred vs yTest: \" + assetName)\n    axis[i, 2].legend(loc = 1)\n\nfig.suptitle('Metrics by Asset')\nfig.tight_layout() \n\npd.DataFrame.from_dict(correlations, orient = 'index', columns = ['Correlation'])","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-29T14:29:27.544150Z","iopub.status.idle":"2022-01-29T14:29:27.544485Z","shell.execute_reply.started":"2022-01-29T14:29:27.544306Z","shell.execute_reply":"2022-01-29T14:29:27.544325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize feature importance\nfor asset in preppedData:\n    xgb.plot_importance(asset.builtModel)\n    plt.rcParams['figure.figsize'] = [5, 5]\n    plt.show()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-29T14:29:27.545780Z","iopub.status.idle":"2022-01-29T14:29:27.546133Z","shell.execute_reply.started":"2022-01-29T14:29:27.545938Z","shell.execute_reply":"2022-01-29T14:29:27.545958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Submit Predictions Via API**","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.547203Z","iopub.status.idle":"2022-01-29T14:29:27.547556Z","shell.execute_reply.started":"2022-01-29T14:29:27.547368Z","shell.execute_reply":"2022-01-29T14:29:27.547387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.548942Z","iopub.status.idle":"2022-01-29T14:29:27.549460Z","shell.execute_reply.started":"2022-01-29T14:29:27.549248Z","shell.execute_reply":"2022-01-29T14:29:27.549272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.551213Z","iopub.status.idle":"2022-01-29T14:29:27.551786Z","shell.execute_reply.started":"2022-01-29T14:29:27.551568Z","shell.execute_reply":"2022-01-29T14:29:27.551592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    \n    # clean input df, set index\n    test_df['timestamp'] = pd.to_datetime(test_df['timestamp'], unit = 's')\n    test_df.set_index(['Asset_ID', 'timestamp'], inplace = True)\n\n    test_df.fillna(0) # fill na's\n    test_df = FeatureCols(test_df) # add in feature cols\n\n    rowId = test_df['row_id'].to_list()\n    assetId = test_df.index.get_level_values('Asset_ID')\n    data = test_df[featureCols[:-1]].to_numpy()\n    \n    # make predictions\n    predictions = []\n    \n    for i, row in enumerate(data):\n        asset = assetId[i]\n        newRow = pd.DataFrame(row, index = featureCols[:-1]).T # create a row with just the new data\n        \n        # add in rsi\n        closingPrices[i].append(newRow.iloc[0]['Close']) # add close\n        newRow['rsi'] = addRSI(closingPrices[i])[-1] # calculate RSI & add to df\n        closingPrices[i].pop(0) # remove first element so list doesn't get too long\n        \n        # predict\n        scaledRow = assetScalers[asset].transform(newRow) # scale data \n        dInputs = xgb.DMatrix(scaledRow, feature_names = featureCols) # change data type for model\n        prediction = preppedData[asset].builtModel.predict(dInputs) # predict\n        predictions.append(np.float16(prediction[0]))\n        \n    prediction_df = pd.DataFrame(list(zip(rowId, predictions)), columns = ['row_id', 'Target'])\n    \n    env.predict(prediction_df) # submit","metadata":{"execution":{"iopub.status.busy":"2022-01-29T14:29:27.553408Z","iopub.status.idle":"2022-01-29T14:29:27.553977Z","shell.execute_reply.started":"2022-01-29T14:29:27.553676Z","shell.execute_reply":"2022-01-29T14:29:27.553708Z"},"trusted":true},"execution_count":null,"outputs":[]}]}