{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pandas.tseries.offsets import DateOffset\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport scipy.stats as stats\nimport warnings\nimport datetime as dt\nfrom statsmodels.graphics.tsaplots import plot_pacf\nimport copy\nfrom IPython.display import Image\nwarnings.simplefilter(action = 'ignore', category = FutureWarning)\npd.options.mode.chained_assignment = None","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-30T17:12:10.885612Z","iopub.execute_input":"2022-01-30T17:12:10.885944Z","iopub.status.idle":"2022-01-30T17:12:12.577146Z","shell.execute_reply.started":"2022-01-30T17:12:10.885857Z","shell.execute_reply":"2022-01-30T17:12:12.576079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:12:12.578921Z","iopub.execute_input":"2022-01-30T17:12:12.579450Z","iopub.status.idle":"2022-01-30T17:12:12.597218Z","shell.execute_reply.started":"2022-01-30T17:12:12.579410Z","shell.execute_reply":"2022-01-30T17:12:12.595316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loadData(file):\n    df = pd.read_csv(file)\n    df['timestamp'] = pd.to_datetime(df['timestamp'], unit = 's')\n    df.set_index(['timestamp', 'Asset_ID'], inplace = True) # make multi-index\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:12:12.598986Z","iopub.execute_input":"2022-01-30T17:12:12.599621Z","iopub.status.idle":"2022-01-30T17:12:12.606292Z","shell.execute_reply.started":"2022-01-30T17:12:12.599586Z","shell.execute_reply":"2022-01-30T17:12:12.605430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read data, format, filter time\ndata = loadData('/kaggle/input/g-research-crypto-forecasting/train.csv')\ndata = data[data.index.get_level_values('timestamp') > '2020-12-30'] \nprint(data.info(show_counts = True))\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:12:12.609157Z","iopub.execute_input":"2022-01-30T17:12:12.609492Z","iopub.status.idle":"2022-01-30T17:13:16.126357Z","shell.execute_reply.started":"2022-01-30T17:12:12.609459Z","shell.execute_reply":"2022-01-30T17:13:16.124517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get supp train data\nsuppData = loadData('../input/g-research-crypto-forecasting/supplemental_train.csv')\nprint(suppData.info(show_counts = True))\nsuppData.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:16.128797Z","iopub.execute_input":"2022-01-30T17:13:16.129162Z","iopub.status.idle":"2022-01-30T17:13:23.673879Z","shell.execute_reply.started":"2022-01-30T17:13:16.129119Z","shell.execute_reply":"2022-01-30T17:13:23.672716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stack dataframes without overlapping index\n\noverlapDate = suppData.index.get_level_values('timestamp').min() # returns earliest time from suppTrain\ndata = data[data.index.get_level_values('timestamp') < overlapDate] # filter original DF so there's no overlap\n\nstacked = pd.concat([data, suppData], ignore_index = False, levels = 'timestamp')\n\ndouplicateRows = stacked.shape[0] - data.shape[0] - suppData.shape[0]\nprint(f\"There are {douplicateRows} missing rows\")","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:23.675499Z","iopub.execute_input":"2022-01-30T17:13:23.675733Z","iopub.status.idle":"2022-01-30T17:13:24.578051Z","shell.execute_reply.started":"2022-01-30T17:13:23.675709Z","shell.execute_reply":"2022-01-30T17:13:24.576393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the asset details into dictionaries\n\nfile = '../input/g-research-crypto-forecasting/asset_details.csv'\n\nassetDetails = (pd.read_csv(file)).sort_values(by = ['Asset_ID']).reset_index(drop = True)\n\nnames = {}\nweights = {}\n\nfor row in assetDetails.index:\n    assetID = assetDetails.at[row, 'Asset_ID'] \n    names[assetID] = assetDetails.at[row, 'Asset_Name']\n    weights[assetID] = assetDetails.at[row, 'Weight']\n\nprint(names)\nprint(weights)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:24.580356Z","iopub.execute_input":"2022-01-30T17:13:24.580593Z","iopub.status.idle":"2022-01-30T17:13:24.598622Z","shell.execute_reply.started":"2022-01-30T17:13:24.580569Z","shell.execute_reply":"2022-01-30T17:13:24.597464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create functions to add in feature cols\n    \ndef FeatureCols(df):\n    df['hlDiff'] = df['High'] - df['Low'] # high - low to measure volitility\n    df['avgSize'] = df['Volume'] // df['Count'] # average size of each trade as int\n    \n    # shadows\n    df['uShadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n    df['bShadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n    \n    # encode minute from timestamp\n    df['minute'] = df.index.get_level_values('timestamp').minute\n    df['minSin'] = np.sin(df.minute*(2.*np.pi/60))\n    df['minCos'] = np.cos(df.minute*(2.*np.pi/60))\n    \n    # encode day of month from timestamp\n    df['mDay'] = df.index.get_level_values('timestamp').day\n    df['daySin'] = np.sin(df.mDay*(2.*np.pi/31))\n    df['dayCos'] = np.cos(df.mDay*(2.*np.pi/31))\n    \n    df.drop(columns = ['minute', 'mDay'], axis = 1, inplace = True) # clear progress columns\n    \n    return (df)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:24.601858Z","iopub.execute_input":"2022-01-30T17:13:24.602090Z","iopub.status.idle":"2022-01-30T17:13:24.613119Z","shell.execute_reply.started":"2022-01-30T17:13:24.602068Z","shell.execute_reply":"2022-01-30T17:13:24.611876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final = FeatureCols(stacked) # Apply feature cols to the entire dataset\nfinal = final[ [ col for col in final.columns if col != 'Target' ] + ['Target'] ] # move target to end\nfinal.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:24.615480Z","iopub.execute_input":"2022-01-30T17:13:24.616126Z","iopub.status.idle":"2022-01-30T17:13:28.768195Z","shell.execute_reply.started":"2022-01-30T17:13:24.616050Z","shell.execute_reply":"2022-01-30T17:13:28.766965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final.info(show_counts = True) # check dataset after feature cols were added","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:28.771293Z","iopub.execute_input":"2022-01-30T17:13:28.771618Z","iopub.status.idle":"2022-01-30T17:13:29.082328Z","shell.execute_reply.started":"2022-01-30T17:13:28.771590Z","shell.execute_reply":"2022-01-30T17:13:29.080236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get libraries for preprocessing \nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, FunctionTransformer","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:29.084354Z","iopub.execute_input":"2022-01-30T17:13:29.084672Z","iopub.status.idle":"2022-01-30T17:13:29.594474Z","shell.execute_reply.started":"2022-01-30T17:13:29.084642Z","shell.execute_reply":"2022-01-30T17:13:29.593522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add in rsi\n\nwindowLen = 30\n\n# https://www.alpharithms.com/relative-strength-index-rsi-in-python-470209/\ndef addRSI(closePrices):\n\n    # initalize variables\n    gains = [] # Initialize containers for avg. gains and losses\n    losses = []\n\n    window = [] # Create a container for current lookback prices\n\n    prev_avg_gain = None # Keeps track of previous average values\n    prev_avg_loss = None\n\n    output = [] # Create a container for our final output\n    \n    # caclulate price differences\n    for i, price in enumerate(closePrices): # keep track of the price for the first period but don't calculate a difference value.\n        \n        if i == 0:\n            window.append(price)\n            output.append(0)\n            continue\n    \n        difference = round(closePrices[i] - closePrices[i - 1], 2) # calculate the difference between price and previous price as a rounded value\n    \n        # Calculate Gains & Losses\n        if difference > 0: # Record positive differences as gains\n            gain = difference\n            loss = 0\n\n        elif difference < 0: # Record negative differences as losses\n            gain = 0\n            loss = abs(difference)\n\n        else: # Record no movements as neutral\n            gain = 0\n            loss = 0\n    \n        gains.append(gain) # Save gains/losses\n        losses.append(loss)\n\n        if i < windowLen: # Continue to iterate until enough gains/losses data is available to calculate the initial RS value\n            window.append(price)\n            output.append(0)\n            continue\n        \n        # Calculate Average Gains & Losses\n    \n        if i == windowLen: # Calculate SMA for first gain\n            avg_gain = sum(gains) / len(gains)\n            avg_loss = sum(losses) / len(losses)\n    \n        else: # Use WSM after initial window-length period\n            avg_gain = (prev_avg_gain * (windowLen - 1) + gain) / windowLen\n            avg_loss = (prev_avg_loss * (windowLen - 1) + loss) / windowLen\n        \n        prev_avg_gain = avg_gain # Keep in memory\n        prev_avg_loss = avg_loss\n        \n        if avg_loss == 0:\n            rsi = 0\n        \n        else:\n            # Calculate the RS Value\n            rs = np.float16(avg_gain) /  np.float16(avg_loss)\n    \n            # Calculate the RSI Value\n            rsi = np.float16(100 - (100 / (1 + rs)))\n    \n        # Remove oldest values\n        window.append(price)\n        window.pop(0)\n        gains.pop(0)\n        losses.pop(0)\n\n        output.append(rsi)\n    \n    return output","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:29.595594Z","iopub.execute_input":"2022-01-30T17:13:29.597232Z","iopub.status.idle":"2022-01-30T17:13:29.614812Z","shell.execute_reply.started":"2022-01-30T17:13:29.597163Z","shell.execute_reply":"2022-01-30T17:13:29.612605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select cols\nnoScale_features = ['minSin', 'minCos', 'daySin', 'dayCos']\nhighVol_features = ['Volume']\ncont_features = ['Close', 'avgSize', 'uShadow', 'bShadow', 'rsi']\nfeatureCols = noScale_features + highVol_features + cont_features\n\n# set up pipeline for different data types\ndef ScaleData(inputDF, noScale_features = noScale_features, highVol_features = highVol_features, cont_features = cont_features):\n\n    noScale_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'most_frequent'))])\n\n    highVol_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'constant', fill_value = 1)),\n        ('encoder', RobustScaler(quantile_range = (20.0, 80.0)))])\n\n    cont_transformer = Pipeline(steps = [\n        ('imputer', SimpleImputer(strategy = 'mean')),\n        ('encoder', MinMaxScaler(feature_range = (0, 1)))])\n\n    # process\n    preprosessor = ColumnTransformer(transformers = [\n        ('noScale', noScale_transformer, noScale_features),\n        ('negPos', highVol_transformer, highVol_features),\n        ('cont', cont_transformer, cont_features)])\n    \n    cols = noScale_features + highVol_features + cont_features # get cols we want to transform\n    df_to_scale = inputDF[cols] # select these cols from input df\n    fitScaler = preprosessor.fit(df_to_scale) # fit scaler\n    scaled = fitScaler.transform(df_to_scale) # scale\n    \n    return [(pd.DataFrame(scaled, columns = cols).set_index(inputDF.index)), fitScaler] # df with scaled data & fit model to be used later","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:13:29.616654Z","iopub.execute_input":"2022-01-30T17:13:29.617911Z","iopub.status.idle":"2022-01-30T17:13:29.641263Z","shell.execute_reply.started":"2022-01-30T17:13:29.617852Z","shell.execute_reply":"2022-01-30T17:13:29.640591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split each table into a df, fill missing values, create feature cols, scale\n\nassets = []\nassetScalers = {} # save the scaler\nclosingPrices = {}\n\nfor asset in names.keys():\n    df = final.xs(asset, level = 'Asset_ID')\n    \n    timeStamps = df.index\n    \n    # set index so there's no missing times\n    minDate = timeStamps.min()\n    maxDate = timeStamps.max()\n    df = df.reindex(index = list(pd.date_range(minDate, maxDate, freq = 'min')), method = 'pad')\n    \n    # add in rsi\n    df['rsi'] = addRSI(df['Close'].to_list())\n    df = df.iloc[windowLen:, :] # remove first rows with nan\n    closingPrices[asset] = df['Close'].to_list()[-windowLen:] # save closing prices for predictions\n    \n    # remove last rows with missing target\n    df = df.iloc[:-250, :]\n\n    # scale data\n    result = ScaleData(df)\n    scaledDF, fitScaler = result[0], result[1] \n    \n    # fill na's for target\n    scaledDF['Target'] = df['Target'].fillna(0)\n    \n    assets.append(scaledDF) # save transformed df\n    assetScalers[asset] = fitScaler # save scaler \n    \n    # visualize data\n    print(names[asset])\n    \n    plt.figure(figsize = (10, 10))\n    sns.pairplot(scaledDF.sample(10000, random_state = 10, ignore_index = True))\n    plt.show()\n    \n    plot_pacf(scaledDF['Target'].to_list(), lags = 50)\n    plt.show()\n    \ndel data # we no longer need the table. Free up memory.","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-30T17:13:29.642501Z","iopub.execute_input":"2022-01-30T17:13:29.642867Z","iopub.status.idle":"2022-01-30T17:24:13.274275Z","shell.execute_reply.started":"2022-01-30T17:13:29.642841Z","shell.execute_reply":"2022-01-30T17:24:13.273115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create class to store data\nclass Asset():\n    def __init__(self, xTrain, xTest, yTrain, yTest, quants, trainWeights, testWeights, builtModel = None):\n        self.xTrain = xTrain\n        self.xTest = xTest\n        self.yTrain = yTrain\n        self.yTest = yTest\n        self.quants = quants\n        self.trainWeights = trainWeights\n        self.testWeights = testWeights\n        self.builtModel = builtModel","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:24:13.275540Z","iopub.execute_input":"2022-01-30T17:24:13.275809Z","iopub.status.idle":"2022-01-30T17:24:13.284300Z","shell.execute_reply.started":"2022-01-30T17:24:13.275767Z","shell.execute_reply":"2022-01-30T17:24:13.282806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.utils import class_weight # weight each class\n\ntrainPct = 0.8\nassetNames = names.keys()\n\npreppedData = [] # store a instance for each asset\nassetShapes = [] # store the shape of each assets df\norigTargets = {} # save the original target values\n\nfor i, asset in enumerate(assets):\n    \n    # drop outlier returns\n    minOutlier = asset['Target'].quantile(0.025)\n    maxOutlier = asset['Target'].quantile(0.985)\n    asset = asset.loc[(asset['Target'] > minOutlier) & (asset['Target'] < maxOutlier)]\n    \n    # get quants\n    middleL = asset['Target'].quantile(0.3333)\n    middleH = asset['Target'].quantile(0.6666)\n    \n    bSplit = asset.loc[(asset['Target'] < middleL)]['Target']\n    mSplit = asset.loc[(asset['Target'] > middleL) & (asset['Target'] < middleH)]['Target']\n    tSplit =asset.loc[(asset['Target'] > middleH)]['Target']\n\n    bottom = bSplit.mean()\n    middle = mSplit.mean()\n    top = tSplit.mean()\n    \n    quants = {'b' : bottom, 'ml' : middleL, 'm' : middle, \n              'mh' : middleH, 't' : top}\n    \n    # create target function\n    def createTarget(x):\n        if x < middleL:\n            return 0\n        elif x > middleH:\n            return 1\n        else:\n            return 2\n    \n    asset['cTarget'] = asset.apply(lambda x: createTarget(x['Target']), axis = 1)\n    \n    # compute class weights\n    weights = class_weight.compute_sample_weight(class_weight = 'balanced', y = asset['cTarget'])\n    \n    origTargets[i] = np.array(asset['Target'].values.tolist()) # save original target values\n    \n    # split data \n    y = np.array(asset['cTarget'].values.tolist())\n    asset.drop(['Target', 'cTarget'], axis = 1, inplace = True)\n    \n    x = np.array(asset) \n    \n    # append a class instance with the training and testing data\n    trainIndex = int(len(x) * trainPct)\n    preppedData.append(Asset(x[:trainIndex], x[trainIndex:], y[:trainIndex], y[trainIndex:], \n                             quants, weights[:trainIndex], weights[trainIndex:]))\n    \n    # append shapes\n    shapes = {}\n    shapes['xTrain_shape'] = np.shape(preppedData[i].xTrain)\n    shapes['xTest_shape'] = np.shape(preppedData[i].xTest)\n    shapes['yTrain_shape'] = np.shape(preppedData[i].yTrain)\n    shapes['yTest_shape'] = np.shape(preppedData[i].yTest)\n    assetShapes.append(shapes)\n    \nshapes = pd.DataFrame(assetShapes, index = assetNames)\nshapes","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:24:13.285939Z","iopub.execute_input":"2022-01-30T17:24:13.286500Z","iopub.status.idle":"2022-01-30T17:26:20.653884Z","shell.execute_reply.started":"2022-01-30T17:24:13.286468Z","shell.execute_reply":"2022-01-30T17:26:20.653231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize quants\nfrom scipy.stats import kde\n\n#list of all plot locations\nrows, cols = 3, 5\nplotList = [] \nfor row in list(range(rows)):\n    for col in list(range(cols)):\n        plotList.append([row, col])\n\n# loss values by epoch\nfig, axis = plt.subplots(rows, cols, figsize = (10, 10))\n\nfor i, asset in enumerate(preppedData):\n    target = origTargets[i]\n    prob_density = kde.gaussian_kde(target)\n    prob_density.covariance_factor = lambda : 0.25\n    prob_density._compute_covariance()\n    \n    x = np.linspace(-0.02, 0.02, 300)\n    y = prob_density(x)\n    \n    pltRow, pltCol = plotList[i][0], plotList[i][1]\n    axis[pltRow, pltCol].plot(x, y)\n    axis[pltRow, pltCol].axvline(x = asset.quants['b'], color = 'red', linestyle = '--')\n    axis[pltRow, pltCol].axvline(x = asset.quants['ml'], color = 'red', linestyle = '--')\n    axis[pltRow, pltCol].axvline(x = asset.quants['mh'], color = 'red', linestyle = '--')\n    axis[pltRow, pltCol].axvline(x = asset.quants['t'], color = 'red', linestyle = '--')\n    axis[pltRow, pltCol].set_title(names[i], y = 1.05)\n    axis[pltRow, pltCol].ticklabel_format(style = 'sci')\n\nfig.suptitle('Density of Target')\nfig.tight_layout() ","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:26:20.655027Z","iopub.execute_input":"2022-01-30T17:26:20.655435Z","iopub.status.idle":"2022-01-30T17:27:30.643717Z","shell.execute_reply.started":"2022-01-30T17:26:20.655401Z","shell.execute_reply":"2022-01-30T17:27:30.642427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get libraries\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:27:30.645585Z","iopub.execute_input":"2022-01-30T17:27:30.645988Z","iopub.status.idle":"2022-01-30T17:27:30.804213Z","shell.execute_reply.started":"2022-01-30T17:27:30.645947Z","shell.execute_reply":"2022-01-30T17:27:30.803316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set params\nparam = {'max_depth' : 15,\n         'eta' : 0.05,\n         'num_class' : 3,\n         'eval_metric' : 'merror',\n         'objective' : 'multi:softmax',\n         'min_child_weight' : 1,\n         'tree_method' : 'hist',\n         'gamma' : 0.02}\n\nepochs = 200\nesRounds = 10\n\n# save metrics\naccScores = {}\npredictions = {}\n\n# fit\nfor i, asset in enumerate(preppedData):\n    print(names[i])\n    \n    dtrain = xgb.DMatrix(asset.xTrain, feature_names = featureCols, label = asset.yTrain) \n    dtest = xgb.DMatrix(asset.xTest, feature_names = featureCols, label = asset.yTest) \n    \n    asset.builtModel = xgb.train(param, dtrain, epochs, evals = [(dtest, \"Test\")], early_stopping_rounds = esRounds)\n    prediction = asset.builtModel.predict(dtest) # predict\n    \n    predictions[i] = prediction # save predictions\n    accScore = accuracy_score(asset.yTest, prediction) # get accuracy\n    accScores[names[i]] = \"{:.2%}\".format(accScore) # append\n    \npd.DataFrame.from_dict(accScores, orient = 'index', columns = ['Accuracy Score'])","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-30T17:27:30.805471Z","iopub.execute_input":"2022-01-30T17:27:30.805775Z","iopub.status.idle":"2022-01-30T17:33:22.587952Z","shell.execute_reply.started":"2022-01-30T17:27:30.805729Z","shell.execute_reply":"2022-01-30T17:33:22.586437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize feature importance\nfor asset in preppedData:\n    xgb.plot_importance(asset.builtModel)\n    plt.rcParams['figure.figsize'] = [5, 5]\n    plt.show()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-30T17:33:22.590063Z","iopub.execute_input":"2022-01-30T17:33:22.590332Z","iopub.status.idle":"2022-01-30T17:33:27.169085Z","shell.execute_reply.started":"2022-01-30T17:33:22.590305Z","shell.execute_reply":"2022-01-30T17:33:27.167978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize results\nfrom sklearn.metrics import confusion_matrix\n\n# confusion matrix\nfor i, asset in enumerate(preppedData):\n    plt.title('Confusion Matrix for: ' + names[i])\n    sns.heatmap(confusion_matrix(asset.yTest, predictions[i]), annot = True, cmap = \"YlGn\", fmt = 'g')\n    plt.xlabel('Predicted classes')\n    plt.ylabel('True Classes')\n    plt.show()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-01-30T17:33:27.170676Z","iopub.execute_input":"2022-01-30T17:33:27.170953Z","iopub.status.idle":"2022-01-30T17:33:37.019848Z","shell.execute_reply.started":"2022-01-30T17:33:27.170916Z","shell.execute_reply":"2022-01-30T17:33:37.018149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:33:37.021950Z","iopub.execute_input":"2022-01-30T17:33:37.022199Z","iopub.status.idle":"2022-01-30T17:33:37.065389Z","shell.execute_reply.started":"2022-01-30T17:33:37.022173Z","shell.execute_reply":"2022-01-30T17:33:37.064550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:33:37.068252Z","iopub.execute_input":"2022-01-30T17:33:37.068514Z","iopub.status.idle":"2022-01-30T17:33:37.073762Z","shell.execute_reply.started":"2022-01-30T17:33:37.068488Z","shell.execute_reply":"2022-01-30T17:33:37.072574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:33:37.075176Z","iopub.execute_input":"2022-01-30T17:33:37.075425Z","iopub.status.idle":"2022-01-30T17:33:37.096779Z","shell.execute_reply.started":"2022-01-30T17:33:37.075395Z","shell.execute_reply":"2022-01-30T17:33:37.094790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    \n    # clean input df, set index\n    test_df['timestamp'] = pd.to_datetime(test_df['timestamp'], unit = 's')\n    test_df.set_index(['Asset_ID', 'timestamp'], inplace = True)\n\n    test_df.fillna(0) # fill na's\n    test_df = FeatureCols(test_df) # add in feature cols\n    \n    rowId = test_df['row_id'].to_list()\n    assetId = test_df.index.get_level_values('Asset_ID')\n    data = test_df[featureCols[:-1]].to_numpy()\n  \n    # make predictions\n    predictions = []\n    \n    for i, row in enumerate(data):\n        asset = assetId[i]\n        newRow = pd.DataFrame(row, index = featureCols[:-1]).T # create a row with just the new data\n        \n        # add in rsi\n        closingPrices[i].append(newRow.iloc[0]['Close']) # add close\n        newRow['rsi'] = addRSI(closingPrices[i])[-1] # calculate RSI & add to df\n        closingPrices[i].pop(0) # remove first element so list doesn't get too long\n        \n        # predict\n        scaledRow = assetScalers[asset].transform(newRow) # scale data\n        dInputs = xgb.DMatrix(scaledRow, feature_names = featureCols) # change data type for model\n        prediction = int(preppedData[asset].builtModel.predict(dInputs)) # predict\n        \n        quants = preppedData[asset].quants # get quants for the asset\n    \n        if prediction == 0:\n            target = quants['b']\n        elif prediction == 1:\n            target = quants['t']\n        else:\n            target = quants['m']\n            \n        predictions.append(np.float16(target))\n        \n    prediction_df = pd.DataFrame(list(zip(rowId, predictions)), columns = ['row_id', 'Target'])\n    \n    env.predict(prediction_df) # submit","metadata":{"execution":{"iopub.status.busy":"2022-01-30T17:33:37.098557Z","iopub.execute_input":"2022-01-30T17:33:37.100112Z","iopub.status.idle":"2022-01-30T17:33:37.842124Z","shell.execute_reply.started":"2022-01-30T17:33:37.100048Z","shell.execute_reply":"2022-01-30T17:33:37.841563Z"},"trusted":true},"execution_count":null,"outputs":[]}]}