{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data import\n","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\npd.options.mode.chained_assignment = None\nimport gc\n\ndirectory = '/kaggle/input/g-research-crypto-forecasting/'\nfile_path = os.path.join(directory, 'train.csv')\ndtypes = {\n    'timestamp': np.int64,\n    'Asset_ID': np.int8,\n     'Count': np.int32,\n     'Open': np.float64,\n     'High': np.float64,\n     'Low': np.float64,\n    'Close': np.float64,\n     'Volume': np.float64,\n     'VWAP': np.float64,\n    'Target': np.float64,\n}\ndata = pd.read_csv(file_path, dtype=dtypes, usecols=list(dtypes.keys()))\ndata['Time'] = pd.to_datetime(data['timestamp'], unit='s')\n\nfile_path = os.path.join(directory, 'asset_details.csv')\ndetails = pd.read_csv(file_path)\n\ndata = pd.merge(data, \n                details, \n                on ='Asset_ID', \n                how ='left')\n\nprint(data.head())\nprint(details)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T20:48:51.045261Z","iopub.execute_input":"2021-11-30T20:48:51.045482Z","iopub.status.idle":"2021-11-30T20:49:55.713867Z","shell.execute_reply.started":"2021-11-30T20:48:51.045459Z","shell.execute_reply":"2021-11-30T20:49:55.712847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize the data and choose a train-test-split","metadata":{}},{"cell_type":"code","source":"import matplotlib.pylab as plt\n\n\nbtc = data[data.Asset_ID == 1]\nbtc.set_index('timestamp', inplace = True)\nbtc = btc.reindex(range(btc.index[0], btc.index[-1] + 60, 60), method = 'pad')\nbtc.sort_index(inplace = True)\n\nfig, ax = plt.subplots(figsize = (12, 4))\n\nax.plot(btc.Time, btc.Close)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T20:49:55.716633Z","iopub.execute_input":"2021-11-30T20:49:55.716938Z","iopub.status.idle":"2021-11-30T20:49:57.376973Z","shell.execute_reply.started":"2021-11-30T20:49:55.716901Z","shell.execute_reply":"2021-11-30T20:49:57.376155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The level of Bitcoin drastically changed over time, so lets take a closer look, how its progression looks like for different time periods...","metadata":{}},{"cell_type":"code","source":"from datetime import datetime\nfrom dateutil.relativedelta import relativedelta\n\nstarting_date = btc.Time.iloc[0]\ntimesplits = [starting_date + i * relativedelta(months = 6) for i in range(8)] + [btc.Time.iloc[-1]]\n\nimport matplotlib.pyplot as plt\n\n# Subplots are organized in a Rows x Cols Grid\n# Tot and Cols are known\nTot = len(timesplits) - 1\nCols = 2\n\n\n# Compute Rows required\nRows = Tot // Cols \nRows += Tot % Cols\n\n# Create a Position index\nPosition = range(1,Tot + 1)\n\n# Create main figure\nfig = plt.figure(1)\nfig.set_figheight(30)\nfig.set_figwidth(20)\n\nfor k in range(Tot):\n  # add every single subplot to the figure with a for loop\n    btc_tmp = btc.loc[datetime.timestamp(timesplits[k]):datetime.timestamp(timesplits[k+1])]\n    ax = fig.add_subplot(Rows,Cols,Position[k])\n    ax.plot(btc_tmp.Time, btc_tmp.Close)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T20:49:57.378346Z","iopub.execute_input":"2021-11-30T20:49:57.378608Z","iopub.status.idle":"2021-11-30T20:49:58.860863Z","shell.execute_reply.started":"2021-11-30T20:49:57.378578Z","shell.execute_reply":"2021-11-30T20:49:58.860234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, we have different market conditions in the subsamples, as an example I will use the first half of 2021 as training and the remaining data as testing data. As laid out in a previous notebook, I do not think that the original variables (High, Low, Close, Volume, etc.) will serve well as predictive features, which is why I will give technical indicators a chance. To select the best suited for the job, I am calculating all technical indicators of the ta-package and select the first 20 indicators which exhibit the highest rank-correlation to the target variable.","metadata":{}},{"cell_type":"code","source":"!pip install ta","metadata":{"execution":{"iopub.status.busy":"2021-11-30T20:49:58.861883Z","iopub.execute_input":"2021-11-30T20:49:58.862084Z","iopub.status.idle":"2021-11-30T20:50:12.85986Z","shell.execute_reply.started":"2021-11-30T20:49:58.86205Z","shell.execute_reply":"2021-11-30T20:50:12.858968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ta\n\nstart_train, end_train = datetime.timestamp(timesplits[-3]), datetime.timestamp(timesplits[-2])\nstart_test, end_test = datetime.timestamp(timesplits[-2]), datetime.timestamp(timesplits[-1])\n\ntrain_data, test_data = btc.loc[start_train:end_train], btc.loc[start_test:end_test][3600:]\n\nupper_shadow = lambda asset: asset.High - np.maximum(asset.Close,asset.Open)\nlower_shadow = lambda asset: np.minimum(asset.Close,asset.Open)- asset.Low\n\ntrain_data['close_1'] = train_data.Close.diff()\ntrain_data['close_15'] = train_data.Close.diff(15)\ntrain_data['close_60'] = train_data.Close.diff(60)\n\ntrain_data['count_1'] = train_data.Count.diff()\ntrain_data['count_15'] = train_data.Count.diff(15)\ntrain_data['count_60'] = train_data.Count.diff(60)\n\ntrain_data['volume_1'] = train_data.Volume.diff()\ntrain_data['volume_15'] = train_data.Volume.diff(15)\ntrain_data['volume_60'] = train_data.Volume.diff(60)\n\ntrain_data['upper_shadow'] = upper_shadow(train_data)\ntrain_data['lower_shadow'] = lower_shadow(train_data)\n\ntrain_data = ta.add_all_ta_features(train_data,\n                                       open = 'Open',\n                                       high = 'High',\n                                       low = 'Low',\n                                       close = 'Close',\n                                       volume = 'Volume',\n                                       fillna = False)\n\ntest_data['close_1'] = test_data.Close.diff()\ntest_data['close_15'] = test_data.Close.diff(15)\ntest_data['close_60'] = test_data.Close.diff(60)\n\ntest_data['count_1'] = test_data.Count.diff()\ntest_data['count_15'] = test_data.Count.diff(15)\ntest_data['count_60'] = test_data.Count.diff(60)\n\ntest_data['volume_1'] = test_data.Volume.diff()\ntest_data['volume_15'] = test_data.Volume.diff(15)\ntest_data['volume_60'] = test_data.Volume.diff(60)\n\ntest_data['upper_shadow'] = upper_shadow(test_data)\ntest_data['lower_shadow'] = lower_shadow(test_data)\n\ntest_data = ta.add_all_ta_features(test_data,\n                                       open = 'Open',\n                                       high = 'High',\n                                       low = 'Low',\n                                       close = 'Close',\n                                       volume = 'Volume',\n                                       fillna = False)\n\n# delete variables with many missing values\ntrain_data = train_data.drop(train_data.columns[train_data.isnull().sum() > 100], axis = 1)\ntest_data = test_data.drop(test_data.columns[test_data.isnull().sum() > 100], axis = 1)\n\nfind_corr_features = train_data.drop(['Asset_ID', 'Time', 'Weight'], axis = 1).corr(method = 'spearman')['Target'].abs().sort_values(ascending = False)\nfind_corr_features[1:21]","metadata":{"execution":{"iopub.status.busy":"2021-11-30T20:54:29.185933Z","iopub.execute_input":"2021-11-30T20:54:29.186334Z","iopub.status.idle":"2021-11-30T21:02:54.978042Z","shell.execute_reply.started":"2021-11-30T20:54:29.186287Z","shell.execute_reply":"2021-11-30T21:02:54.976854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us delete rows with missing values and visualize correlations among feature variables and the target variable.","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n\ntrain_data.dropna(inplace = True)\ntest_data.dropna(inplace = True)\n\nuse_features = list(find_corr_features[:20].index)\n\nfig, axs = plt.subplots(1, 2, figsize = (20, 10))\nsns.heatmap(train_data[use_features].corr(method = 'spearman').abs(), ax = axs[0])\nsns.heatmap(test_data[use_features].corr(method = 'spearman').abs(), ax = axs[1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:04:24.404459Z","iopub.execute_input":"2021-11-30T21:04:24.404684Z","iopub.status.idle":"2021-11-30T21:04:28.683323Z","shell.execute_reply.started":"2021-11-30T21:04:24.404661Z","shell.execute_reply":"2021-11-30T21:04:28.681944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us further visualize the behavior of feature variables and the target variable over time as well as the pairwise relationship between feature variables and the target variable.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Subplots are organized in a Rows x Cols Grid\n# Tot and Cols are known\n\nTot = len(use_features)\nCols = 5\n\n\n# Compute Rows required\nRows = Tot // Cols \nRows += Tot % Cols\n\n# Create a Position index\nPosition = range(1,Tot + 1)\n\n# Create main figure\nfig = plt.figure(1)\nfig.set_figheight(30)\nfig.set_figwidth(20)\n\nfor k in range(Tot):\n  # add every single subplot to the figure with a for loop\n\n    ax = fig.add_subplot(Rows,Cols,Position[k])\n    ax.plot(train_data['Time'], train_data[use_features][use_features[k]].values)\n    ax.set_title(use_features[k])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:04:28.68565Z","iopub.execute_input":"2021-11-30T21:04:28.685897Z","iopub.status.idle":"2021-11-30T21:04:33.961338Z","shell.execute_reply.started":"2021-11-30T21:04:28.685866Z","shell.execute_reply":"2021-11-30T21:04:33.959861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Subplots are organized in a Rows x Cols Grid\n# Tot and Cols are known\n\nTot = len(use_features)\nCols = 5\n\n\n# Compute Rows required\nRows = Tot // Cols \nRows += Tot % Cols\n\n# Create a Position index\nPosition = range(1,Tot + 1)\n\n# Create main figure\nfig = plt.figure(1)\nfig.set_figheight(30)\nfig.set_figwidth(20)\n\nfor k in range(Tot):\n  # add every single subplot to the figure with a for loop\n\n    ax = fig.add_subplot(Rows,Cols,Position[k])\n    ax.plot(test_data['Time'], test_data[use_features][use_features[k]].values)\n    ax.set_title(use_features[k])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:04:38.594715Z","iopub.execute_input":"2021-11-30T21:04:38.59495Z","iopub.status.idle":"2021-11-30T21:04:43.571559Z","shell.execute_reply.started":"2021-11-30T21:04:38.594927Z","shell.execute_reply":"2021-11-30T21:04:43.570219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Subplots are organized in a Rows x Cols Grid\n# Tot and Cols are known\nTot = len(use_features)\nCols = 5\n\n\n# Compute Rows required\nRows = Tot // Cols \nRows += Tot % Cols\n\n# Create a Position index\nPosition = range(1,Tot + 1)\n\n# Create main figure\nfig = plt.figure(1)\nfig.set_figheight(30)\nfig.set_figwidth(20)\n\nfor k in range(Tot):\n  # add every single subplot to the figure with a for loop\n\n    ax = fig.add_subplot(Rows,Cols,Position[k])\n    ax.scatter(train_data['Target'].values, train_data[use_features[k]].values)\n    ax.set_title(use_features[k])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:04:57.434711Z","iopub.execute_input":"2021-11-30T21:04:57.434968Z","iopub.status.idle":"2021-11-30T21:05:11.582939Z","shell.execute_reply.started":"2021-11-30T21:04:57.434928Z","shell.execute_reply":"2021-11-30T21:05:11.582037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Subplots are organized in a Rows x Cols Grid\n# Tot and Cols are known\nTot = len(use_features)\nCols = 5\n\n\n# Compute Rows required\nRows = Tot // Cols \nRows += Tot % Cols\n\n# Create a Position index\nPosition = range(1,Tot + 1)\n\n# Create main figure\nfig = plt.figure(1)\nfig.set_figheight(30)\nfig.set_figwidth(20)\n\nfor k in range(Tot):\n  # add every single subplot to the figure with a for loop\n\n    ax = fig.add_subplot(Rows,Cols,Position[k])\n    ax.scatter(test_data['Target'].values, test_data[use_features[k]].values)\n    ax.set_title(use_features[k])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:05:11.584582Z","iopub.execute_input":"2021-11-30T21:05:11.584773Z","iopub.status.idle":"2021-11-30T21:05:19.380217Z","shell.execute_reply.started":"2021-11-30T21:05:11.584749Z","shell.execute_reply":"2021-11-30T21:05:19.379314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To be honest, I do not think that this looks promising (at least on a pairwise perspective). Anyway, let us scale the data and estimate a plain forward neural network.","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n\nX_scaler = MinMaxScaler(feature_range = (0, 1))\nX_train = train_data[use_features].drop(['Target'], axis = 1)\nX_test = test_data[use_features].drop(['Target'], axis = 1)\n\ny_train = train_data['Target'].values\ny_test = test_data['Target'].values\n\nX_train_ = X_scaler.fit_transform(X_train)\nX_test_ = X_scaler.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:05:19.38129Z","iopub.execute_input":"2021-11-30T21:05:19.381982Z","iopub.status.idle":"2021-11-30T21:05:19.595412Z","shell.execute_reply.started":"2021-11-30T21:05:19.38194Z","shell.execute_reply":"2021-11-30T21:05:19.594664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nmodel = tf.keras.Sequential([\n    tf.keras.layers.InputLayer(input_shape = (X_train_.shape[1])),\n    tf.keras.layers.Dense(20, activation = 'selu'),\n    tf.keras.layers.Dropout(0.50),\n    tf.keras.layers.Dense(10, activation = 'selu'),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(1)\n])\n\nmodel.compile(loss = 'mean_absolute_error', optimizer = 'adam')\nhistory = model.fit(X_train_, y_train, epochs = 5, validation_data = (X_test_, y_test))\nplt.plot(history.history['loss'], label = 'training')\nplt.plot(history.history['val_loss'], label = 'test')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:05:19.596844Z","iopub.execute_input":"2021-11-30T21:05:19.598006Z","iopub.status.idle":"2021-11-30T21:07:49.749922Z","shell.execute_reply.started":"2021-11-30T21:05:19.597963Z","shell.execute_reply":"2021-11-30T21:07:49.74779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us check correlation between predictions and target realizations visually and quantitatively.","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize = (12,8))\n\naxs[0].scatter(model.predict(X_train_).flatten(), y_train)\naxs[1].scatter(model.predict(X_test_).flatten(), y_test)\nplt.show()\n\nprint(np.corrcoef(model.predict(X_train_).flatten(), y_train)[0, 1])\nprint(np.corrcoef(model.predict(X_test_).flatten(), y_test)[0, 1])","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:07:56.871698Z","iopub.execute_input":"2021-11-30T21:07:56.872316Z","iopub.status.idle":"2021-11-30T21:08:19.62278Z","shell.execute_reply.started":"2021-11-30T21:07:56.872285Z","shell.execute_reply":"2021-11-30T21:08:19.621865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Not that good at all, but in my experience quite typical for financial data. Furthermore, we can see something which I would consider as critical with respect to the performance metric correlation - it is sensitive towards outliers which makes it very prone to single extreme observations, this will get more clear with the results below for the recurrent network architecture. \n\nNow, let us find out which of the feature variables are most important for predictions. This is gradient based and by help of another notebook of another user (just search for gradient and feature)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nX = tf.Variable(X_train_)\ny = tf.Variable(y_train)\nwith tf.GradientTape() as tape:\n    tape.watch(X)\n    pred = model(X)\ngrad = tf.abs(tape.gradient(pred,X))\ngrad = tf.reduce_mean(grad,axis=0)\nfeature_importance = grad.numpy() / grad.numpy().sum()\n\n\nplt.figure(figsize=(10,20))\nplt.barh(X_train.columns[np.argsort(feature_importance)], np.sort(feature_importance))\nplt.title('Feature importance')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:10:47.429152Z","iopub.execute_input":"2021-11-30T21:10:47.43032Z","iopub.status.idle":"2021-11-30T21:10:47.817052Z","shell.execute_reply.started":"2021-11-30T21:10:47.430244Z","shell.execute_reply":"2021-11-30T21:10:47.816014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Obviously, recurrent networks may be a better choice for autocorrelated time series data like this. So let us find out...first, we need to reshape the data, such that each sample consists of a series with a certain \"lookback\" time frame.","metadata":{}},{"cell_type":"code","source":"lookback = 30\n\nX_train_rnn = []\ny_train_rnn = []\n\nfor t in range(len(X_train_) - lookback):\n    X_train_rnn.append(X_train_[t:(t + lookback)])\n    y_train_rnn.append(y_train[(t + lookback)])\n    \nX_test_rnn = []\ny_test_rnn = []\n\nfor t in range(len(X_test_) - lookback):\n    X_test_rnn.append(X_test_[t:(t + lookback)])\n    y_test_rnn.append(y_test[(t + lookback)])\n    \nX_train_rnn = np.array(X_train_rnn)\nX_test_rnn = np.array(X_test_rnn)\n\ny_train_rnn = np.array(y_train_rnn)\ny_test_rnn = np.array(y_test_rnn)","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:11:06.430123Z","iopub.execute_input":"2021-11-30T21:11:06.430442Z","iopub.status.idle":"2021-11-30T21:11:07.426596Z","shell.execute_reply.started":"2021-11-30T21:11:06.430413Z","shell.execute_reply":"2021-11-30T21:11:07.425211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nmodel = tf.keras.Sequential([\n    tf.keras.layers.InputLayer(input_shape = (X_train_rnn.shape[1], X_train_rnn.shape[2])),\n    #tf.keras.layers.GRU(10, return_sequences = True),\n    #tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.GRU(10),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(1)\n])\n\nmodel.compile(loss = 'mean_absolute_error', optimizer = 'adam')\nhistory = model.fit(X_train_rnn, y_train_rnn, epochs = 3, validation_data = (X_test_rnn, y_test_rnn))\nplt.plot(history.history['loss'], label = 'training')\nplt.plot(history.history['val_loss'], label = 'test')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:11:09.990571Z","iopub.execute_input":"2021-11-30T21:11:09.990845Z","iopub.status.idle":"2021-11-30T21:19:11.595446Z","shell.execute_reply.started":"2021-11-30T21:11:09.990816Z","shell.execute_reply":"2021-11-30T21:19:11.594058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1, 2, figsize = (12,8))\n\naxs[0].scatter(model.predict(X_train_rnn).flatten(), y_train_rnn)\naxs[1].scatter(model.predict(X_test_rnn).flatten(), y_test_rnn)\nplt.show()\n\nprint(np.corrcoef(model.predict(X_train_rnn).flatten(), y_train_rnn)[0, 1])\nprint(np.corrcoef(model.predict(X_test_rnn).flatten(), y_test_rnn)[0, 1])","metadata":{"execution":{"iopub.status.busy":"2021-11-30T21:19:18.722665Z","iopub.execute_input":"2021-11-30T21:19:18.722919Z","iopub.status.idle":"2021-11-30T21:22:20.482132Z","shell.execute_reply.started":"2021-11-30T21:19:18.722893Z","shell.execute_reply":"2021-11-30T21:22:20.481415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The result improved, but this seems to be related to the ability of our model of making a few good predictions of negative price developments. As stated before, it is typical for correlation to be exposed to a few extreme realizations.\n\nAnyway, if you see any mistakes or room for improvement, let me know! Cheers!","metadata":{}}]}