{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:39:16.0185Z","iopub.execute_input":"2021-11-03T07:39:16.01876Z","iopub.status.idle":"2021-11-03T07:39:16.022744Z","shell.execute_reply.started":"2021-11-03T07:39:16.018723Z","shell.execute_reply":"2021-11-03T07:39:16.022178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"../input/g-research-crypto-forecasting/\"\n!ls $data_folder","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:39:16.023755Z","iopub.execute_input":"2021-11-03T07:39:16.024521Z","iopub.status.idle":"2021-11-03T07:39:16.790441Z","shell.execute_reply.started":"2021-11-03T07:39:16.024484Z","shell.execute_reply":"2021-11-03T07:39:16.789232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df = pd.read_csv(data_folder + 'train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:39:41.393012Z","iopub.execute_input":"2021-11-03T07:39:41.393338Z","iopub.status.idle":"2021-11-03T07:40:36.416544Z","shell.execute_reply.started":"2021-11-03T07:39:41.393303Z","shell.execute_reply":"2021-11-03T07:40:36.415736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:40:48.607307Z","iopub.execute_input":"2021-11-03T07:40:48.607958Z","iopub.status.idle":"2021-11-03T07:40:48.641202Z","shell.execute_reply.started":"2021-11-03T07:40:48.607912Z","shell.execute_reply":"2021-11-03T07:40:48.640223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details = pd.read_csv(data_folder + 'asset_details.csv')\nasset_details","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:41:10.407895Z","iopub.execute_input":"2021-11-03T07:41:10.408193Z","iopub.status.idle":"2021-11-03T07:41:10.427614Z","shell.execute_reply.started":"2021-11-03T07:41:10.40816Z","shell.execute_reply":"2021-11-03T07:41:10.426767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc = crypto_df[crypto_df[\"Asset_ID\"]==1].set_index(\"timestamp\") # Asset_ID = 1 for Bitcoin\nbtc_mini = btc.iloc[-200:] # Select recent data rows","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:41:31.426196Z","iopub.execute_input":"2021-11-03T07:41:31.426485Z","iopub.status.idle":"2021-11-03T07:41:31.814774Z","shell.execute_reply.started":"2021-11-03T07:41:31.426457Z","shell.execute_reply":"2021-11-03T07:41:31.813831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\n\nfig = go.Figure(data=[go.Candlestick(x=btc_mini.index, open=btc_mini['Open'], high=btc_mini['High'], low=btc_mini['Low'], close=btc_mini['Close'])])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:41:54.997135Z","iopub.execute_input":"2021-11-03T07:41:54.997403Z","iopub.status.idle":"2021-11-03T07:41:55.131824Z","shell.execute_reply.started":"2021-11-03T07:41:54.997374Z","shell.execute_reply":"2021-11-03T07:41:55.131011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth = crypto_df[crypto_df[\"Asset_ID\"]==6].set_index(\"timestamp\") # Asset_ID = 6 for Ethereum\neth.info(show_counts =True)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:42:25.638756Z","iopub.execute_input":"2021-11-03T07:42:25.639046Z","iopub.status.idle":"2021-11-03T07:42:26.015748Z","shell.execute_reply.started":"2021-11-03T07:42:25.639017Z","shell.execute_reply":"2021-11-03T07:42:26.014879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:42:42.979709Z","iopub.execute_input":"2021-11-03T07:42:42.980057Z","iopub.status.idle":"2021-11-03T07:42:43.02083Z","shell.execute_reply.started":"2021-11-03T07:42:42.980022Z","shell.execute_reply":"2021-11-03T07:42:43.019947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:42:55.286566Z","iopub.execute_input":"2021-11-03T07:42:55.286894Z","iopub.status.idle":"2021-11-03T07:42:55.303538Z","shell.execute_reply.started":"2021-11-03T07:42:55.286862Z","shell.execute_reply":"2021-11-03T07:42:55.30258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"beg_btc = btc.index[0].astype('datetime64[s]')\nend_btc = btc.index[-1].astype('datetime64[s]')\nbeg_eth = eth.index[0].astype('datetime64[s]')\nend_eth = eth.index[-1].astype('datetime64[s]')\n\nprint('BTC data goes from ', beg_btc, 'to ', end_btc)\nprint('Ethereum data goes from ', beg_eth, 'to ', end_eth)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:43:10.565604Z","iopub.execute_input":"2021-11-03T07:43:10.566185Z","iopub.status.idle":"2021-11-03T07:43:10.573427Z","shell.execute_reply.started":"2021-11-03T07:43:10.566144Z","shell.execute_reply":"2021-11-03T07:43:10.572596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(eth.index[1:]-eth.index[:-1]).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:43:24.568409Z","iopub.execute_input":"2021-11-03T07:43:24.568731Z","iopub.status.idle":"2021-11-03T07:43:24.596325Z","shell.execute_reply.started":"2021-11-03T07:43:24.568696Z","shell.execute_reply":"2021-11-03T07:43:24.595565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth = eth.reindex(range(eth.index[0],eth.index[-1]+60,60),method='pad')","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:43:41.289095Z","iopub.execute_input":"2021-11-03T07:43:41.290097Z","iopub.status.idle":"2021-11-03T07:43:41.675964Z","shell.execute_reply.started":"2021-11-03T07:43:41.290017Z","shell.execute_reply":"2021-11-03T07:43:41.675187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(eth.index[1:]-eth.index[:-1]).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:43:53.069087Z","iopub.execute_input":"2021-11-03T07:43:53.069362Z","iopub.status.idle":"2021-11-03T07:43:53.114863Z","shell.execute_reply.started":"2021-11-03T07:43:53.069334Z","shell.execute_reply":"2021-11-03T07:43:53.114079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# plot vwap time series for both chosen assets\nf = plt.figure(figsize=(15,4))\n\n# fill missing values for BTC\nbtc = btc.reindex(range(btc.index[0],btc.index[-1]+60,60),method='pad')\n\nax = f.add_subplot(121)\nplt.plot(btc['Close'], label='BTC')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Bitcoin')\n\nax2 = f.add_subplot(122)\nax2.plot(eth['Close'], color='red', label='ETH')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Ethereum')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:44:14.744006Z","iopub.execute_input":"2021-11-03T07:44:14.744313Z","iopub.status.idle":"2021-11-03T07:44:18.043815Z","shell.execute_reply.started":"2021-11-03T07:44:14.74428Z","shell.execute_reply":"2021-11-03T07:44:18.043005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\n# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\n# create intervals\nbtc_mini_2021 = btc.loc[totimestamp('01/06/2021'):totimestamp('01/07/2021')]\neth_mini_2021 = eth.loc[totimestamp('01/06/2021'):totimestamp('01/07/2021')]","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:44:33.622176Z","iopub.execute_input":"2021-11-03T07:44:33.622466Z","iopub.status.idle":"2021-11-03T07:44:33.629618Z","shell.execute_reply.started":"2021-11-03T07:44:33.622437Z","shell.execute_reply":"2021-11-03T07:44:33.628615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot time series for both chosen assets\nf = plt.figure(figsize=(7,8))\n\nax = f.add_subplot(211)\nplt.plot(btc_mini_2021['Close'], label='btc')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Bitcoin Close')\n\nax2 = f.add_subplot(212)\nax2.plot(eth_mini_2021['Close'], color='red', label='eth')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Ethereum Close')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:44:48.288093Z","iopub.execute_input":"2021-11-03T07:44:48.288528Z","iopub.status.idle":"2021-11-03T07:44:48.779851Z","shell.execute_reply.started":"2021-11-03T07:44:48.288498Z","shell.execute_reply":"2021-11-03T07:44:48.778865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:45:08.930609Z","iopub.execute_input":"2021-11-03T07:45:08.931419Z","iopub.status.idle":"2021-11-03T07:45:08.9354Z","shell.execute_reply.started":"2021-11-03T07:45:08.931378Z","shell.execute_reply":"2021-11-03T07:45:08.934766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scipy.stats as stats\n\nlret_btc = log_return(btc_mini_2021.Close)[1:]\nlret_eth = log_return(eth_mini_2021.Close)[1:]\nlret_btc.rename('lret_btc', inplace=True)\nlret_eth.rename('lret_eth', inplace=True)\n\nplt.figure(figsize=(8,4))\nplt.plot(lret_btc);\nplt.plot(lret_eth);\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:45:24.316433Z","iopub.execute_input":"2021-11-03T07:45:24.317048Z","iopub.status.idle":"2021-11-03T07:45:25.464155Z","shell.execute_reply.started":"2021-11-03T07:45:24.316996Z","shell.execute_reply":"2021-11-03T07:45:25.46328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join two asset in single DataFrame\n\nlret_btc_long = log_return(btc.Close)[1:]\nlret_eth_long = log_return(eth.Close)[1:]\nlret_btc_long.rename('lret_btc', inplace=True)\nlret_eth_long.rename('lret_eth', inplace=True)\ntwo_assets = pd.concat([lret_btc_long, lret_eth_long], axis=1)\n\n# group consecutive rows and use .corr() for correlation between columns\ncorr_time = two_assets.groupby(two_assets.index//(10000*60)).corr().loc[:,\"lret_btc\"].loc[:,\"lret_eth\"]\n\ncorr_time.plot();\nplt.xticks([])\nplt.ylabel(\"Correlation\")\nplt.title(\"Correlation between BTC and ETH over time\");","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:45:49.593365Z","iopub.execute_input":"2021-11-03T07:45:49.593642Z","iopub.status.idle":"2021-11-03T07:45:50.149836Z","shell.execute_reply.started":"2021-11-03T07:45:49.593613Z","shell.execute_reply":"2021-11-03T07:45:50.148985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create dataframe with returns for all assets\nall_assets_2021 = pd.DataFrame([])\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n  asset = crypto_df[crypto_df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n  asset = asset.loc[totimestamp('01/01/2021'):totimestamp('01/05/2021')]\n  asset = asset.reindex(range(asset.index[0],asset.index[-1]+60,60),method='pad')\n  lret = log_return(asset.Close.fillna(0))[1:]\n  all_assets_2021 = all_assets_2021.join(lret, rsuffix=asset_name, how=\"outer\")","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:46:06.810474Z","iopub.execute_input":"2021-11-03T07:46:06.811241Z","iopub.status.idle":"2021-11-03T07:46:11.217514Z","shell.execute_reply.started":"2021-11-03T07:46:06.8112Z","shell.execute_reply":"2021-11-03T07:46:11.216547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(all_assets_2021.corr());\nplt.yticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values);\nplt.xticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values, rotation='vertical');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:46:22.57548Z","iopub.execute_input":"2021-11-03T07:46:22.576223Z","iopub.status.idle":"2021-11-03T07:46:23.085398Z","shell.execute_reply.started":"2021-11-03T07:46:22.576178Z","shell.execute_reply":"2021-11-03T07:46:23.084832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select some input features from the trading data: \n# 5 min log return, abs(5 min log return), upper shadow, and lower shadow.\nupper_shadow = lambda asset: asset.High - np.maximum(asset.Close,asset.Open)\nlower_shadow = lambda asset: np.minimum(asset.Close,asset.Open)- asset.Low\n\nX_btc = pd.concat([log_return(btc.VWAP,periods=5), log_return(btc.VWAP,periods=1).abs(), \n               upper_shadow(btc), lower_shadow(btc)], axis=1)\ny_btc = btc.Target\n\nX_eth = pd.concat([log_return(eth.VWAP,periods=5), log_return(eth.VWAP,periods=1).abs(), \n               upper_shadow(eth), lower_shadow(eth)], axis=1)\ny_eth = eth.Target","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:46:49.692941Z","iopub.execute_input":"2021-11-03T07:46:49.693397Z","iopub.status.idle":"2021-11-03T07:46:50.034362Z","shell.execute_reply.started":"2021-11-03T07:46:49.693354Z","shell.execute_reply":"2021-11-03T07:46:50.033347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select training and test periods\ntrain_window = [totimestamp(\"01/05/2021\"), totimestamp(\"30/05/2021\")]\ntest_window = [totimestamp(\"01/06/2021\"), totimestamp(\"30/06/2021\")]\n\n# divide data into train and test, compute X and y\n# we aim to build simple regression models using a window_size of 1\nX_btc_train = X_btc.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  # filling NaN's with zeros\ny_btc_train = y_btc.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \n\nX_btc_test = X_btc.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() \ny_btc_test = y_btc.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() \n\nX_eth_train = X_eth.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \ny_eth_train = y_eth.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \n\nX_eth_test = X_eth.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() \ny_eth_test = y_eth.loc[test_window[0]:test_window[1]].fillna(0).to_numpy()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:47:13.468498Z","iopub.execute_input":"2021-11-03T07:47:13.469471Z","iopub.status.idle":"2021-11-03T07:47:13.484402Z","shell.execute_reply.started":"2021-11-03T07:47:13.469373Z","shell.execute_reply":"2021-11-03T07:47:13.483374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n# simple preprocessing of the data \nscaler = StandardScaler()\n\nX_btc_train_scaled = scaler.fit_transform(X_btc_train)\nX_btc_test_scaled = scaler.transform(X_btc_test)\n\nX_eth_train_scaled = scaler.fit_transform(X_eth_train)\nX_eth_test_scaled = scaler.transform(X_eth_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:47:36.660676Z","iopub.execute_input":"2021-11-03T07:47:36.662206Z","iopub.status.idle":"2021-11-03T07:47:36.833232Z","shell.execute_reply.started":"2021-11-03T07:47:36.662129Z","shell.execute_reply":"2021-11-03T07:47:36.832351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\n# implement basic ML baseline (one per asset)\nlr = LinearRegression()\nlr.fit(X_btc_train_scaled,y_btc_train)\ny_pred_lr_btc = lr.predict(X_btc_test_scaled)\n\nlr.fit(X_eth_train_scaled,y_eth_train)\ny_pred_lr_eth = lr.predict(X_eth_test_scaled)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:47:50.830651Z","iopub.execute_input":"2021-11-03T07:47:50.831526Z","iopub.status.idle":"2021-11-03T07:47:50.992562Z","shell.execute_reply.started":"2021-11-03T07:47:50.831489Z","shell.execute_reply":"2021-11-03T07:47:50.991552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# implement more complex baseline (multiple output regression model)\nfrom sklearn.multioutput import MultiOutputRegressor\n\n# we concatenate X and y for both assets\nX_both_train = np.concatenate((X_btc_train_scaled, X_eth_train_scaled), axis=1)\nX_both_test = np.concatenate((X_btc_test_scaled, X_eth_test_scaled), axis=1)\ny_both_train = np.column_stack((y_btc_train, y_eth_train))\ny_both_test = np.column_stack((y_btc_test, y_eth_test))\n\n# define the direct multioutput model and fit it\nmlr = MultiOutputRegressor(LinearRegression())\nlr.fit(X_both_train,y_both_train)\ny_pred_lr_both = lr.predict(X_both_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:48:04.486441Z","iopub.execute_input":"2021-11-03T07:48:04.486727Z","iopub.status.idle":"2021-11-03T07:48:04.51874Z","shell.execute_reply.started":"2021-11-03T07:48:04.486695Z","shell.execute_reply":"2021-11-03T07:48:04.517391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Test score for LR baseline: BTC', f\"{np.corrcoef(y_pred_lr_btc, y_btc_test)[0,1]:.2f}\", \n                                ', ETH', f\"{np.corrcoef(y_pred_lr_eth, y_eth_test)[0,1]:.2f}\")\nprint('Test score for multiple output LR baseline: BTC', f\"{np.corrcoef(y_pred_lr_both[:,0], y_btc_test)[0,1]:.2f}\", \n                                                ', ETH', f\"{np.corrcoef(y_pred_lr_both[:,1], y_eth_test)[0,1]:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-03T07:48:34.140875Z","iopub.execute_input":"2021-11-03T07:48:34.141193Z","iopub.status.idle":"2021-11-03T07:48:34.152457Z","shell.execute_reply.started":"2021-11-03T07:48:34.141152Z","shell.execute_reply":"2021-11-03T07:48:34.151874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}