{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom datetime import datetime\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-13T17:10:16.277307Z","iopub.execute_input":"2022-01-13T17:10:16.2779Z","iopub.status.idle":"2022-01-13T17:10:16.303364Z","shell.execute_reply.started":"2022-01-13T17:10:16.277733Z","shell.execute_reply":"2022-01-13T17:10:16.302354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"../input/g-research-crypto-forecasting/\"\n!ls $data_folder","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:27:02.459973Z","iopub.execute_input":"2022-01-13T17:27:02.460387Z","iopub.status.idle":"2022-01-13T17:27:03.234971Z","shell.execute_reply.started":"2022-01-13T17:27:02.460349Z","shell.execute_reply":"2022-01-13T17:27:03.233918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df = pd.read_csv(data_folder + 'train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:27:18.770641Z","iopub.execute_input":"2022-01-13T17:27:18.770995Z","iopub.status.idle":"2022-01-13T17:28:23.803618Z","shell.execute_reply.started":"2022-01-13T17:27:18.770958Z","shell.execute_reply":"2022-01-13T17:28:23.802396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:12.483133Z","iopub.execute_input":"2022-01-13T17:35:12.483595Z","iopub.status.idle":"2022-01-13T17:35:12.507092Z","shell.execute_reply.started":"2022-01-13T17:35:12.483541Z","shell.execute_reply":"2022-01-13T17:35:12.505507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details = pd.read_csv(data_folder + 'asset_details.csv')\nasset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:16.802076Z","iopub.execute_input":"2022-01-13T17:35:16.803161Z","iopub.status.idle":"2022-01-13T17:35:16.822333Z","shell.execute_reply.started":"2022-01-13T17:35:16.803103Z","shell.execute_reply":"2022-01-13T17:35:16.82144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc = crypto_df[crypto_df[\"Asset_ID\"]==1].set_index(\"timestamp\") # Asset_ID = 1 for Bitcoin\nbtc_mini = btc.iloc[-200:] # Select recent data rows","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:19.520053Z","iopub.execute_input":"2022-01-13T17:35:19.520633Z","iopub.status.idle":"2022-01-13T17:35:20.074603Z","shell.execute_reply.started":"2022-01-13T17:35:19.520558Z","shell.execute_reply":"2022-01-13T17:35:20.073056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\n\nfig = go.Figure(data=[go.Candlestick(x=btc_mini.index, open=btc_mini['Open'], high=btc_mini['High'], low=btc_mini['Low'], close=btc_mini['Close'])])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:22.726959Z","iopub.execute_input":"2022-01-13T17:35:22.727742Z","iopub.status.idle":"2022-01-13T17:35:22.743207Z","shell.execute_reply.started":"2022-01-13T17:35:22.727701Z","shell.execute_reply":"2022-01-13T17:35:22.742339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth = crypto_df[crypto_df[\"Asset_ID\"]==6].set_index(\"timestamp\") # Asset_ID = 6 for Ethereum\neth.info(show_counts =True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:25.590096Z","iopub.execute_input":"2022-01-13T17:35:25.590844Z","iopub.status.idle":"2022-01-13T17:35:26.171828Z","shell.execute_reply.started":"2022-01-13T17:35:25.590797Z","shell.execute_reply":"2022-01-13T17:35:26.170692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:29.022577Z","iopub.execute_input":"2022-01-13T17:35:29.022933Z","iopub.status.idle":"2022-01-13T17:35:29.077758Z","shell.execute_reply.started":"2022-01-13T17:35:29.022899Z","shell.execute_reply":"2022-01-13T17:35:29.076562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:31.258683Z","iopub.execute_input":"2022-01-13T17:35:31.259007Z","iopub.status.idle":"2022-01-13T17:35:31.277925Z","shell.execute_reply.started":"2022-01-13T17:35:31.258975Z","shell.execute_reply":"2022-01-13T17:35:31.276861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"beg_btc = btc.index[0].astype('datetime64[s]')\nend_btc = btc.index[-1].astype('datetime64[s]')\nbeg_eth = eth.index[0].astype('datetime64[s]')\nend_eth = eth.index[-1].astype('datetime64[s]')\n\nprint('BTC data goes from ', beg_btc, 'to ', end_btc)\nprint('Ethereum data goes from ', beg_eth, 'to ', end_eth)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:33.51092Z","iopub.execute_input":"2022-01-13T17:35:33.511257Z","iopub.status.idle":"2022-01-13T17:35:33.519866Z","shell.execute_reply.started":"2022-01-13T17:35:33.511225Z","shell.execute_reply":"2022-01-13T17:35:33.518552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(eth.index[1:]-eth.index[:-1]).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:36.322826Z","iopub.execute_input":"2022-01-13T17:35:36.323175Z","iopub.status.idle":"2022-01-13T17:35:36.353741Z","shell.execute_reply.started":"2022-01-13T17:35:36.32314Z","shell.execute_reply":"2022-01-13T17:35:36.352878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth = eth.reindex(range(eth.index[0],eth.index[-1]+60,60),method='pad')","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:38.698394Z","iopub.execute_input":"2022-01-13T17:35:38.698721Z","iopub.status.idle":"2022-01-13T17:35:39.231142Z","shell.execute_reply.started":"2022-01-13T17:35:38.69869Z","shell.execute_reply":"2022-01-13T17:35:39.229866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(eth.index[1:]-eth.index[:-1]).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:41.142323Z","iopub.execute_input":"2022-01-13T17:35:41.143312Z","iopub.status.idle":"2022-01-13T17:35:41.183029Z","shell.execute_reply.started":"2022-01-13T17:35:41.143253Z","shell.execute_reply":"2022-01-13T17:35:41.182039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# plot vwap time series for both chosen assets\nf = plt.figure(figsize=(15,4))\n\n# fill missing values for BTC\nbtc = btc.reindex(range(btc.index[0],btc.index[-1]+60,60),method='pad')\n\nax = f.add_subplot(121)\nplt.plot(btc['Close'], label='BTC')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Bitcoin')\n\nax2 = f.add_subplot(122)\nax2.plot(eth['Close'], color='red', label='ETH')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Ethereum')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:35:43.174182Z","iopub.execute_input":"2022-01-13T17:35:43.175083Z","iopub.status.idle":"2022-01-13T17:35:46.534066Z","shell.execute_reply.started":"2022-01-13T17:35:43.17503Z","shell.execute_reply":"2022-01-13T17:35:46.533114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\n\n# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\n# create intervals\nbtc_mini_2021 = btc.loc[totimestamp('01/06/2021'):totimestamp('01/07/2021')]\neth_mini_2021 = eth.loc[totimestamp('01/06/2021'):totimestamp('01/07/2021')]","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:44:40.505993Z","iopub.execute_input":"2022-01-13T17:44:40.506318Z","iopub.status.idle":"2022-01-13T17:44:40.512404Z","shell.execute_reply.started":"2022-01-13T17:44:40.506283Z","shell.execute_reply":"2022-01-13T17:44:40.511758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot time series for both chosen assets\nf = plt.figure(figsize=(7,8))\n\nax = f.add_subplot(211)\nplt.plot(btc_mini_2021['Close'], label='btc')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Bitcoin Close')\n\nax2 = f.add_subplot(212)\nax2.plot(eth_mini_2021['Close'], color='red', label='eth')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Ethereum Close')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:45:02.419255Z","iopub.execute_input":"2022-01-13T17:45:02.419727Z","iopub.status.idle":"2022-01-13T17:45:02.897689Z","shell.execute_reply.started":"2022-01-13T17:45:02.419696Z","shell.execute_reply":"2022-01-13T17:45:02.896639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:45:17.406237Z","iopub.execute_input":"2022-01-13T17:45:17.406609Z","iopub.status.idle":"2022-01-13T17:45:17.412305Z","shell.execute_reply.started":"2022-01-13T17:45:17.406571Z","shell.execute_reply":"2022-01-13T17:45:17.411038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scipy.stats as stats\n\nlret_btc = log_return(btc_mini_2021.Close)[1:]\nlret_eth = log_return(eth_mini_2021.Close)[1:]\nlret_btc.rename('lret_btc', inplace=True)\nlret_eth.rename('lret_eth', inplace=True)\n\nplt.figure(figsize=(8,4))\nplt.plot(lret_btc);\nplt.plot(lret_eth);\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:45:27.123546Z","iopub.execute_input":"2022-01-13T17:45:27.123958Z","iopub.status.idle":"2022-01-13T17:45:28.144454Z","shell.execute_reply.started":"2022-01-13T17:45:27.123927Z","shell.execute_reply":"2022-01-13T17:45:28.143179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join two asset in single DataFrame\n\nlret_btc_long = log_return(btc.Close)[1:]\nlret_eth_long = log_return(eth.Close)[1:]\nlret_btc_long.rename('lret_btc', inplace=True)\nlret_eth_long.rename('lret_eth', inplace=True)\ntwo_assets = pd.concat([lret_btc_long, lret_eth_long], axis=1)\n\n# group consecutive rows and use .corr() for correlation between columns\ncorr_time = two_assets.groupby(two_assets.index//(10000*60)).corr().loc[:,\"lret_btc\"].loc[:,\"lret_eth\"]\n\ncorr_time.plot();\nplt.xticks([])\nplt.ylabel(\"Correlation\")\nplt.title(\"Correlation between BTC and ETH over time\");","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:45:39.427214Z","iopub.execute_input":"2022-01-13T17:45:39.427609Z","iopub.status.idle":"2022-01-13T17:45:39.97858Z","shell.execute_reply.started":"2022-01-13T17:45:39.427574Z","shell.execute_reply":"2022-01-13T17:45:39.977413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create dataframe with returns for all assets\nall_assets_2021 = pd.DataFrame([])\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n  asset = crypto_df[crypto_df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n  asset = asset.loc[totimestamp('01/01/2021'):totimestamp('01/05/2021')]\n  asset = asset.reindex(range(asset.index[0],asset.index[-1]+60,60),method='pad')\n  lret = log_return(asset.Close.fillna(0))[1:]\n  all_assets_2021 = all_assets_2021.join(lret, rsuffix=asset_name, how=\"outer\")","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:45:53.779077Z","iopub.execute_input":"2022-01-13T17:45:53.780051Z","iopub.status.idle":"2022-01-13T17:45:58.52301Z","shell.execute_reply.started":"2022-01-13T17:45:53.780006Z","shell.execute_reply":"2022-01-13T17:45:58.522086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details.Asset_ID, asset_details.Asset_Name","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:46:10.807497Z","iopub.execute_input":"2022-01-13T17:46:10.807919Z","iopub.status.idle":"2022-01-13T17:46:10.817118Z","shell.execute_reply.started":"2022-01-13T17:46:10.807877Z","shell.execute_reply":"2022-01-13T17:46:10.816471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(all_assets_2021.corr());\nplt.yticks(range(14), asset_details.Asset_Name.values);\nplt.xticks(range(14), asset_details.Asset_Name.values, rotation='vertical');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:46:23.175756Z","iopub.execute_input":"2022-01-13T17:46:23.176338Z","iopub.status.idle":"2022-01-13T17:46:23.689391Z","shell.execute_reply.started":"2022-01-13T17:46:23.176263Z","shell.execute_reply":"2022-01-13T17:46:23.688265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select some input features from the trading data: \n# 5 min log return, abs(5 min log return), upper shadow, and lower shadow.\nupper_shadow = lambda asset: asset.High - np.maximum(asset.Close,asset.Open)\nlower_shadow = lambda asset: np.minimum(asset.Close,asset.Open)- asset.Low\n\nX_btc = pd.concat([log_return(btc.VWAP,periods=5), log_return(btc.VWAP,periods=1).abs(), \n               upper_shadow(btc), lower_shadow(btc)], axis=1)\ny_btc = btc.Target\n\nX_eth = pd.concat([log_return(eth.VWAP,periods=5), log_return(eth.VWAP,periods=1).abs(), \n               upper_shadow(eth), lower_shadow(eth)], axis=1)\ny_eth = eth.Target","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:46:38.479626Z","iopub.execute_input":"2022-01-13T17:46:38.479976Z","iopub.status.idle":"2022-01-13T17:46:38.786611Z","shell.execute_reply.started":"2022-01-13T17:46:38.479943Z","shell.execute_reply":"2022-01-13T17:46:38.785693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select training and test periods\ntrain_window = [totimestamp(\"01/05/2021\"), totimestamp(\"30/05/2021\")]\ntest_window = [totimestamp(\"01/06/2021\"), totimestamp(\"30/06/2021\")]\n\n# divide data into train and test, compute X and y\n# we aim to build simple regression models using a window_size of 1\nX_btc_train = X_btc.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  # filling NaN's with zeros\ny_btc_train = y_btc.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \n\nX_btc_test = X_btc.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() \ny_btc_test = y_btc.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() \n\nX_eth_train = X_eth.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \ny_eth_train = y_eth.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \n\nX_eth_test = X_eth.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() \ny_eth_test = y_eth.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() ","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:46:49.890527Z","iopub.execute_input":"2022-01-13T17:46:49.890869Z","iopub.status.idle":"2022-01-13T17:46:49.910303Z","shell.execute_reply.started":"2022-01-13T17:46:49.890837Z","shell.execute_reply":"2022-01-13T17:46:49.909391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n# simple preprocessing of the data \nscaler = StandardScaler()\n\nX_btc_train_scaled = scaler.fit_transform(X_btc_train)\nX_btc_test_scaled = scaler.transform(X_btc_test)\n\nX_eth_train_scaled = scaler.fit_transform(X_eth_train)\nX_eth_test_scaled = scaler.transform(X_eth_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:47:01.547471Z","iopub.execute_input":"2022-01-13T17:47:01.547935Z","iopub.status.idle":"2022-01-13T17:47:01.690668Z","shell.execute_reply.started":"2022-01-13T17:47:01.547901Z","shell.execute_reply":"2022-01-13T17:47:01.689713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\n# implement basic ML baseline (one per asset)\nlr = LinearRegression()\nlr.fit(X_btc_train_scaled,y_btc_train)\ny_pred_lr_btc = lr.predict(X_btc_test_scaled)\n\nlr.fit(X_eth_train_scaled,y_eth_train)\ny_pred_lr_eth = lr.predict(X_eth_test_scaled)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:47:11.299439Z","iopub.execute_input":"2022-01-13T17:47:11.299812Z","iopub.status.idle":"2022-01-13T17:47:11.672498Z","shell.execute_reply.started":"2022-01-13T17:47:11.299774Z","shell.execute_reply":"2022-01-13T17:47:11.671599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# implement more complex baseline (multiple input/output regression model)\nfrom sklearn.multioutput import MultiOutputRegressor\n\n# we concatenate X and y for both assets\nX_both_train = np.concatenate((X_btc_train_scaled, X_eth_train_scaled), axis=1)\nX_both_test = np.concatenate((X_btc_test_scaled, X_eth_test_scaled), axis=1)\ny_both_train = np.column_stack((y_btc_train, y_eth_train))\ny_both_test = np.column_stack((y_btc_test, y_eth_test))\n\n# define the direct multioutput model and fit it\nmlr = MultiOutputRegressor(LinearRegression())\nmlr.fit(X_both_train,y_both_train)\ny_pred_lr_both = mlr.predict(X_both_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:47:21.258323Z","iopub.execute_input":"2022-01-13T17:47:21.258677Z","iopub.status.idle":"2022-01-13T17:47:21.295247Z","shell.execute_reply.started":"2022-01-13T17:47:21.258639Z","shell.execute_reply":"2022-01-13T17:47:21.294169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Test score for LR baseline: BTC', f\"{np.corrcoef(y_pred_lr_btc, y_btc_test)[0,1]:.2f}\", \n                                ', ETH', f\"{np.corrcoef(y_pred_lr_eth, y_eth_test)[0,1]:.2f}\")\nprint('Test score for multiple output LR baseline: BTC', f\"{np.corrcoef(y_pred_lr_both[:,0], y_btc_test)[0,1]:.2f}\", \n                                                ', ETH', f\"{np.corrcoef(y_pred_lr_both[:,1], y_eth_test)[0,1]:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-13T17:47:31.643582Z","iopub.execute_input":"2022-01-13T17:47:31.643936Z","iopub.status.idle":"2022-01-13T17:47:31.658546Z","shell.execute_reply.started":"2022-01-13T17:47:31.643899Z","shell.execute_reply":"2022-01-13T17:47:31.657479Z"},"trusted":true},"execution_count":null,"outputs":[]}]}