{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datetime import datetime\n\ndata_folder = \"../input/g-research-crypto-forecasting/\"\n\n#Check the train set\ncrypto_df = pd.read_csv(data_folder + 'train.csv')\ncrypto_df.head(10)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-10T14:09:59.126771Z","iopub.execute_input":"2022-01-10T14:09:59.127301Z","iopub.status.idle":"2022-01-10T14:11:00.140976Z","shell.execute_reply.started":"2022-01-10T14:09:59.127206Z","shell.execute_reply":"2022-01-10T14:11:00.140198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#View the asset information, including the list of all assets and the weight of each asset used to weigh their relative importance in the evaluation metric\nasset_details = pd.read_csv(data_folder + 'asset_details.csv')\nasset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:00.142753Z","iopub.execute_input":"2022-01-10T14:11:00.143210Z","iopub.status.idle":"2022-01-10T14:11:00.164689Z","shell.execute_reply.started":"2022-01-10T14:11:00.143169Z","shell.execute_reply":"2022-01-10T14:11:00.163836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Construct candlestick chart for slice of data for Bitcoin\nbtc = crypto_df[crypto_df[\"Asset_ID\"]==1].set_index(\"timestamp\") # Asset_ID = 1 for Bitcoin\nbtc_mini = btc.iloc[-200:] # Select recent data rows\nimport plotly.graph_objects as go\n\nfig = go.Figure(data=[go.Candlestick(x=btc_mini.index, open=btc_mini['Open'], high=btc_mini['High'], low=btc_mini['Low'], close=btc_mini['Close'])])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:00.165794Z","iopub.execute_input":"2022-01-10T14:11:00.166089Z","iopub.status.idle":"2022-01-10T14:11:00.598692Z","shell.execute_reply.started":"2022-01-10T14:11:00.166059Z","shell.execute_reply":"2022-01-10T14:11:00.597907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Inspect data for ethereum\n\neth = crypto_df[crypto_df[\"Asset_ID\"]==6].set_index(\"timestamp\") # Asset_ID = 6 for Ethereum\neth.info(show_counts =True)\n\n# Results suggest missing data for target","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:00.599886Z","iopub.execute_input":"2022-01-10T14:11:00.600201Z","iopub.status.idle":"2022-01-10T14:11:00.947568Z","shell.execute_reply.started":"2022-01-10T14:11:00.600170Z","shell.execute_reply":"2022-01-10T14:11:00.946738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's check the time range for Bitcoin and Ethereum data, using the coversion from timestamp to datetime\nbeg_btc = btc.index[0].astype('datetime64[s]')\nend_btc = btc.index[-1].astype('datetime64[s]')\nbeg_eth = eth.index[0].astype('datetime64[s]')\nend_eth = eth.index[-1].astype('datetime64[s]')\n\nprint('BTC data goes from ', beg_btc, 'to ', end_btc)\nprint('Ethereum data goes from ', beg_eth, 'to ', end_eth)\n\n#Missing asset data, for a given minute, is not represented by NaN's, but instead by the absence of those rows.\n#We can check the timestamp difference between consecutive rows to see if there is missing data.","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:00.949669Z","iopub.execute_input":"2022-01-10T14:11:00.949905Z","iopub.status.idle":"2022-01-10T14:11:00.958409Z","shell.execute_reply.started":"2022-01-10T14:11:00.949875Z","shell.execute_reply":"2022-01-10T14:11:00.957632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(eth.index[1:]-eth.index[:-1]).value_counts().head()\n\n#Notice that there are many gaps in the data. \n#We should preprocess our data into a format without time gaps. ","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:00.959546Z","iopub.execute_input":"2022-01-10T14:11:00.959776Z","iopub.status.idle":"2022-01-10T14:11:00.988499Z","shell.execute_reply.started":"2022-01-10T14:11:00.959740Z","shell.execute_reply":"2022-01-10T14:11:00.987644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Padding to fix the time gaps issue\neth = eth.reindex(range(eth.index[0],eth.index[-1]+60,60),method='pad')\n\n#Check that there are no time gaps now\n(eth.index[1:]-eth.index[:-1]).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:00.989622Z","iopub.execute_input":"2022-01-10T14:11:00.990215Z","iopub.status.idle":"2022-01-10T14:11:01.334291Z","shell.execute_reply.started":"2022-01-10T14:11:00.990183Z","shell.execute_reply":"2022-01-10T14:11:01.333447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Visualising the Close prices for the two assets we have selected\n\nimport matplotlib.pyplot as plt\n\n# plot vwap time series for both chosen assets\nf = plt.figure(figsize=(15,4))\n\n# fill missing values for BTC\nbtc = btc.reindex(range(btc.index[0],btc.index[-1]+60,60),method='pad')\n\nax = f.add_subplot(121)\nplt.plot(btc['Close'], label='BTC')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Bitcoin')\n\nax2 = f.add_subplot(122)\nax2.plot(eth['Close'], color='red', label='ETH')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Ethereum')\n\nplt.tight_layout()\nplt.show()\n\n#The assets have quite different history, but we could check if they correlate in recent times.","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:01.335404Z","iopub.execute_input":"2022-01-10T14:11:01.335638Z","iopub.status.idle":"2022-01-10T14:11:04.542650Z","shell.execute_reply.started":"2022-01-10T14:11:01.335610Z","shell.execute_reply":"2022-01-10T14:11:04.541885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\n# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\n# create intervals\nbtc_mini_2021 = btc.loc[totimestamp('01/06/2021'):totimestamp('01/07/2021')]\neth_mini_2021 = eth.loc[totimestamp('01/06/2021'):totimestamp('01/07/2021')]\n\n# plot time series for both chosen assets\nf = plt.figure(figsize=(7,8))\n\nax = f.add_subplot(211)\nplt.plot(btc_mini_2021['Close'], label='btc')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Bitcoin Close')\n\nax2 = f.add_subplot(212)\nax2.plot(eth_mini_2021['Close'], color='red', label='eth')\nplt.legend()\nplt.xlabel('Time')\nplt.ylabel('Ethereum Close')\n\nplt.tight_layout()\nplt.show()\n\n#On shorter intervals we can visually see some potential correlation between both assets, with some simultaneous ups and downs. \n#A better format for analyzing such movements is by calculating asset returns.","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:04.543917Z","iopub.execute_input":"2022-01-10T14:11:04.544735Z","iopub.status.idle":"2022-01-10T14:11:05.001725Z","shell.execute_reply.started":"2022-01-10T14:11:04.544692Z","shell.execute_reply":"2022-01-10T14:11:05.000895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)\n\n#visualize the log return for our two assets\nimport scipy.stats as stats\n\nlret_btc = log_return(btc_mini_2021.Close)[1:]\nlret_eth = log_return(eth_mini_2021.Close)[1:]\nlret_btc.rename('lret_btc', inplace=True)\nlret_eth.rename('lret_eth', inplace=True)\n\nplt.figure(figsize=(8,4))\nplt.plot(lret_btc);\nplt.plot(lret_eth);\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:05.002932Z","iopub.execute_input":"2022-01-10T14:11:05.003190Z","iopub.status.idle":"2022-01-10T14:11:05.978026Z","shell.execute_reply.started":"2022-01-10T14:11:05.003161Z","shell.execute_reply":"2022-01-10T14:11:05.977459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Investigate correlation between bitcoin and ethereum\n# join two asset in single DataFrame\n\nlret_btc_long = log_return(btc.Close)[1:]\nlret_eth_long = log_return(eth.Close)[1:]\nlret_btc_long.rename('lret_btc', inplace=True)\nlret_eth_long.rename('lret_eth', inplace=True)\ntwo_assets = pd.concat([lret_btc_long, lret_eth_long], axis=1)\n\n# group consecutive rows and use .corr() for correlation between columns\ncorr_time = two_assets.groupby(two_assets.index//(10000*60)).corr().loc[:,\"lret_btc\"].loc[:,\"lret_eth\"]\n\ncorr_time.plot();\nplt.xticks([])\nplt.ylabel(\"Correlation\")\nplt.title(\"Correlation between BTC and ETH over time\");\n\n#Note the high but variable correlation between the assets. \n#There is some changing dynamics over time. Need to perform forecasts in a highly non-stationary environment.","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:05.979317Z","iopub.execute_input":"2022-01-10T14:11:05.980012Z","iopub.status.idle":"2022-01-10T14:11:06.450456Z","shell.execute_reply.started":"2022-01-10T14:11:05.979977Z","shell.execute_reply":"2022-01-10T14:11:06.449656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Check the correlation between all assets visualizing the correlation matrix. \n#Note how some assets have much higher pairwise correlation than others\n\n# Create dataframe with returns for all assets\nall_assets_2021_returns = pd.DataFrame([])\n\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    asset = crypto_df[crypto_df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n    asset = asset.loc[totimestamp('01/01/2021'):totimestamp('21/09/2021')]\n    asset = asset.reindex(range(asset.index[0],asset.index[-1]+60,60),method='pad')\n    lret = log_return(asset.Close.fillna(0))[1:]\n    all_assets_2021_returns = all_assets_2021_returns.join(lret, rsuffix=asset_name, how=\"outer\")\n\nplt.imshow(all_assets_2021_returns.corr());\nplt.yticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values);\nplt.xticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values, rotation='vertical');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:06.451562Z","iopub.execute_input":"2022-01-10T14:11:06.451766Z","iopub.status.idle":"2022-01-10T14:11:11.771284Z","shell.execute_reply.started":"2022-01-10T14:11:06.451740Z","shell.execute_reply":"2022-01-10T14:11:11.770409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Set timestamp and Asset ID as indexes and drop rows for targets with missing values\n\ncrypto_df=crypto_df.set_index(['timestamp','Asset_ID']).sort_index()\ncrypto_df_2021 = crypto_df.loc[totimestamp('01/01/2021'):totimestamp('21/09/2021')]\ncrypto_df_2021_clean=crypto_df_2021.dropna(axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:11.772797Z","iopub.execute_input":"2022-01-10T14:11:11.773233Z","iopub.status.idle":"2022-01-10T14:11:21.955820Z","shell.execute_reply.started":"2022-01-10T14:11:11.773190Z","shell.execute_reply":"2022-01-10T14:11:21.954940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select some input features from the trading data: \n# 5 min log return, abs(5 min log return), upper shadow, and lower shadow.\nupper_shadow = lambda asset: asset.High - np.maximum(asset.Close,asset.Open)\nlower_shadow = lambda asset: np.minimum(asset.Close,asset.Open)- asset.Low\n\nX = pd.concat([log_return(crypto_df_2021_clean.VWAP,periods=5), log_return(crypto_df_2021_clean.VWAP,periods=1).abs(), \n               upper_shadow(crypto_df_2021_clean), lower_shadow(crypto_df_2021_clean)], axis=1)\ny = crypto_df_2021_clean.Target","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:21.958593Z","iopub.execute_input":"2022-01-10T14:11:21.958824Z","iopub.status.idle":"2022-01-10T14:11:22.955734Z","shell.execute_reply.started":"2022-01-10T14:11:21.958793Z","shell.execute_reply":"2022-01-10T14:11:22.954810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select training and test periods\ntrain_window = [totimestamp(\"01/01/2021\"), totimestamp(\"31/08/2021\")]\ntest_window = [totimestamp(\"01/09/2021\"), totimestamp(\"21/09/2021\")]\n\n# divide data into train and test, compute X and y\n# we aim to build simple regression models using a window_size of 1\nX_train = X.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  # filling NaN's with zeros\ny_train = y.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \nX_test = X.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() \ny_test = y.loc[test_window[0]:test_window[1]].fillna(0).to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:22.956800Z","iopub.execute_input":"2022-01-10T14:11:22.956988Z","iopub.status.idle":"2022-01-10T14:11:23.244226Z","shell.execute_reply.started":"2022-01-10T14:11:22.956963Z","shell.execute_reply":"2022-01-10T14:11:23.243089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Standardise input data\n\nfrom sklearn.preprocessing import StandardScaler\n# Simple preprocessing of the selected assets\nscaler = StandardScaler()\n\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:23.247948Z","iopub.execute_input":"2022-01-10T14:11:23.248196Z","iopub.status.idle":"2022-01-10T14:11:23.817532Z","shell.execute_reply.started":"2022-01-10T14:11:23.248165Z","shell.execute_reply":"2022-01-10T14:11:23.816537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Model the data\n\nfrom xgboost import XGBRegressor\nmodel=XGBRegressor()\nmodel.fit(X_train,y_train)\ny_pred=model.predict(X_test)\n\n#The competition performance metric is weighted correlation. \n#However, for now we will use simple correlation to evaluate the two baseline models built.\nprint(np.corrcoef(y_pred, y_test)[0,1])","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:11:23.818919Z","iopub.execute_input":"2022-01-10T14:11:23.819307Z","iopub.status.idle":"2022-01-10T14:16:59.724046Z","shell.execute_reply.started":"2022-01-10T14:11:23.819261Z","shell.execute_reply":"2022-01-10T14:16:59.723107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()   # initialize the environment","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:16:59.726055Z","iopub.execute_input":"2022-01-10T14:16:59.726517Z","iopub.status.idle":"2022-01-10T14:16:59.755045Z","shell.execute_reply.started":"2022-01-10T14:16:59.726470Z","shell.execute_reply":"2022-01-10T14:16:59.754173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df.set_index(['timestamp','Asset_ID']).sort_index()\n    X_test_submission = pd.concat([log_return(test_df.VWAP,periods=5), log_return(test_df.VWAP,periods=1).abs(), \n               upper_shadow(test_df), lower_shadow(test_df)], axis=1)\n    X_test_submission = X_test_submission.fillna(0).to_numpy()\n    X_test_submission = scaler.transform(X_test_submission)\n    sample_prediction_df['Target'] = model.predict(X_test_submission )\n    env.predict(sample_prediction_df)   # register your predictions","metadata":{"execution":{"iopub.status.busy":"2022-01-10T14:16:59.756775Z","iopub.execute_input":"2022-01-10T14:16:59.757080Z","iopub.status.idle":"2022-01-10T14:16:59.834582Z","shell.execute_reply.started":"2022-01-10T14:16:59.757039Z","shell.execute_reply":"2022-01-10T14:16:59.833874Z"},"trusted":true},"execution_count":null,"outputs":[]}]}