{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-26T13:33:01.149601Z","iopub.execute_input":"2022-01-26T13:33:01.150034Z","iopub.status.idle":"2022-01-26T13:33:01.171661Z","shell.execute_reply.started":"2022-01-26T13:33:01.149911Z","shell.execute_reply":"2022-01-26T13:33:01.170818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"/kaggle/input/g-research-crypto-forecasting/\"","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:01.174392Z","iopub.execute_input":"2022-01-26T13:33:01.174723Z","iopub.status.idle":"2022-01-26T13:33:01.179992Z","shell.execute_reply.started":"2022-01-26T13:33:01.174678Z","shell.execute_reply":"2022-01-26T13:33:01.179088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df = pd.read_csv(data_folder + 'train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:01.181397Z","iopub.execute_input":"2022-01-26T13:33:01.181884Z","iopub.status.idle":"2022-01-26T13:33:33.355170Z","shell.execute_reply.started":"2022-01-26T13:33:01.181846Z","shell.execute_reply":"2022-01-26T13:33:33.354261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:33.356467Z","iopub.execute_input":"2022-01-26T13:33:33.357828Z","iopub.status.idle":"2022-01-26T13:33:33.382685Z","shell.execute_reply.started":"2022-01-26T13:33:33.357785Z","shell.execute_reply":"2022-01-26T13:33:33.381749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis\n\nThe test data is contained in the original train data , so the LB score of 0.313 is overfitting.\n\nFor more information:\n* __[Watch out!: test LB period is contained in the train csv](https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/285505) (topic)__\n* __[G-Research- Using the overlap fully [LB=0.99]](https://www.kaggle.com/julian3833/g-research-using-the-overlap-fully-lb-0-99) (notebook)__\n* __[Meaningful submission scores / sharing the lower boundary of public test data](https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/285289) (topic)__\n\n\nSo we need to check and exclude the test data from the train data","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(data_folder + 'example_test.csv')\ntest_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:33.385693Z","iopub.execute_input":"2022-01-26T13:33:33.386009Z","iopub.status.idle":"2022-01-26T13:33:33.411189Z","shell.execute_reply.started":"2022-01-26T13:33:33.385966Z","shell.execute_reply":"2022-01-26T13:33:33.410399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Before excluding test data\nprint (\"ACTUAL TRAIN DATA\")\n\nprint (\"Time stamp the train data set ends to: \"+ str(pd.to_datetime(crypto_df['timestamp'], unit='s').min()))\nprint (\"Time stamp the train data set ends to: \"+ str(pd.to_datetime(crypto_df['timestamp'], unit='s').max()), end='\\n\\n')\n\nprint (\"TEST DATA\")\n\nprint (\"Time stamp the test data set starts from: \"+ str(pd.to_datetime(test_df['timestamp'], unit='s').min()))\nprint (\"Time stamp the test data set ends to: \"+ str(pd.to_datetime(test_df['timestamp'], unit='s').max()), end='\\n\\n')","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:33.412434Z","iopub.execute_input":"2022-01-26T13:33:33.412746Z","iopub.status.idle":"2022-01-26T13:33:35.922296Z","shell.execute_reply.started":"2022-01-26T13:33:33.412704Z","shell.execute_reply":"2022-01-26T13:33:35.921585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now this symbolize the test data is part of train data set.\nSo we need to exclude data from train dataset on after 2021-06-13 00:00:00","metadata":{}},{"cell_type":"code","source":"if True:\n    crypto_df_train = crypto_df[crypto_df['timestamp'] < test_df['timestamp'].min()]","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:35.923438Z","iopub.execute_input":"2022-01-26T13:33:35.924046Z","iopub.status.idle":"2022-01-26T13:33:36.800254Z","shell.execute_reply.started":"2022-01-26T13:33:35.924006Z","shell.execute_reply":"2022-01-26T13:33:36.799502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (\"ACTUAL TRAIN DATA\")\nprint (\"Time stamp the actual train data set starts from: \"+ str(pd.to_datetime(crypto_df['timestamp'], unit='s').min()))\nprint (\"Time stamp the actual train data set ends to: \"+ str(pd.to_datetime(crypto_df['timestamp'], unit='s').max()), end='\\n\\n')\nprint (\"DERIVED TRAIN DATA\")\nprint (\"Time stamp the derived train data set starts from: \"+ str(pd.to_datetime(crypto_df_train['timestamp'], unit='s').min()))\nprint (\"Time stamp the derived train data set ends to: \"+ str(pd.to_datetime(crypto_df_train['timestamp'], unit='s').max()), end = '\\n\\n')\nprint (\"TEST DATA\")\nprint (\"Time stamp the test data set starts from: \"+ str(pd.to_datetime(test_df['timestamp'], unit='s').min()))\nprint (\"Time stamp the test data set ends to: \"+ str(pd.to_datetime(test_df['timestamp'], unit='s').max()), end = '\\n\\n')\n","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:36.801511Z","iopub.execute_input":"2022-01-26T13:33:36.801786Z","iopub.status.idle":"2022-01-26T13:33:41.900205Z","shell.execute_reply.started":"2022-01-26T13:33:36.801751Z","shell.execute_reply":"2022-01-26T13:33:41.899500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom datetime import datetime\nfrom matplotlib.dates import date2num","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:41.903099Z","iopub.execute_input":"2022-01-26T13:33:41.903297Z","iopub.status.idle":"2022-01-26T13:33:41.907904Z","shell.execute_reply.started":"2022-01-26T13:33:41.903273Z","shell.execute_reply":"2022-01-26T13:33:41.907099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfig, ax = plt.subplots(figsize=(30,1))\n\nax.plot(datetime(2021,6,12),1)\nax.axvspan(date2num(datetime(2021,6,12)), date2num(datetime(2021,6,14)), \n           label=\"ACTUAL TRAIN DATA\",color=\"yellow\", alpha=0.5)\n\nax.axvspan(date2num(datetime(2021,6,12)), date2num(pd.to_datetime(crypto_df_train['timestamp'], unit='s').max()), \n           label=\"DERIVED TRAIN DATA\",color=\"green\", alpha=0.3)\n\nax.axvspan(date2num(pd.to_datetime(test_df['timestamp'], unit='s').min()), date2num(pd.to_datetime(test_df['timestamp'], unit='s').max()), \n           label=\"TEST DATA\",color=\"red\", alpha=1.0)\n\nax.legend()\n\nax.set_title('Data Distribution Over Time', size=18)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:41.908949Z","iopub.execute_input":"2022-01-26T13:33:41.909882Z","iopub.status.idle":"2022-01-26T13:33:43.441143Z","shell.execute_reply.started":"2022-01-26T13:33:41.909820Z","shell.execute_reply":"2022-01-26T13:33:43.440314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The **RED** line in the above graph is the test data and if we would have considered the whole train dataset it will lead to overfit.","metadata":{"execution":{"iopub.status.busy":"2022-01-14T15:23:33.470433Z","iopub.execute_input":"2022-01-14T15:23:33.471506Z","iopub.status.idle":"2022-01-14T15:23:35.208052Z","shell.execute_reply.started":"2022-01-14T15:23:33.471449Z","shell.execute_reply":"2022-01-14T15:23:35.207089Z"}}},{"cell_type":"markdown","source":"# Data features\nWe can see the different features included in the dataset. Specifically, the features included per asset are the following:\n*   **timestamp**: All timestamps are returned as second Unix timestamps (the number of seconds elapsed since 1970-01-01 00:00:00.000 UTC). Timestamps in this dataset are multiple of 60, indicating minute-by-minute data.\n*   **Asset_ID**: The asset ID corresponding to one of the crytocurrencies (e.g. `Asset_ID = 1` for Bitcoin). The mapping from `Asset_ID` to crypto asset is contained in `asset_details.csv`.\n*   **Count**: Total number of trades in the time interval (last minute).\n*   **Open**:\tOpening price of the time interval (in USD).\n*   **High**:\tHighest price reached during time interval (in USD).\n*   **Low**: Lowest price reached during time interval (in USD).\n*   **Close**:\tClosing price of the time interval (in USD).\n*   **Volume**:\tQuantity of asset bought or sold, displayed in base currency USD.\n*   **VWAP**: The average price of the asset over the time interval, weighted by volume. VWAP is an aggregated form of trade data.\n*   **Target**: Residual log-returns for the asset over a 15 minute horizon. \n\nThe first two columns define the time and asset indexes for this data row. The 6 middle columns are feature columns with the trading data for this asset and minute in time. The last column is the prediction target, which we will get to later in more detail.\n\nWe also view the asset information, including the list of all assets, the `Asset_ID` to asset mapping, and the weight of each asset used to weigh their relative importance in the evaluation metric.","metadata":{}},{"cell_type":"code","source":"asset_details = pd.read_csv(data_folder + 'asset_details.csv')\nasset_details.sort_values(by=['Asset_ID'])","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:43.442447Z","iopub.execute_input":"2022-01-26T13:33:43.442839Z","iopub.status.idle":"2022-01-26T13:33:43.459660Z","shell.execute_reply.started":"2022-01-26T13:33:43.442792Z","shell.execute_reply":"2022-01-26T13:33:43.458835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dealing with missing data\n\nLet us inspect the data for another important asset, Ethereum","metadata":{}},{"cell_type":"code","source":"#Example with Ethereum data\neth = crypto_df_train[crypto_df_train[\"Asset_ID\"]==6].set_index(\"timestamp\") # Asset_ID = 6 for Ethereum\neth.info(show_counts =True)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:43.461178Z","iopub.execute_input":"2022-01-26T13:33:43.461893Z","iopub.status.idle":"2022-01-26T13:33:43.857792Z","shell.execute_reply.started":"2022-01-26T13:33:43.461849Z","shell.execute_reply":"2022-01-26T13:33:43.857003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see the number of rows in the training set, and that there are missing values for the targets columns, which we will address later. Let's confirm that:","metadata":{}},{"cell_type":"code","source":"eth.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:43.859077Z","iopub.execute_input":"2022-01-26T13:33:43.859514Z","iopub.status.idle":"2022-01-26T13:33:43.897317Z","shell.execute_reply.started":"2022-01-26T13:33:43.859474Z","shell.execute_reply":"2022-01-26T13:33:43.896438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"beg_eth = eth.index[0].astype('datetime64[s]')\nend_eth = eth.index[-1].astype('datetime64[s]')\n\nprint('Ethereum data goes from ', beg_eth, 'to ', end_eth)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:43.901710Z","iopub.execute_input":"2022-01-26T13:33:43.901920Z","iopub.status.idle":"2022-01-26T13:33:43.907348Z","shell.execute_reply.started":"2022-01-26T13:33:43.901894Z","shell.execute_reply":"2022-01-26T13:33:43.906612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Missing asset data, for a given minute, is not represented by NaN's, but instead by the absence of those rows. We can check the timestamp difference between consecutive rows to see if there is missing data.","metadata":{}},{"cell_type":"code","source":"(eth.index[1:]-eth.index[:-1]).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:43.909057Z","iopub.execute_input":"2022-01-26T13:33:43.909663Z","iopub.status.idle":"2022-01-26T13:33:43.941219Z","shell.execute_reply.started":"2022-01-26T13:33:43.909621Z","shell.execute_reply":"2022-01-26T13:33:43.940402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Notice that there are many gaps in the data. To work with most time series models, we should preprocess our data into a format without time gaps. To fill the gaps, we can use the .reindex() method for forward filling, filling gaps with the previous valid value.","metadata":{"execution":{"iopub.status.busy":"2022-01-14T16:08:26.521828Z","iopub.execute_input":"2022-01-14T16:08:26.52276Z","iopub.status.idle":"2022-01-14T16:08:26.52836Z","shell.execute_reply.started":"2022-01-14T16:08:26.522713Z","shell.execute_reply":"2022-01-14T16:08:26.527528Z"}}},{"cell_type":"code","source":"eth = eth.reindex(range(eth.index[0],eth.index[-1]+60,60),method='pad')","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:43.942653Z","iopub.execute_input":"2022-01-26T13:33:43.942922Z","iopub.status.idle":"2022-01-26T13:33:44.190539Z","shell.execute_reply.started":"2022-01-26T13:33:43.942884Z","shell.execute_reply":"2022-01-26T13:33:44.189790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(eth.index[1:]-eth.index[:-1]).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:44.191829Z","iopub.execute_input":"2022-01-26T13:33:44.192097Z","iopub.status.idle":"2022-01-26T13:33:44.223366Z","shell.execute_reply.started":"2022-01-26T13:33:44.192051Z","shell.execute_reply":"2022-01-26T13:33:44.222639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth.info(show_counts =True)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:44.224361Z","iopub.execute_input":"2022-01-26T13:33:44.226609Z","iopub.status.idle":"2022-01-26T13:33:44.270467Z","shell.execute_reply.started":"2022-01-26T13:33:44.226566Z","shell.execute_reply":"2022-01-26T13:33:44.269523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:44.272156Z","iopub.execute_input":"2022-01-26T13:33:44.272463Z","iopub.status.idle":"2022-01-26T13:33:44.311009Z","shell.execute_reply.started":"2022-01-26T13:33:44.272408Z","shell.execute_reply":"2022-01-26T13:33:44.310056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Candlestick charts\n\nThe trading data format is an aggregated form of market data including for Open, High, Low and Close. We can visualize this data through the commonly used candlestick bar chart, which allows traders to perform technical analysis on intraday values. The bar's body length represents the price range between the open and close of that day's trading. When the bar is red, it means the close was lower than the open, and green otherwise. These are also referred to as bullish and bearish candlesticks. The wicks above and below the bars show the high and low prices of that interval's trading.\n\nWe can visualize a slice of the Bitcoin prices using the `plotly` library. The bottom part of the plot shows a rangeslider, which you can use to zoom in the plot.","metadata":{}},{"cell_type":"code","source":"import plotly.graph_objects as go\neth_mini = eth.iloc[-200:] # Select recent data rows\nfig = go.Figure(data=[go.Candlestick(x=eth_mini.index, open=eth_mini['Open'], high=eth_mini['High'], low=eth_mini['Low'], close=eth_mini['Close'])])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:44.312691Z","iopub.execute_input":"2022-01-26T13:33:44.312980Z","iopub.status.idle":"2022-01-26T13:33:44.360196Z","shell.execute_reply.started":"2022-01-26T13:33:44.312939Z","shell.execute_reply":"2022-01-26T13:33:44.359372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Log returns\n\nIn order to analyze price changes for an asset we can deal with the price difference. However, different assets exhibit different price scales, so that the their returns are not readily comparable. We can solve this problem by computing the percentage change in price instead, also known as the return. This return coincides with the percentage change in our invested capital.\n\nReturns are widely used in finance, however log returns are preferred for mathematical modelling of time series, as they are additive across time. Also, while regular returns cannot go below -100%, log returns are not bounded.\n\nTo compute the log return, we can simply take the logarithm of the ratio between two consecutive prices. The first row will have an empty return as the previous value is unknown, therefore the empty return data point will be dropped.","metadata":{}},{"cell_type":"code","source":"# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:44.361578Z","iopub.execute_input":"2022-01-26T13:33:44.361873Z","iopub.status.idle":"2022-01-26T13:33:44.366370Z","shell.execute_reply.started":"2022-01-26T13:33:44.361808Z","shell.execute_reply":"2022-01-26T13:33:44.365436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example with Ethereum data\nimport scipy.stats as stats\n\nlret_eth = log_return(eth_mini.Close)[1:]\n\nlret_eth.rename('lret_eth', inplace=True)\n\nplt.figure(figsize=(8,4))\nplt.plot(lret_eth);\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:44.367915Z","iopub.execute_input":"2022-01-26T13:33:44.368435Z","iopub.status.idle":"2022-01-26T13:33:44.783463Z","shell.execute_reply.started":"2022-01-26T13:33:44.368395Z","shell.execute_reply":"2022-01-26T13:33:44.782756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation between assets","metadata":{}},{"cell_type":"code","source":"# create dataframe with returns for all assets\nall_assets = pd.DataFrame([])\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n    asset = crypto_df_train[crypto_df_train[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n    asset = asset.reindex(range(asset.index[0],asset.index[-1]+60,60),method='pad')\n    lret = log_return(asset.Close.fillna(0))[1:]\n    all_assets = all_assets.join(lret, rsuffix=asset_name, how=\"outer\")","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:44.784586Z","iopub.execute_input":"2022-01-26T13:33:44.785169Z","iopub.status.idle":"2022-01-26T13:33:53.732756Z","shell.execute_reply.started":"2022-01-26T13:33:44.785127Z","shell.execute_reply":"2022-01-26T13:33:53.731945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(all_assets.corr());\nplt.yticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values);\nplt.xticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values, rotation='vertical');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:53.734151Z","iopub.execute_input":"2022-01-26T13:33:53.734585Z","iopub.status.idle":"2022-01-26T13:33:55.082981Z","shell.execute_reply.started":"2022-01-26T13:33:53.734537Z","shell.execute_reply":"2022-01-26T13:33:55.082288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This forecasting competition aims to predict returns in the near future for prices $P^a$, for each asset $a$. For each row in the dataset, we include the target for prediction, `Target`. `Target` is derived from log returns ($R^a$) over 15 minutes.\n\n$$R^a(t) = log (P^a(t+16)\\ /\\ P^a(t+1))$$\n\nCrypto asset returns are highly correlated, following to a large extend the overall crypto market. As we want to test your ability to predict returns for individual assets, we perform a linear residualization, removing the market signal from individual asset returns when creating the target. In more detail, if $M(t)$ is the weighted average market returns, the target is:\n\n$$M(t) = \\frac{\\sum_a w^a R^a(t)}{\\sum_a w^a}  \\\\\n\\beta^a = \\frac{\\langle M \\cdot R^a \\rangle}{\\langle M^2 \\rangle} \\\\\n\\text{Target}^a(t) = R^a(t) - \\beta^a M(t)$$\n\nwhere the bracket $\\langle .\\rangle$ represent the rolling average over time (3750 minute windows), and same asset weights $w^a$ used for the evaluation metric.\n\nSome rows have null values for targets due to missing values in future prices. Rows with nulls in the test set ground truth are ignored for scoring purposes.\n\nIn the competition, your predictions will be evaluated on a weighted version of the Pearson correlation coefficient, with weights given by the `Weight` column in the Asset Details file.\n\nIn this tutorial, we will simplify things and use correlation (without weights) for evaluation, and consider only two assets, BTC and ETH.","metadata":{"execution":{"iopub.status.busy":"2022-01-15T05:59:16.503049Z","iopub.execute_input":"2022-01-15T05:59:16.503388Z","iopub.status.idle":"2022-01-15T05:59:16.517551Z","shell.execute_reply.started":"2022-01-15T05:59:16.503352Z","shell.execute_reply":"2022-01-15T05:59:16.516366Z"}}},{"cell_type":"code","source":"btc = crypto_df_train[crypto_df_train[\"Asset_ID\"]==1].set_index(\"timestamp\")\nbtc = asset.reindex(range(asset.index[0],asset.index[-1]+60,60),method='pad')","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.084405Z","iopub.execute_input":"2022-01-26T13:33:55.084846Z","iopub.status.idle":"2022-01-26T13:33:55.367407Z","shell.execute_reply.started":"2022-01-26T13:33:55.084807Z","shell.execute_reply":"2022-01-26T13:33:55.366635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.368673Z","iopub.execute_input":"2022-01-26T13:33:55.368923Z","iopub.status.idle":"2022-01-26T13:33:55.386826Z","shell.execute_reply.started":"2022-01-26T13:33:55.368890Z","shell.execute_reply":"2022-01-26T13:33:55.385801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select some input features from the trading data: \n# 5 min log return, abs(5 min log return), upper shadow, and lower shadow.\nupper_shadow = lambda asset: asset.High - np.maximum(asset.Close,asset.Open)\nlower_shadow = lambda asset: np.minimum(asset.Close,asset.Open)- asset.Low\n\nX_btc = pd.concat([log_return(btc.VWAP,periods=5), log_return(btc.VWAP,periods=1).abs(), \n               upper_shadow(btc), lower_shadow(btc)], axis=1)\ny_btc = btc.Target","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.388198Z","iopub.execute_input":"2022-01-26T13:33:55.389060Z","iopub.status.idle":"2022-01-26T13:33:55.445071Z","shell.execute_reply.started":"2022-01-26T13:33:55.389010Z","shell.execute_reply":"2022-01-26T13:33:55.444241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\n# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\n# select training and test periods\ntrain_window = [totimestamp(\"01/01/2018\"), totimestamp(\"12/05/2021\")]\ntest_window = [totimestamp(\"13/05/2021\"), totimestamp(\"12/06/2021\")]\n\n# divide data into train and test, compute X and y\n# we aim to build simple regression models using a window_size of 1\nX_btc_train = X_btc.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  # filling NaN's with zeros\ny_btc_train = y_btc.loc[train_window[0]:train_window[1]].fillna(0).to_numpy()  \n\nX_btc_test = X_btc.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() \ny_btc_test = y_btc.loc[test_window[0]:test_window[1]].fillna(0).to_numpy() ","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.446240Z","iopub.execute_input":"2022-01-26T13:33:55.446601Z","iopub.status.idle":"2022-01-26T13:33:55.480364Z","shell.execute_reply.started":"2022-01-26T13:33:55.446558Z","shell.execute_reply":"2022-01-26T13:33:55.479607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We now standardize the input data. Standardization is the process of putting different variables on the same scale. In regression analysis, it is often crucial to standardize your independent variables or you may risk obtaining misleading results.","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n# simple preprocessing of the data \nscaler = StandardScaler()\n\nX_btc_train_scaled = scaler.fit_transform(X_btc_train)\nX_btc_test_scaled = scaler.transform(X_btc_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.481688Z","iopub.execute_input":"2022-01-26T13:33:55.481928Z","iopub.status.idle":"2022-01-26T13:33:55.591825Z","shell.execute_reply.started":"2022-01-26T13:33:55.481894Z","shell.execute_reply":"2022-01-26T13:33:55.590981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baseline model: Linear Regression\n\nWe will try a simple Linear Regression model on the features we designed. Note that Linear Regression is not commonly used in time series analysis, specially with only one time step! \n\nWe compare two Linear Regression baselines, one that considers each asset independently and one multiple inputs that models all assets together.","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\n# implement basic ML baseline (one per asset)\nlr = LinearRegression()\nlr.fit(X_btc_train_scaled,y_btc_train)\ny_pred_lr_btc = lr.predict(X_btc_test_scaled)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.593346Z","iopub.execute_input":"2022-01-26T13:33:55.593831Z","iopub.status.idle":"2022-01-26T13:33:55.733463Z","shell.execute_reply.started":"2022-01-26T13:33:55.593789Z","shell.execute_reply":"2022-01-26T13:33:55.732374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Test score for LR baseline: BTC', f\"{np.corrcoef(y_pred_lr_btc, y_btc_test)[0,1]:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.735219Z","iopub.execute_input":"2022-01-26T13:33:55.735538Z","iopub.status.idle":"2022-01-26T13:33:55.744832Z","shell.execute_reply.started":"2022-01-26T13:33:55.735483Z","shell.execute_reply":"2022-01-26T13:33:55.743624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We shall forecast three months (30 days) worth of data, based on the 180 days prior to the start of the forecasting period","metadata":{"execution":{"iopub.status.busy":"2022-01-19T15:27:44.964567Z","iopub.execute_input":"2022-01-19T15:27:44.964919Z","iopub.status.idle":"2022-01-19T15:27:44.972011Z","shell.execute_reply.started":"2022-01-19T15:27:44.964885Z","shell.execute_reply":"2022-01-19T15:27:44.970286Z"}}},{"cell_type":"markdown","source":"Defining Class errors having all the evaluation metrics","metadata":{}},{"cell_type":"code","source":"test_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.748205Z","iopub.execute_input":"2022-01-26T13:33:55.755469Z","iopub.status.idle":"2022-01-26T13:33:55.785330Z","shell.execute_reply.started":"2022-01-26T13:33:55.755416Z","shell.execute_reply":"2022-01-26T13:33:55.784491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM pipeline\n\nRef:\n\n**Credits:**\nThe following notebook is heavily based on the following notebooks. If you find it useful, spare some upvotes to the originals. They earned it!\n\n* __[G-Research: LGBM pipeline Notebook](https://www.kaggle.com/julian3833/g-research-starter-lgbm-pipeline) (notebook)__","metadata":{}},{"cell_type":"markdown","source":"## Training with LGBM","metadata":{}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nimport gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:55.790053Z","iopub.execute_input":"2022-01-26T13:33:55.790662Z","iopub.status.idle":"2022-01-26T13:33:56.794264Z","shell.execute_reply.started":"2022-01-26T13:33:55.790593Z","shell.execute_reply":"2022-01-26T13:33:56.793505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hyperparameter Tuning\n\nPost Hyperparameter Tuning in Google colab with more resource I have got the below best parameter to train :\n\n- {'**learning_rate**': 0.04945819653484207, \n    '**boosting_type**': 'dart', \n    'objective': 'regression', \n    'metric': 'mae', \n    'sub_feature': 0.12567008013847558, \n    '**num_leaves**': 160, \n    'min_data': 93, \n    '**max_depth**': 136} ","metadata":{}},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    return df_feat.fillna(0)\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"].fillna(0)\n    \n    # TODO: Try different models here!\n    model = LGBMRegressor(\n    boosting_type = 'dart',\n    num_leaves = 160,\n    max_depth = 136,\n    learning_rate = 0.04945819653484207,\n    n_estimators = 100)\n    model.fit(X, y)\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:56.795599Z","iopub.execute_input":"2022-01-26T13:33:56.795849Z","iopub.status.idle":"2022-01-26T13:33:56.804906Z","shell.execute_reply.started":"2022-01-26T13:33:56.795815Z","shell.execute_reply":"2022-01-26T13:33:56.804224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df_train['datetime'] = pd.to_datetime(crypto_df_train['timestamp'], unit='s')\ncrypto_df_train = crypto_df_train.set_index('datetime')\ncrypto_df_train = crypto_df_train[(crypto_df_train.index.year == 2021) & (crypto_df_train.index.month > 5)]\ncrypto_df_train = crypto_df_train.set_index('timestamp')","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:33:56.806112Z","iopub.execute_input":"2022-01-26T13:33:56.806806Z","iopub.status.idle":"2022-01-26T13:34:02.572734Z","shell.execute_reply.started":"2022-01-26T13:33:56.806766Z","shell.execute_reply":"2022-01-26T13:34:02.571816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df_train","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:34:02.577055Z","iopub.execute_input":"2022-01-26T13:34:02.577832Z","iopub.status.idle":"2022-01-26T13:34:02.609342Z","shell.execute_reply.started":"2022-01-26T13:34:02.577788Z","shell.execute_reply":"2022-01-26T13:34:02.608514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(asset_details['Asset_ID'], asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(crypto_df_train, asset_id)    \n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:34:02.613762Z","iopub.execute_input":"2022-01-26T13:34:02.615892Z","iopub.status.idle":"2022-01-26T13:34:20.757659Z","shell.execute_reply.started":"2022-01-26T13:34:02.615849Z","shell.execute_reply":"2022-01-26T13:34:20.756987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df_train.iloc[1]","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:34:20.761377Z","iopub.execute_input":"2022-01-26T13:34:20.762945Z","iopub.status.idle":"2022-01-26T13:34:20.771286Z","shell.execute_reply.started":"2022-01-26T13:34:20.762910Z","shell.execute_reply":"2022-01-26T13:34:20.770438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model interface\nx = get_features(crypto_df_train.iloc[1])\ny_pred = models[0].predict([x])\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:34:20.773098Z","iopub.execute_input":"2022-01-26T13:34:20.773517Z","iopub.status.idle":"2022-01-26T13:34:20.786423Z","shell.execute_reply.started":"2022-01-26T13:34:20.773476Z","shell.execute_reply":"2022-01-26T13:34:20.785518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df_test = []\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        model = models[row['Asset_ID']]\n        x_test = get_features(row)\n        y_pred = model.predict([x_test])[0]\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        \n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n    all_df_test.append(df_test)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T13:34:20.788143Z","iopub.execute_input":"2022-01-26T13:34:20.788699Z","iopub.status.idle":"2022-01-26T13:34:20.972371Z","shell.execute_reply.started":"2022-01-26T13:34:20.788659Z","shell.execute_reply":"2022-01-26T13:34:20.971601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}