{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Dependencies","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import gc\nimport warnings","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:45:00.562359Z","iopub.execute_input":"2021-11-24T00:45:00.563031Z","iopub.status.idle":"2021-11-24T00:45:00.589755Z","shell.execute_reply.started":"2021-11-24T00:45:00.562922Z","shell.execute_reply":"2021-11-24T00:45:00.589018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.offline import plot, iplot, init_notebook_mode\ninit_notebook_mode(connected=True)\nimport tensorflow as tf\nimport statsmodels.api as sm\nfrom pylab import rcParams\nfrom statsmodels.tsa.stattools import adfuller\nfrom statsmodels.graphics.tsaplots import plot_acf, plot_pacf\nimport time\nimport datetime","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:45:00.591534Z","iopub.execute_input":"2021-11-24T00:45:00.592030Z","iopub.status.idle":"2021-11-24T00:45:07.868685Z","shell.execute_reply.started":"2021-11-24T00:45:00.591990Z","shell.execute_reply":"2021-11-24T00:45:07.867889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collect_gc():\n    i = 0\n    while i <= 6:\n        print('collecting garbage...')\n        print(gc.collect())\n        i = i + 1\n        \nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:45:07.870013Z","iopub.execute_input":"2021-11-24T00:45:07.870742Z","iopub.status.idle":"2021-11-24T00:45:07.875570Z","shell.execute_reply.started":"2021-11-24T00:45:07.870709Z","shell.execute_reply":"2021-11-24T00:45:07.874820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading the Data","metadata":{}},{"cell_type":"code","source":"folder = '../input/g-research-crypto-forecasting/'\n\n# Train data\nbase_train = pd.read_csv(folder+'train.csv', low_memory=False)\n\n# Assets\nasset_mapping = pd.read_csv(folder+'asset_details.csv', index_col=0)\nasset_mapping = asset_mapping.reindex([i for i in range(14)])\nasset_mapping","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-24T00:45:07.877505Z","iopub.execute_input":"2021-11-24T00:45:07.878238Z","iopub.status.idle":"2021-11-24T00:46:04.703191Z","shell.execute_reply.started":"2021-11-24T00:45:07.878187Z","shell.execute_reply":"2021-11-24T00:46:04.702381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing Data","metadata":{}},{"cell_type":"code","source":"# converting the timestamp to minutes\nbase_train['time'] = pd.to_datetime(base_train['timestamp'], unit='s')\nbase_train.head(50)\n\nbase_train.isnull().sum()","metadata":{"_kg_hide-output":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2021-11-24T00:46:04.704903Z","iopub.execute_input":"2021-11-24T00:46:04.705447Z","iopub.status.idle":"2021-11-24T00:46:06.523125Z","shell.execute_reply.started":"2021-11-24T00:46:04.705403Z","shell.execute_reply":"2021-11-24T00:46:06.522424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_coin, btc, btc_cash, card, doge, eos_io, eth, eth_class, iota, ltc, maker, mon, stela, tron = [\n    base_train.loc[base_train.Asset_ID == i] for i in range(14)]\n\nassets = [bin_coin, btc, btc_cash, card, doge, eos_io, eth, eth_class, iota, ltc, maker, mon, stela, tron]","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:06.524646Z","iopub.execute_input":"2021-11-24T00:46:06.524895Z","iopub.status.idle":"2021-11-24T00:46:10.323710Z","shell.execute_reply.started":"2021-11-24T00:46:06.524861Z","shell.execute_reply":"2021-11-24T00:46:10.322775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"--> Checking for physical Tensorflow devices\")\nfor device in tf.config.list_physical_devices():\n    print(\": {}\".format(device.name))\n\nbtc.set_index(btc.time, drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:10.324978Z","iopub.execute_input":"2021-11-24T00:46:10.325272Z","iopub.status.idle":"2021-11-24T00:46:10.392357Z","shell.execute_reply.started":"2021-11-24T00:46:10.325236Z","shell.execute_reply":"2021-11-24T00:46:10.391640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(btc.info())\nbtc_plt = btc.iloc[-8000:]\n\nwith tf.device('/GPU:0'):\n    fig = go.Figure(data=go.Ohlc(x=btc_plt.time,\n                open=btc_plt.Open,\n                high=btc_plt.High,\n                low=btc_plt.Low,\n                close=btc_plt.Close))\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:10.393804Z","iopub.execute_input":"2021-11-24T00:46:10.394273Z","iopub.status.idle":"2021-11-24T00:46:12.863759Z","shell.execute_reply.started":"2021-11-24T00:46:10.394234Z","shell.execute_reply":"2021-11-24T00:46:12.861635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handling missing Values\neth = base_train.loc[base_train['Asset_ID'] == 6]\neth.set_index(eth.time, drop=True, inplace=True)\n\nbtc.isnull().sum()\n\nbtc.fillna(method='ffill', inplace=True)\neth.fillna(method='ffill', inplace=True)\neth.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:12.865245Z","iopub.execute_input":"2021-11-24T00:46:12.865494Z","iopub.status.idle":"2021-11-24T00:46:13.421957Z","shell.execute_reply.started":"2021-11-24T00:46:12.865462Z","shell.execute_reply":"2021-11-24T00:46:13.421229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysis and Visualization","metadata":{}},{"cell_type":"markdown","source":"### Seasonality and Trend","metadata":{}},{"cell_type":"code","source":"def decompose(price):\n    rcParams['figure.figsize'] = 15, 7\n    decomp_series = sm.tsa.seasonal_decompose(price, model='addictive')\n    decomp_series.plot()\n    plt.show()\n\nfor count, asset in enumerate(assets):\n    asset2 = asset.set_index(asset.time)\n    print(f' \\033[92m \\033[1m{asset_mapping.Asset_Name[count]}\\033[1m \\033[92m ')\n    decompose(asset2.Close.resample('M').mean())\n    print('\\n');print('\\n')","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:13.424911Z","iopub.execute_input":"2021-11-24T00:46:13.425166Z","iopub.status.idle":"2021-11-24T00:46:27.391542Z","shell.execute_reply.started":"2021-11-24T00:46:13.425133Z","shell.execute_reply":"2021-11-24T00:46:27.390825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Stationarity Test","metadata":{}},{"cell_type":"code","source":"def stationarity_test(col):\n    \n    print('--------------Stationarity Test--------------')\n    adf_result = adfuller(col)\n    print('ADF Statistic:', adf_result[0])\n    print('p-value:', adf_result[1])\n    print('No. of lags used:', adf_result[2])\n    print('No. of observations used :', adf_result[3])\n    if adf_result[1] < 0.05:\n        print('TSD is Stationary')\n    else:\n        print('TSD is not Stationary')\n    print( 'Critical Values:' )\n    for k, v in adf_result[4].items():\n        print( f' {k} : {v} ' )\n    print('\\n')\n\nto_datetime = lambda s: pd.to_datetime(s, unit='s')\nfor a in assets:\n    a.index = to_datetime(a.index)\n    \nfor count, asset in enumerate(assets):\n    print(asset_mapping.Asset_Name[count])\n    stationarity_test(asset.Close.resample('D').mean())","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:27.393187Z","iopub.execute_input":"2021-11-24T00:46:27.393710Z","iopub.status.idle":"2021-11-24T00:46:28.640772Z","shell.execute_reply.started":"2021-11-24T00:46:27.393668Z","shell.execute_reply":"2021-11-24T00:46:28.639997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"collect_gc()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:28.643676Z","iopub.execute_input":"2021-11-24T00:46:28.643909Z","iopub.status.idle":"2021-11-24T00:46:29.894918Z","shell.execute_reply.started":"2021-11-24T00:46:28.643874Z","shell.execute_reply":"2021-11-24T00:46:29.894166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Price Change","metadata":{}},{"cell_type":"code","source":"def get_cmap(n, name='hsv'):\n    return plt.cm.get_cmap(name, n)\ncmap = get_cmap(len(assets))\n\ndef diff(tsd, periods=1):\n    return np.log(tsd).diff(periods=periods)\n\nfig, ax = plt.subplots(len(assets), 1, figsize=(17, 25))\nfor count, asset in enumerate(assets):\n    ax[count].plot(asset.time[1:], diff(asset.Close)[1:], c=cmap(count))\n    ax[count].set_title(asset_mapping.Asset_Name[count])\n\nplt.suptitle('Price Change \\n')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:29.897996Z","iopub.execute_input":"2021-11-24T00:46:29.898205Z","iopub.status.idle":"2021-11-24T00:46:37.058172Z","shell.execute_reply.started":"2021-11-24T00:46:29.898179Z","shell.execute_reply":"2021-11-24T00:46:37.057447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for count, asset in enumerate(assets):\n    print(asset_mapping.Asset_Name[count])\n    stationarity_test(diff(asset.Close).resample('D').mean())","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:37.059367Z","iopub.execute_input":"2021-11-24T00:46:37.059798Z","iopub.status.idle":"2021-11-24T00:46:38.083424Z","shell.execute_reply.started":"2021-11-24T00:46:37.059760Z","shell.execute_reply":"2021-11-24T00:46:38.082650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Autocorrelation and Partial Autocorrelation","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(len(assets), 2, figsize=(15, 35))\n\nfor count, asset in enumerate(assets):\n    fig = plot_acf(diff(asset.Close)[1:].resample('D').mean(), lags=20, ax=axes[count, 0])\n    fig = plot_pacf(diff(asset.Close)[1:].resample('D').mean(), lags=20, ax=axes[count, 1])\n    axes[count, 0].set_title(f'{asset_mapping.Asset_Name[count]} ACF')\n    axes[count, 1].set_title(f'{asset_mapping.Asset_Name[count]} PACF')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:38.084699Z","iopub.execute_input":"2021-11-24T00:46:38.085091Z","iopub.status.idle":"2021-11-24T00:46:46.332117Z","shell.execute_reply.started":"2021-11-24T00:46:38.085045Z","shell.execute_reply":"2021-11-24T00:46:46.331379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Checking for correlation between assets","metadata":{}},{"cell_type":"markdown","source":"#### 2018-2021","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(len(assets), 1, figsize=(20, 40))\n\nfor count, asset in enumerate(assets):\n    axes[count].plot(asset.time, asset.Close, color=cmap(count))\n    axes[count].set_title(asset_mapping.Asset_Name[count])\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:46.333432Z","iopub.execute_input":"2021-11-24T00:46:46.333963Z","iopub.status.idle":"2021-11-24T00:46:53.225179Z","shell.execute_reply.started":"2021-11-24T00:46:46.333919Z","shell.execute_reply":"2021-11-24T00:46:53.224426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### September 2021","metadata":{}},{"cell_type":"code","source":"# 1 Month Plot\nbtc_sep = btc.loc['2021-09-01 00:01:00':'2021-09-30 00:01:00']\neth_sep = eth.loc['2021-09-01 00:01:00':'2021-09-30 00:01:00']\n\nfig = plt.figure(figsize=(20, 8))\nfig.add_subplot(121)\nplt.title('Ethereum (September)')\nplt.xlabel('Time')\nplt.ylabel('Price')\nplt.plot(eth_sep.time, eth_sep.Close)\n\nfig.add_subplot(122)\nplt.title('Bitcoin (September)')\nplt.plot(btc_sep.time, btc_sep.Close, color='r')\nplt.xlabel('Time')\nplt.ylabel('Price')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:53.226539Z","iopub.execute_input":"2021-11-24T00:46:53.227414Z","iopub.status.idle":"2021-11-24T00:46:53.701446Z","shell.execute_reply.started":"2021-11-24T00:46:53.227372Z","shell.execute_reply":"2021-11-24T00:46:53.700760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"collect_gc()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:53.702526Z","iopub.execute_input":"2021-11-24T00:46:53.702949Z","iopub.status.idle":"2021-11-24T00:46:55.213194Z","shell.execute_reply.started":"2021-11-24T00:46:53.702911Z","shell.execute_reply":"2021-11-24T00:46:55.212307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc_change = diff(btc.Close)[1:].rename('Bitcoin')\neth_change = diff(eth.Close)[1:].rename('Etheruem')\nasset_corr = pd.concat([btc_change, eth_change], axis=1)\n\nto_timestamp = lambda a: [time.mktime(t.timetuple()) for t in a]\nasset_corr.index = to_timestamp(asset_corr.index)\ncorr = asset_corr.groupby(asset_corr.index//(10000*60)).corr().loc[:,\"Bitcoin\"].loc[:,\"Etheruem\"]\nrcParams['figure.figsize'] = 15, 8\ncorr.plot()\nplt.title('Bitcoin and Ethereum Correlation since 2018')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-24T00:46:55.215426Z","iopub.execute_input":"2021-11-24T00:46:55.216172Z","iopub.status.idle":"2021-11-24T00:47:12.688181Z","shell.execute_reply.started":"2021-11-24T00:46:55.216129Z","shell.execute_reply":"2021-11-24T00:47:12.687488Z"},"trusted":true},"execution_count":null,"outputs":[]}]}