{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Predicting Crypto Prices\n\n![Dodge](https://d.newsweek.com/en/full/1784128/dogecoin.jpg?w=790&f=5c22adba14c4c3d31d31d006f7a4f669)\n\nThis is a starter notebook for the G-Research Crypto forecasting competition. It was created during a live coding session on twitch. Check it out here: https://www.twitch.tv/medallionstallion_","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pylab as plt\nimport seaborn as sns\nfrom itertools import cycle\nimport datetime as dt\npd.set_option('max_columns', 50)\nplt.style.use('bmh')\ncolor_pal = plt.rcParams['axes.prop_cycle'].by_key()['color']\ncolor_cycle = cycle(plt.rcParams['axes.prop_cycle'].by_key()['color'])","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-03T01:53:21.269307Z","iopub.execute_input":"2021-11-03T01:53:21.270284Z","iopub.status.idle":"2021-11-03T01:53:22.325324Z","shell.execute_reply.started":"2021-11-03T01:53:21.270151Z","shell.execute_reply":"2021-11-03T01:53:22.324541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read in the training data\nSome things of note:\n- The training data is ~2.7G\n- We are given supplimental data about the assets we are trying to predict.","metadata":{}},{"cell_type":"code","source":"!ls -GFlash --color ../input/g-research-crypto-forecasting","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:53:47.396320Z","iopub.execute_input":"2021-11-03T01:53:47.396722Z","iopub.status.idle":"2021-11-03T01:53:48.161647Z","shell.execute_reply.started":"2021-11-03T01:53:47.396678Z","shell.execute_reply":"2021-11-03T01:53:48.160612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\nasset = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:53:48.164348Z","iopub.execute_input":"2021-11-03T01:53:48.164980Z","iopub.status.idle":"2021-11-03T01:54:48.849068Z","shell.execute_reply.started":"2021-11-03T01:53:48.164909Z","shell.execute_reply":"2021-11-03T01:54:48.848066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What are we trying to predict?\n- We are trying to predict the prices of 14 \"assets\".\n- Each asset is given a different weight in the evaluation metric.\n- We are given historic prices by the minute: the start, end, max and min price.\n- Bitcoin predictions are the most important... Marker coins are the least.","metadata":{}},{"cell_type":"code","source":"ax = asset.set_index('Asset_Name')['Weight'] \\\n    .sort_values(ascending=False) \\\n    .plot(kind='bar',\n          figsize=(12, 5),\n          color=color_pal[0]\n         )\nax.set_title('Asset Weight in Evaluation Metric', fontsize=18)\nax.set_xlabel('Asset Name')\nax.set_ylabel('Metric Weight')\nax.bar_label(ax.containers[0], fmt='%0.2f', color='white', padding=-12, fontsize=10)\nplt.xticks(rotation=45, ha='right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:57:01.237510Z","iopub.execute_input":"2021-11-03T01:57:01.237851Z","iopub.status.idle":"2021-11-03T01:57:01.687211Z","shell.execute_reply.started":"2021-11-03T01:57:01.237818Z","shell.execute_reply":"2021-11-03T01:57:01.685926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Data And Assets\n\nFor most assets we are provided 19M rows of historical data. Some of the coins have less historical data.","metadata":{}},{"cell_type":"code","source":"asset_map = asset.set_index('Asset_ID')['Asset_Name'].to_dict()\nasset_count = train['Asset_ID'].value_counts() \\\n    .sort_values()\nasset_count.index = asset_count.index.map(asset_map)\nax = asset_count.plot(kind='barh',\n                 title='Count of Rows per Asset ID in Training Dataset',\n                 figsize=(12, 6),\n                color=color_pal[1])\nax.set_xlabel('Rows in Training Set')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:58:29.218552Z","iopub.execute_input":"2021-11-03T01:58:29.219055Z","iopub.status.idle":"2021-11-03T01:58:29.823634Z","shell.execute_reply.started":"2021-11-03T01:58:29.219001Z","shell.execute_reply":"2021-11-03T01:58:29.822690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Coin Prices over Time\n- Lets take a look at some time trends.\n- We convert the timestamp column into datetime\n- Plot historic prices per coin.","metadata":{}},{"cell_type":"code","source":"train['Asset_Name'] = train['Asset_ID'].map(asset_map)\ntrain['datetime'] = pd.to_datetime(train['timestamp'], unit='s')","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:58:34.285347Z","iopub.execute_input":"2021-11-03T01:58:34.286186Z","iopub.status.idle":"2021-11-03T01:58:36.717153Z","shell.execute_reply.started":"2021-11-03T01:58:34.286137Z","shell.execute_reply":"2021-11-03T01:58:36.716248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(5, 3,\n                        figsize=(15, 15),\n                       sharex=True)\naxs = axs.flatten()\ni = 0\nfor asn, d in train.sample(1_000_00, random_state=529).groupby('Asset_Name'):\n    d.set_index('datetime')['Open'] \\\n        .plot(color=next(color_cycle),\n              ax=axs[i],\n              title=asn)\n    i += 1\nfig.suptitle('Training Data Historic Prices', fontsize=25, y=0.95)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:58:42.400293Z","iopub.execute_input":"2021-11-03T01:58:42.401154Z","iopub.status.idle":"2021-11-03T01:58:50.017327Z","shell.execute_reply.started":"2021-11-03T01:58:42.401099Z","shell.execute_reply":"2021-11-03T01:58:50.016325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(5, 3,\n                        figsize=(15, 15),\n                       sharex=True)\naxs = axs.flatten()\ni = 0\nfor asn, d in train.sample(1_000_00, random_state=529).groupby('Asset_Name'):\n    d.set_index('datetime')['Volume'] \\\n        .plot(color=next(color_cycle),\n              ax=axs[i],\n              title=asn)\n    i += 1\nfig.suptitle('Training Data - Historic Trading Volume', fontsize=25, y=0.95)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:58:57.100712Z","iopub.execute_input":"2021-11-03T01:58:57.101059Z","iopub.status.idle":"2021-11-03T01:59:05.009634Z","shell.execute_reply.started":"2021-11-03T01:58:57.101023Z","shell.execute_reply":"2021-11-03T01:59:05.008559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(5, 3,\n                        figsize=(15, 15),\n                       sharex=True)\naxs = axs.flatten()\ni = 0\nfor asn, d in train.sample(1_000_00, random_state=529).groupby('Asset_Name'):\n    d.set_index('datetime')['VWAP'] \\\n        .plot(color=next(color_cycle),\n              ax=axs[i],\n              title=asn)\n    i += 1\nfig.suptitle('Training Data - Historic Trading Volume weighted average price for the minute.',\n             fontsize=25, y=0.95)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:59:14.337400Z","iopub.execute_input":"2021-11-03T01:59:14.338341Z","iopub.status.idle":"2021-11-03T01:59:22.043662Z","shell.execute_reply.started":"2021-11-03T01:59:14.338303Z","shell.execute_reply":"2021-11-03T01:59:22.042687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## What is VWAP?\n\nPlotting VWAP vs the opening prices show they are very similar. VWAP is \"The volume weighted average price for the minute.\"","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(5, 3,\n                        figsize=(15, 15))\naxs = axs.flatten()\ni = 0\nfor asn, d in train.sample(1_000_00, random_state=529).groupby('Asset_Name'):\n    d.set_index('datetime')['Volume'].apply(np.log) \\\n        .plot(color=next(color_cycle),\n              ax=axs[i],\n              kind='kde',\n              title=asn)\n    i += 1\nfig.suptitle('Training Data - Historic Trading Log(Volume)',\n             fontsize=25, y=0.95)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T02:08:33.249488Z","iopub.execute_input":"2021-11-03T02:08:33.250054Z","iopub.status.idle":"2021-11-03T02:08:39.798867Z","shell.execute_reply.started":"2021-11-03T02:08:33.249992Z","shell.execute_reply":"2021-11-03T02:08:39.797765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Time Series Trends.\n- How does the day of week, time of year, etc impact the trading value?\n- Does our target have time series trends?","metadata":{}},{"cell_type":"code","source":"def time_series_features(df, dt_col='datetime', label=None):\n    \"\"\"\n    Creates time series features from datetime index.\n    \"\"\"\n    df = df.copy()\n    df['hour'] = df[dt_col].dt.hour\n    df['dayofweek'] = df[dt_col].dt.dayofweek\n    df['quarter'] = df[dt_col].dt.quarter\n    df['month'] = df[dt_col].dt.month\n    df['year'] = df[dt_col].dt.year\n    df['dayofyear'] = df[dt_col].dt.dayofyear\n    df['dayofmonth'] = df[dt_col].dt.day\n    df['weekofyear'] = df[dt_col].dt.isocalendar().week\n    return df\n\ntrain = time_series_features(train)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T01:59:31.639377Z","iopub.execute_input":"2021-11-03T01:59:31.639718Z","iopub.status.idle":"2021-11-03T01:59:31.648948Z","shell.execute_reply.started":"2021-11-03T01:59:31.639682Z","shell.execute_reply":"2021-11-03T01:59:31.647895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2021 VWAP by Month","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(5, 3,\n                        figsize=(15, 15), sharex=True)\naxs = axs.flatten()\ni = 0\nfor asn, d in train.query('year == 2021').sample(1_000_00, random_state=529).groupby('Asset_Name'):\n    sns.boxplot(data=d, x='month', y='VWAP', ax=axs[i])\n    axs[i].set_title(asn)\n    i += 1\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T02:45:12.920326Z","iopub.execute_input":"2021-11-03T02:45:12.920701Z","iopub.status.idle":"2021-11-03T02:45:20.286631Z","shell.execute_reply.started":"2021-11-03T02:45:12.920662Z","shell.execute_reply":"2021-11-03T02:45:20.285396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target for each Asset\n\nTarget is derived from log returns ( `Ra` ) over 15 minutes. We can visualize how the target is different over time for each asset.","metadata":{"execution":{"iopub.status.busy":"2021-11-03T02:24:42.090651Z","iopub.execute_input":"2021-11-03T02:24:42.090997Z","iopub.status.idle":"2021-11-03T02:24:42.116597Z","shell.execute_reply.started":"2021-11-03T02:24:42.090940Z","shell.execute_reply":"2021-11-03T02:24:42.115463Z"}}},{"cell_type":"code","source":"fig, axs = plt.subplots(5, 3,\n                        figsize=(15, 15),\n                       sharex=True)\naxs = axs.flatten()\ni = 0\nfor asn, d in train.sample(1_000_00, random_state=529).groupby('Asset_Name'):\n    d.set_index('datetime')['Target'] \\\n        .plot(color=next(color_cycle),\n              ax=axs[i],\n              title=asn)\n    i += 1\nfig.suptitle('Training Data - Target',\n             fontsize=25, y=0.95)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T02:33:10.286602Z","iopub.execute_input":"2021-11-03T02:33:10.287011Z","iopub.status.idle":"2021-11-03T02:33:18.975128Z","shell.execute_reply.started":"2021-11-03T02:33:10.286975Z","shell.execute_reply":"2021-11-03T02:33:18.973898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bitcoin Features","metadata":{}},{"cell_type":"code","source":"train_subset = train.query('Asset_Name == \"Bitcoin\"') \\\n    .sample(1_000, random_state=529)\nax = sns.pairplot(train_subset,\n             hue='year',\n             vars=['Volume',\n                   'Count',\n                   'VWAP',\n                   'Target',\n                  ])","metadata":{"execution":{"iopub.status.busy":"2021-11-03T02:33:50.532036Z","iopub.execute_input":"2021-11-03T02:33:50.532682Z","iopub.status.idle":"2021-11-03T02:33:59.916929Z","shell.execute_reply.started":"2021-11-03T02:33:50.532640Z","shell.execute_reply":"2021-11-03T02:33:59.915879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ethereum Features","metadata":{}},{"cell_type":"code","source":"train_subset = train.query('Asset_Name == \"Ethereum\"') \\\n    .sample(1_000, random_state=529)\nax = sns.pairplot(train_subset,\n             hue='year',\n                  palette='Spectral',\n             vars=['Volume',\n                   'Count',\n                   'VWAP',\n                   'Target',\n                  ])","metadata":{"execution":{"iopub.status.busy":"2021-11-03T02:34:27.424691Z","iopub.execute_input":"2021-11-03T02:34:27.425031Z","iopub.status.idle":"2021-11-03T02:34:36.681911Z","shell.execute_reply.started":"2021-11-03T02:34:27.424990Z","shell.execute_reply":"2021-11-03T02:34:36.680802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TODO:\nStart a baseline Model","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}