{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA and Holt-Winters methods on Bitcoin prediction during 365 days","metadata":{}},{"cell_type":"markdown","source":"- Utils","metadata":{}},{"cell_type":"code","source":"# system libraries\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# data manipulation libraries\nimport pandas as pd\nimport numpy as np\n\n# graphical libraries\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\n# modelisation libraries\nfrom datetime import datetime\nfrom statsmodels.graphics.tsaplots import plot_acf, plot_pacf\nfrom statsmodels.tsa.stattools import adfuller\nfrom statsmodels.tsa.api import ExponentialSmoothing\nfrom statsmodels.tsa.seasonal import seasonal_decompose\n\n# metric evaluation\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:24:05.033528Z","iopub.execute_input":"2021-11-27T14:24:05.034019Z","iopub.status.idle":"2021-11-27T14:24:06.524094Z","shell.execute_reply.started":"2021-11-27T14:24:05.033920Z","shell.execute_reply":"2021-11-27T14:24:06.523210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def group_trading_timeseries(df, time_step_size):\n    #Copied from https://www.kaggle.com/alexisalvarez/eda-resampling-the-power-of-technical-analysis\n    df = df.sort_index()\n    df = df.resample('{}'.format(time_step_size)).agg({\n            'Count': lambda df: df.sum(),\n            'Open': lambda s: s.iloc[0] ,\n            'High': lambda df: df.max(), \n            'Low': lambda df: df.min(),\n            'Close': lambda df: df.iloc[-1], \n            'Volume': lambda df: df.sum(),\n            'VWAP' : lambda df: df.mean(),\n            'Target' : lambda df: df.mean()\n            }) \n    return df\n\ndef check_stationarity(series, asset_id):\n    # Copied and edited from https://machinelearningmastery.com/time-series-data-stationary-python/\n\n    result = adfuller(series.values)\n    if (result[1] <= 0.05) & (result[4]['5%'] > result[0]):\n        print(f\"{asset[asset.Asset_ID == asset_id].Asset_Name.values[0]}: \\u001b[32mStationary\\u001b[0m\")\n    else:\n        print(f\"{asset[asset.Asset_ID == asset_id].Asset_Name.values[0]}: \\x1b[31mNon-stationary\\x1b[0m\")\n       \n    print('ADF Statistic: %f' % result[0])\n    print('p-value: %f' % result[1])\n    print('Critical Values:')\n    for key, value in result[4].items():\n        print('\\t%s: %.3f' % (key, value))\n    print('\\n')\n\ndef metric_evaluation(y_true, y_pred):\n    mae = mean_absolute_error(y_true, y_pred)\n    mse = mean_squared_error(y_true, y_pred)\n    rmse = np.sqrt(mse)\n    mape = np.mean(np.abs((y_true - y_pred) / y_true)) * 100\n    print('MAE = ', mae)\n    print('MSE = ', mse)\n    print('RMSE = ', rmse)\n    print('MAPE = ', mape)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:24:06.525532Z","iopub.execute_input":"2021-11-27T14:24:06.526181Z","iopub.status.idle":"2021-11-27T14:24:06.539331Z","shell.execute_reply.started":"2021-11-27T14:24:06.526147Z","shell.execute_reply":"2021-11-27T14:24:06.538490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preprocessing","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\", encoding=\"utf-8\")\nasset = pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\", encoding=\"utf-8\")\n\ndata[\"timestamp\"] = data[\"timestamp\"].apply(lambda x: datetime.fromtimestamp(x))\ndata[\"date\"] = data[\"timestamp\"].astype('datetime64[s]')\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:24:06.540537Z","iopub.execute_input":"2021-11-27T14:24:06.540781Z","iopub.status.idle":"2021-11-27T14:25:44.964632Z","shell.execute_reply.started":"2021-11-27T14:24:06.540751Z","shell.execute_reply":"2021-11-27T14:25:44.963941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.merge(data, asset, on=\"Asset_ID\")\ndata.set_index(\"date\", inplace=True)\ndata = data[[\"Asset_Name\", \"Weight\", \"Count\",\n             \"Open\", \"High\", \"Low\",\n             \"Close\", \"Volume\", \"VWAP\",\n             \"Target\"]]\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:25:44.966250Z","iopub.execute_input":"2021-11-27T14:25:44.966593Z","iopub.status.idle":"2021-11-27T14:25:52.714768Z","shell.execute_reply.started":"2021-11-27T14:25:44.966564Z","shell.execute_reply":"2021-11-27T14:25:52.714227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Columns of the dataset :\n* timestamp - A timestamp for the minute covered by the row.\n* Asset_ID - An ID code for the cryptoasset.\n* Count - The number of trades that took place this minute.\n* Open - The USD price at the beginning of the minute.\n* High - The highest USD price during the minute.\n* Low - The lowest USD price during the minute.\n* Close - The USD price at the end of the minute.\n* Volume - The number of cryptoasset units traded during the minute.\n* VWAP - The volume weighted average price for the minute.\n* Target - 15 minute residualized returns.","metadata":{}},{"cell_type":"code","source":"data.isnull().mean() * 100","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:25:52.715760Z","iopub.execute_input":"2021-11-27T14:25:52.716104Z","iopub.status.idle":"2021-11-27T14:25:55.692310Z","shell.execute_reply.started":"2021-11-27T14:25:52.716076Z","shell.execute_reply":"2021-11-27T14:25:55.691512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data[data.duplicated()]","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:25:55.693426Z","iopub.execute_input":"2021-11-27T14:25:55.693645Z","iopub.status.idle":"2021-11-27T14:27:07.972586Z","shell.execute_reply.started":"2021-11-27T14:25:55.693617Z","shell.execute_reply":"2021-11-27T14:27:07.971960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.drop_duplicates(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:27:07.973477Z","iopub.execute_input":"2021-11-27T14:27:07.974073Z","iopub.status.idle":"2021-11-27T14:28:20.785088Z","shell.execute_reply.started":"2021-11-27T14:27:07.974030Z","shell.execute_reply":"2021-11-27T14:28:20.784447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:20.786115Z","iopub.execute_input":"2021-11-27T14:28:20.786434Z","iopub.status.idle":"2021-11-27T14:28:25.667373Z","shell.execute_reply.started":"2021-11-27T14:28:20.786407Z","shell.execute_reply":"2021-11-27T14:28:25.666738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:25.668416Z","iopub.execute_input":"2021-11-27T14:28:25.668731Z","iopub.status.idle":"2021-11-27T14:28:25.684527Z","shell.execute_reply.started":"2021-11-27T14:28:25.668702Z","shell.execute_reply":"2021-11-27T14:28:25.683498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Analysis","metadata":{}},{"cell_type":"markdown","source":"In this trial, we only use Bitcoin. ","metadata":{}},{"cell_type":"code","source":"df_btc = data.loc[data[\"Asset_Name\"] == \"Bitcoin\"]\ndf_btc = group_trading_timeseries(df_btc, time_step_size=\"1d\")\ndf_btc.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:25.687285Z","iopub.execute_input":"2021-11-27T14:28:25.687992Z","iopub.status.idle":"2021-11-27T14:28:31.135975Z","shell.execute_reply.started":"2021-11-27T14:28:25.687955Z","shell.execute_reply":"2021-11-27T14:28:31.133601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Descriptive Analysis","metadata":{}},{"cell_type":"code","source":"fig = go.Figure(data=[go.Candlestick(x=df_btc.index,\n                                     open=df_btc['Open'],\n                                     high=df_btc['High'],\n                                     low=df_btc['Low'],\n                                     close=df_btc['Close'])])\nfig.update_layout(xaxis_rangeslider_visible=False)\nfig.update_layout(\n    title='Bitcoin from Janv. 2018 to Sep. 2021',\n    yaxis_title='USD',\n    shapes = [dict(\n        x0='2020-03-11', x1='2020-03-11', y0=0, y1=1, xref='x', yref='paper',\n        line_width=2)],\n    annotations=[dict(\n        x='2020-03-11', y=0.05, xref='x', yref='paper',\n        showarrow=False, xanchor='left', text='Covid Sanitary Crisis Begin')]\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:31.140188Z","iopub.execute_input":"2021-11-27T14:28:31.140537Z","iopub.status.idle":"2021-11-27T14:28:31.409264Z","shell.execute_reply.started":"2021-11-27T14:28:31.140416Z","shell.execute_reply":"2021-11-27T14:28:31.408323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in df_btc.columns:\n    print('\\033[1m'+i+'\\033[0m')\n    print(\"Mean: \", round(df_btc[i].mean(), 3))\n    print(\"Median: \", round(df_btc[i].median(), 3))\n    print(\"Standard Deviation: \", round(df_btc[i].std(), 3))\n    print(\"Variance: \", round(df_btc[i].var(), 3))\n    print(\"-------------------\")","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:31.410801Z","iopub.execute_input":"2021-11-27T14:28:31.411284Z","iopub.status.idle":"2021-11-27T14:28:31.463699Z","shell.execute_reply.started":"2021-11-27T14:28:31.411239Z","shell.execute_reply":"2021-11-27T14:28:31.462868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in df_btc.columns:\n    sns.displot(x=i,\n                data=df_btc,\n                kde=True)\n    plt.title(\"Distribution of the variable: \" +  i)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:31.465483Z","iopub.execute_input":"2021-11-27T14:28:31.465742Z","iopub.status.idle":"2021-11-27T14:28:35.557096Z","shell.execute_reply.started":"2021-11-27T14:28:31.465713Z","shell.execute_reply":"2021-11-27T14:28:35.556149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stationarity","metadata":{}},{"cell_type":"code","source":"for i in df_btc.columns:\n    print(i)\n    check_stationarity(df_btc[i], 0)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:35.558951Z","iopub.execute_input":"2021-11-27T14:28:35.559307Z","iopub.status.idle":"2021-11-27T14:28:36.067661Z","shell.execute_reply.started":"2021-11-27T14:28:35.559257Z","shell.execute_reply":"2021-11-27T14:28:36.066786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_stat = [\"Count\", \"Open\", \"High\",\n            \"Low\", \"Close\", \"VWAP\"]\n\nfor i in non_stat:\n    df_btc[i] = df_btc[i].apply(lambda x: np.log(x))\n    df_btc[i] = df_btc[i].diff(1)\ndf_btc.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:36.069268Z","iopub.execute_input":"2021-11-27T14:28:36.069772Z","iopub.status.idle":"2021-11-27T14:28:36.123014Z","shell.execute_reply.started":"2021-11-27T14:28:36.069729Z","shell.execute_reply":"2021-11-27T14:28:36.121880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_btc.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:36.124715Z","iopub.execute_input":"2021-11-27T14:28:36.125198Z","iopub.status.idle":"2021-11-27T14:28:36.150085Z","shell.execute_reply.started":"2021-11-27T14:28:36.125146Z","shell.execute_reply":"2021-11-27T14:28:36.149120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in df_btc.columns:\n    print(i)\n    check_stationarity(df_btc[i], 0)","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:36.152310Z","iopub.execute_input":"2021-11-27T14:28:36.152900Z","iopub.status.idle":"2021-11-27T14:28:36.651178Z","shell.execute_reply.started":"2021-11-27T14:28:36.152850Z","shell.execute_reply":"2021-11-27T14:28:36.650268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure(data=[go.Candlestick(x=df_btc.index,\n                                     open=df_btc['Open'],\n                                     high=df_btc['High'],\n                                     low=df_btc['Low'],\n                                     close=df_btc['Close'])])\nfig.update_layout(xaxis_rangeslider_visible=False)\nfig.update_layout(\n    title='Bitcoin from Janv. 2018 to Sep. 2021',\n    yaxis_title='USD',\n    shapes = [dict(\n        x0='2020-03-11', x1='2020-03-11', y0=0, y1=1, xref='x', yref='paper',\n        line_width=2)],\n    annotations=[dict(\n        x='2020-03-11', y=0.05, xref='x', yref='paper',\n        showarrow=False, xanchor='left', text='Covid Sanitary Crisis Begin')]\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:36.652883Z","iopub.execute_input":"2021-11-27T14:28:36.653450Z","iopub.status.idle":"2021-11-27T14:28:36.768003Z","shell.execute_reply.started":"2021-11-27T14:28:36.653392Z","shell.execute_reply":"2021-11-27T14:28:36.767190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Seasonal decomposition","metadata":{}},{"cell_type":"code","source":"print(\"\\033[1mSeasonal decomposition of each variables\\033[0m\")\nfor i in df_btc.columns:\n    decomp_x = seasonal_decompose(df_btc[i],  model='add')\n    decomp_x.plot()\n    plt.show()\n    print(\"\\033[1m---------------------------------------------------------\\033[0m\")","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:36.770712Z","iopub.execute_input":"2021-11-27T14:28:36.771438Z","iopub.status.idle":"2021-11-27T14:28:44.147573Z","shell.execute_reply.started":"2021-11-27T14:28:36.771398Z","shell.execute_reply":"2021-11-27T14:28:44.146737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Moving averages ","metadata":{}},{"cell_type":"code","source":"window_size = 362\n\nfor i in df_btc.columns:\n    windows = df_btc[i].rolling(window_size)\n    moving_averages = windows.mean()\n    \n    moving_averages_list = moving_averages.tolist()\n    without_nans = moving_averages_list[window_size - 1:]\n    \n    plt.figure(figsize=(15,5))\n    plt.plot(df_btc[i], label=i)\n    plt.plot(moving_averages, label='Moving Average')\n    plt.title(i+\" in function of time\")\n    plt.xlabel('Time')\n    plt.legend(loc='best')\n    plt.show()\n    print(\"\\033[1m---------------------------------------------------------\\033[0m\")","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:44.148773Z","iopub.execute_input":"2021-11-27T14:28:44.149006Z","iopub.status.idle":"2021-11-27T14:28:46.700863Z","shell.execute_reply.started":"2021-11-27T14:28:44.148976Z","shell.execute_reply":"2021-11-27T14:28:46.700042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predictions with ML models","metadata":{}},{"cell_type":"markdown","source":"### Exponential Smoothing","metadata":{}},{"cell_type":"code","source":"for i in df_btc.columns:\n    hw = ExponentialSmoothing(np.asarray(df_btc[i]),\n                                         trend=\"add\",\n                                         seasonal=\"add\",\n                                         seasonal_periods=7).fit()\n    hw_pred = hw.forecast(365)\n    \n    plt.figure(figsize=(14,6))\n    plt.plot(df_btc[i], label=i, color='green')\n    plt.plot(pd.date_range(df_btc.index[len(df_btc)-1], periods=365, freq='D'),\n             hw_pred, label='Prediction', color='red')\n    plt.title(i + \" and its prediction for the next year\")\n    plt.xlabel('Time')\n    plt.legend(loc='best')\n    plt.show()\n    print(\"\\033[1m--------------------------------------------------------------------------------------------------------------------\\033[0m\")","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:46.702129Z","iopub.execute_input":"2021-11-27T14:28:46.702436Z","iopub.status.idle":"2021-11-27T14:28:52.094764Z","shell.execute_reply.started":"2021-11-27T14:28:46.702403Z","shell.execute_reply":"2021-11-27T14:28:52.093929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_2 = df_btc.drop(df_btc.index[-365:])\n\nprint(\"\\033[1mEvaluation of Exponential Smoothing prediction\\033[0m\")\n\nfor i in df_btc.columns:\n    hw = ExponentialSmoothing(np.asarray(df_2[i]), seasonal_periods=7,\n                              trend='add', seasonal='add').fit()\n    hw_pred = hw.forecast(365)\n\n    plt.figure(figsize=(14,6))\n    plt.plot(df_btc[i], label=i, color='gray')\n    plt.plot(pd.date_range(df_2.index[len(df_2)-1], periods=365, freq='D'), hw_pred, \n             label='Prediction', color='green')\n    plt.xlabel('Time')\n    plt.legend(loc='best')\n    plt.show()\n    print(\"*****************************\\n\")\n    y_pred = hw_pred\n    y_true = np.asarray(df_2[i].iloc[-365:])\n    \n    metric_evaluation(y_true, y_pred)\n    print(\"-----------------------------\")","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:28:52.096419Z","iopub.execute_input":"2021-11-27T14:28:52.096892Z","iopub.status.idle":"2021-11-27T14:28:56.970293Z","shell.execute_reply.started":"2021-11-27T14:28:52.096846Z","shell.execute_reply":"2021-11-27T14:28:56.969427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ressources","metadata":{}},{"cell_type":"markdown","source":"- https://www.kaggle.com/iamleonie/to-the-moon-g-research-crypto-forecasting-eda","metadata":{}}]}