{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## note: you might need to upgrade your statsmodels module to run this notebook","metadata":{}},{"cell_type":"code","source":"# !pip install statsmodels --upgrade\nimport pandas as pd\nimport numpy as np\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport seaborn as sns\nfrom processingmethods_py import AllCryptos\nfrom statsmodels.tsa.arima.model import ARIMA\nfrom statsmodels.tsa.statespace.sarimax import SARIMAX\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:43:59.627961Z","iopub.execute_input":"2021-11-14T13:43:59.628454Z","iopub.status.idle":"2021-11-14T13:44:01.03905Z","shell.execute_reply.started":"2021-11-14T13:43:59.628343Z","shell.execute_reply":"2021-11-14T13:44:01.038201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialising AllCryptos Class from ProcessingMethods;\n# this gives us access to all the asset data\ndata = AllCryptos(\"../input/g-research-crypto-forecasting/\")\n","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:44:01.040545Z","iopub.execute_input":"2021-11-14T13:44:01.040818Z","iopub.status.idle":"2021-11-14T13:45:04.631364Z","shell.execute_reply.started":"2021-11-14T13:44:01.040773Z","shell.execute_reply":"2021-11-14T13:45:04.630425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport warnings\nfrom scipy.stats import pearsonr\n\n\n## params\ndef test_ARIMA(start,asset = 'Ethereum', train_length = 1500, test_length = 3000, refit_period =1000):\n    warnings.filterwarnings(\"ignore\")\n\n    ## get data\n    asset_data = data.get_full_crypto_df(asset).Target\n    \n    asset_data = asset_data.reset_index(drop=True)\n    asset_data = asset_data.fillna(0)\n    \n\n    train = asset_data.iloc[start:start+ train_length]\n    test = asset_data.iloc[start+ train_length:start+ train_length+test_length]\n    true_y = asset_data.iloc[start+ train_length +15:start+train_length + test_length+15]\n\n    # cosmetic stuff\n\n\n# prediction loop\n    predictions = pd.Series()\n    model = SARIMAX(train,\n                    order=(2, 0, 1),\n                    seasonal_order=(1, 0, 1, 15))\n    #         model = ARIMA(train, order=(2, 0, 2))\n    model_fit = model.fit(disp = False)\n    output = model_fit.forecast(16).tail(1)\n    predictions = predictions.append(output)\n    for t in range(len(test) - 1):\n        observed = test.iloc[[t]]\n        if (t+1) % refit_period == 0:\n            model_fit = model_fit.append(observed, refit=True, fit_kwargs = {'disp':False})\n            print(f'Step {t} out of {14*3*len(test)}')\n        else:\n            model_fit = model_fit.append(observed, refit=False)\n        train = train.append(observed)\n        output = model_fit.forecast().tail(1)\n        predictions = predictions.append(output)\n    \n    correlation = pearsonr(predictions, true_y[:len(predictions)])[0]\n    \n    return correlation","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:50:22.701155Z","iopub.execute_input":"2021-11-14T13:50:22.701445Z","iopub.status.idle":"2021-11-14T13:50:22.715235Z","shell.execute_reply.started":"2021-11-14T13:50:22.701416Z","shell.execute_reply":"2021-11-14T13:50:22.714422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assets = list(data.get_asset_names().keys())\ncorr_dict = {}\n\nfor asset in tqdm(assets):\n    correlations = []\n    for start in [0,100000,1000000]:\n        correlations.append(test_ARIMA(start, asset))\n        print(correlations)\n#             continue\n    mean_corr = np.mean(correlations)\n    corr_dict[asset]= mean_corr\n\ncorr_dict","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:50:22.891836Z","iopub.execute_input":"2021-11-14T13:50:22.892305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# asset one correlations: [0.054846332251473257, -0.008268965277755454, 0.1685406154581897, 0.048630204999339136, 0.048630204999339136]\n# # cosmetic stuff\n# from tqdm import tqdm\n# import warnings\n# warnings.filterwarnings(\"ignore\")\n\n# # prediction loop\n# predictions = pd.Series()\n# model = SARIMAX(train,\n#                 order=(2, 0, 1),\n#                 seasonal_order=(1, 0, 1, 15))\n# #         model = ARIMA(train, order=(2, 0, 2))\n# model_fit = model.fit()\n# output = model_fit.forecast(16).tail(1)\n# predictions = predictions.append(output)\n# for t in tqdm(range(len(test) - 1)):\n#     observed = test.iloc[[t]]\n#     if t % refit_period == 0:\n#         model_fit = model_fit.append(observed, refit=True)\n#     else:\n#         model_fit = model_fit.append(observed, refit=False)\n#     train = train.append(observed)\n#     output = model_fit.forecast().tail(1)\n#     predictions = predictions.append(output)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:50:21.256964Z","iopub.status.idle":"2021-11-14T13:50:21.257441Z","shell.execute_reply.started":"2021-11-14T13:50:21.257201Z","shell.execute_reply":"2021-11-14T13:50:21.257227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # cosmetic stuff\n# from tqdm import tqdm\n# import warnings\n# warnings.filterwarnings(\"ignore\")\n\n# # prediction loop\n# predictions = pd.Series()\n\n# for t in tqdm(range(len(test) - 1)):\n#     observed = test.iloc[[t]]\n#     if t % refit_period == 0:\n#         train = train.iloc[-refit_period:].reset_index(drop=True)\n#         model = ARIMA(train, order=(2, 0, 2))\n#         model_fit = model.fit()\n#     else:\n#         model_fit = model_fit.extend(train.tail(1))\n\n#     observed.index = [refit_period + (observed.index.max() % refit_period)]\n#     train = train.append(observed)\n#     output = model_fit.forecast().tail(1)\n#     predictions = predictions.append(output)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:50:21.259505Z","iopub.status.idle":"2021-11-14T13:50:21.259877Z","shell.execute_reply.started":"2021-11-14T13:50:21.259666Z","shell.execute_reply":"2021-11-14T13:50:21.259683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(15,5))\n# plt.plot(predictions)\n# plt.plot(true_y)\n# plt.legend(['predictions', 'actual'])\n","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:50:21.262559Z","iopub.status.idle":"2021-11-14T13:50:21.263497Z","shell.execute_reply.started":"2021-11-14T13:50:21.263211Z","shell.execute_reply":"2021-11-14T13:50:21.263245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.scatter(predictions, true_y[:len(predictions)])","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:50:21.264615Z","iopub.status.idle":"2021-11-14T13:50:21.265334Z","shell.execute_reply.started":"2021-11-14T13:50:21.265012Z","shell.execute_reply":"2021-11-14T13:50:21.265047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from scipy.stats import pearsonr\n\n# correlation = pearsonr(predictions, true_y[:len(predictions)])[0]\n# # print(f'Correlation between predictions and test: {correlation}')\n\n# print(\n#     f'Correlation: {correlation}, trained on {train_length} data points,\\\n#  tested on {test_length} data points, refitted every {refit_period} periods'\n# )\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-14T13:50:21.267079Z","iopub.status.idle":"2021-11-14T13:50:21.267523Z","shell.execute_reply.started":"2021-11-14T13:50:21.267303Z","shell.execute_reply":"2021-11-14T13:50:21.267325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}