{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## What we have?","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\ndirectory = \"../input/g-research-crypto-forecasting\" #you need to change this to your data location\ncrypto_df = pd.read_csv(os.path.join(directory, 'train.csv'))\n# data ['Time']=pd.to_datetime(data['timestamp'], unit='s')\ncrypto_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:35:49.934044Z","iopub.execute_input":"2022-01-15T19:35:49.935893Z","iopub.status.idle":"2022-01-15T19:36:42.387865Z","shell.execute_reply.started":"2022-01-15T19:35:49.935811Z","shell.execute_reply":"2022-01-15T19:36:42.386563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"details_df = pd.read_csv(os.path.join(directory, 'asset_details.csv'))\n# data = pd.merge(data, details, on =\"Asset_ID\", how = 'left')\ndetails_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:42.389860Z","iopub.execute_input":"2022-01-15T19:36:42.390671Z","iopub.status.idle":"2022-01-15T19:36:42.408017Z","shell.execute_reply.started":"2022-01-15T19:36:42.390625Z","shell.execute_reply":"2022-01-15T19:36:42.407083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {row[\"Asset_Name\"]: crypto_df[crypto_df['Asset_ID'] == row['Asset_ID']]\n        for _, row in details_df.iterrows()}","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:42.409736Z","iopub.execute_input":"2022-01-15T19:36:42.410117Z","iopub.status.idle":"2022-01-15T19:36:47.448800Z","shell.execute_reply.started":"2022-01-15T19:36:42.410070Z","shell.execute_reply":"2022-01-15T19:36:47.447837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## What is our goal?","metadata":{}},{"cell_type":"markdown","source":"predict the target values","metadata":{}},{"cell_type":"markdown","source":"## Useful Links\n- [Competition Info](https://www.kaggle.com/c/g-research-crypto-forecasting/overview)\n- [API tool for submission](https://www.kaggle.com/sohier/detailed-api-introduction)\n- [Tutorial](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)\n- [Eval Metrics](https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/291845), [Eval Metrics](https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778)","metadata":{}},{"cell_type":"markdown","source":"# Data Preprocess","metadata":{}},{"cell_type":"markdown","source":"## Cleaning and preparing time series data","metadata":{}},{"cell_type":"code","source":"eth_df = data[\"Ethereum\"][-2000:]\neth_df.info(show_counts = True)\neth_df = eth_df.set_index(\"timestamp\")","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:47.452852Z","iopub.execute_input":"2022-01-15T19:36:47.453884Z","iopub.status.idle":"2022-01-15T19:36:47.474307Z","shell.execute_reply.started":"2022-01-15T19:36:47.453827Z","shell.execute_reply":"2022-01-15T19:36:47.473211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## check time gap between rows","metadata":{}},{"cell_type":"markdown","source":"according to tutorial\n> Missing asset data, for a given minute, is not represented by NaN's, but instead by the absence of those rows. \n\nWe can check the timestamp difference between consecutive rows to see if there is missing data.","metadata":{}},{"cell_type":"code","source":"(eth_df.index[1:] - eth_df.index[:-1]).value_counts().head()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:47.476249Z","iopub.execute_input":"2022-01-15T19:36:47.476882Z","iopub.status.idle":"2022-01-15T19:36:47.489207Z","shell.execute_reply.started":"2022-01-15T19:36:47.476821Z","shell.execute_reply":"2022-01-15T19:36:47.488184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## check values still missing","metadata":{}},{"cell_type":"code","source":"eth_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:47.491800Z","iopub.execute_input":"2022-01-15T19:36:47.492123Z","iopub.status.idle":"2022-01-15T19:36:47.502979Z","shell.execute_reply.started":"2022-01-15T19:36:47.492089Z","shell.execute_reply":"2022-01-15T19:36:47.501903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## define the data prepare flow\n\n1. padding the missing rows (according to the tutorial)\n2. naturally fill the missing values with previous observations","metadata":{}},{"cell_type":"code","source":"def prepare_data(df):\n    index_name = \"timestamp\"\n    if df.index.name != index_name:\n        df = df.set_index(index_name)\n    df = df.reindex(range(df.index[0], df.index[-1] + 60, 60), method='pad')\n    df = df.fillna(method='ffill')\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:47.504790Z","iopub.execute_input":"2022-01-15T19:36:47.505158Z","iopub.status.idle":"2022-01-15T19:36:47.516999Z","shell.execute_reply.started":"2022-01-15T19:36:47.505112Z","shell.execute_reply":"2022-01-15T19:36:47.516050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eth_df = prepare_data(eth_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:47.518384Z","iopub.execute_input":"2022-01-15T19:36:47.519632Z","iopub.status.idle":"2022-01-15T19:36:47.531858Z","shell.execute_reply.started":"2022-01-15T19:36:47.519590Z","shell.execute_reply":"2022-01-15T19:36:47.530956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explorate Data","metadata":{}},{"cell_type":"markdown","source":"## Auto correlation","metadata":{"execution":{"iopub.status.busy":"2021-11-27T14:21:09.809314Z","iopub.execute_input":"2021-11-27T14:21:09.809628Z","iopub.status.idle":"2021-11-27T14:21:09.828673Z","shell.execute_reply.started":"2021-11-27T14:21:09.809545Z","shell.execute_reply":"2021-11-27T14:21:09.828033Z"}}},{"cell_type":"code","source":"from statsmodels.graphics.tsaplots import plot_acf, plot_pacf\nimport matplotlib.pyplot as plt\n\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)[1:]\n\nlret = log_return(eth_df[\"Close\"])","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:47.533262Z","iopub.execute_input":"2022-01-15T19:36:47.533528Z","iopub.status.idle":"2022-01-15T19:36:47.548436Z","shell.execute_reply.started":"2022-01-15T19:36:47.533489Z","shell.execute_reply":"2022-01-15T19:36:47.547681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_acf(lret, lags=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:47.551462Z","iopub.execute_input":"2022-01-15T19:36:47.552921Z","iopub.status.idle":"2022-01-15T19:36:47.793897Z","shell.execute_reply.started":"2022-01-15T19:36:47.552855Z","shell.execute_reply":"2022-01-15T19:36:47.792597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_pacf(lret, lags=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:36:47.795342Z","iopub.execute_input":"2022-01-15T19:36:47.795579Z","iopub.status.idle":"2022-01-15T19:36:48.015966Z","shell.execute_reply.started":"2022-01-15T19:36:47.795551Z","shell.execute_reply":"2022-01-15T19:36:48.014917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlation between assets","metadata":{}},{"cell_type":"markdown","source":"# Baselines","metadata":{}},{"cell_type":"code","source":"train_data = eth_df[:1400]\ntest_data = eth_df[1400:]","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:59:53.007736Z","iopub.execute_input":"2022-01-15T19:59:53.008051Z","iopub.status.idle":"2022-01-15T19:59:53.013314Z","shell.execute_reply.started":"2022-01-15T19:59:53.008021Z","shell.execute_reply":"2022-01-15T19:59:53.012561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from statsmodels.tsa.arima_model import ARIMA\nmodel = ARIMA(train_data[\"Target\"].diff().iloc[1:].values, order=(2,1,0))\nresult = model.fit()\nresult.plot_predict(start=1, end=len(train_data))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T19:51:39.965993Z","iopub.execute_input":"2022-01-15T19:51:39.966296Z","iopub.status.idle":"2022-01-15T19:51:40.467707Z","shell.execute_reply.started":"2022-01-15T19:51:39.966265Z","shell.execute_reply":"2022-01-15T19:51:40.466476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_hat = result.predict(start=1, end=600)\nprint('Test score', f\"{np.corrcoef(y_hat, test_data['Target'])[0,1]:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-15T20:09:44.181645Z","iopub.execute_input":"2022-01-15T20:09:44.181938Z","iopub.status.idle":"2022-01-15T20:09:44.191832Z","shell.execute_reply.started":"2022-01-15T20:09:44.181909Z","shell.execute_reply":"2022-01-15T20:09:44.190574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neural_network import MLPRegressor\n\nX_train, y_train = train_data.drop([\"Target\"], axis=1), train_data[\"Target\"]\nX_test, y_test = test_data.drop([\"Target\"], axis=1), test_data[\"Target\"]\n\nclf = MLPRegressor(random_state=1, max_iter=300).fit(X_train.values, y_train.values)\ny_test_hat = clf.predict(X_test)\nprint('Test score', f\"{np.corrcoef(y_test_hat, test_data['Target'])[0,1]:.2f}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-15T20:09:30.603946Z","iopub.execute_input":"2022-01-15T20:09:30.604255Z","iopub.status.idle":"2022-01-15T20:09:34.208743Z","shell.execute_reply.started":"2022-01-15T20:09:30.604225Z","shell.execute_reply":"2022-01-15T20:09:34.207782Z"},"trusted":true},"execution_count":null,"outputs":[]}]}