{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import seaborn as sns\nfrom datetime import datetime\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-15T08:53:58.570257Z","iopub.execute_input":"2022-01-15T08:53:58.570616Z","iopub.status.idle":"2022-01-15T08:53:58.582411Z","shell.execute_reply.started":"2022-01-15T08:53:58.570582Z","shell.execute_reply":"2022-01-15T08:53:58.581473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Read the Data\n\nRead the datasets from 'g-research-crypto-forecasting datasets","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')\nasset_ID = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:53:58.584202Z","iopub.execute_input":"2022-01-15T08:53:58.584717Z","iopub.status.idle":"2022-01-15T08:54:30.219979Z","shell.execute_reply.started":"2022-01-15T08:53:58.584681Z","shell.execute_reply":"2022-01-15T08:54:30.21924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_ID","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:30.222473Z","iopub.execute_input":"2022-01-15T08:54:30.222873Z","iopub.status.idle":"2022-01-15T08:54:30.233082Z","shell.execute_reply.started":"2022-01-15T08:54:30.222835Z","shell.execute_reply":"2022-01-15T08:54:30.232222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **2. Data exploration**\n## **Take the Ethereum data**\nTake the each Asset_ID data, the Bitcoin is Asset_ID == 1","metadata":{}},{"cell_type":"code","source":"Bitcoin_Cash = train_data[train_data['Asset_ID'] == 2]\nBinance_Coin = train_data[train_data['Asset_ID'] == 0]\nBitcoin = train_data[train_data['Asset_ID'] == 1]\nEOS_IO = train_data[train_data['Asset_ID'] == 5]\nEthereum_Classic = train_data[train_data['Asset_ID'] == 6]\nEthereum = train_data[train_data['Asset_ID'] == 6]\nLitecoin = train_data[train_data['Asset_ID'] == 9]\nMonero = train_data[train_data['Asset_ID'] == 11]\nTRON = train_data[train_data['Asset_ID'] == 13]\nStellar = train_data[train_data['Asset_ID'] == 12]\nCardano = train_data[train_data['Asset_ID'] == 3]\nIOTA = train_data[train_data['Asset_ID'] == 8]\nMaker = train_data[train_data['Asset_ID'] == 10]\nDogecoin = train_data[train_data['Asset_ID'] == 4]","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:30.235472Z","iopub.execute_input":"2022-01-15T08:54:30.23575Z","iopub.status.idle":"2022-01-15T08:54:33.395138Z","shell.execute_reply.started":"2022-01-15T08:54:30.235713Z","shell.execute_reply":"2022-01-15T08:54:33.394396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **3. Data Clean**\nAdd the row which is NaN (More information can search the G-Research Crypto Forecasting notebook)","metadata":{}},{"cell_type":"code","source":"Bitcoin_Cash = Bitcoin_Cash.reindex(range(Bitcoin_Cash.index[0], Bitcoin_Cash.index[-1], 60), method = 'pad')\nBinance_Coin = Binance_Coin.reindex(range(Binance_Coin.index[0], Binance_Coin.index[-1], 60), method = 'pad')\nBitcoin = Bitcoin.reindex(range(Bitcoin.index[0], Bitcoin.index[-1], 60), method = 'pad')\nEOS_IO = EOS_IO.reindex(range(EOS_IO.index[0], EOS_IO.index[-1], 60), method = 'pad')\nEthereum_Classic = Ethereum_Classic.reindex(range(Ethereum_Classic.index[0], Ethereum_Classic.index[-1], 60), method = 'pad')\nEthereum = Ethereum.reindex(range(Ethereum.index[0], Ethereum.index[-1], 60), method = 'pad')\nLitecoin = Litecoin.reindex(range(Litecoin.index[0], Litecoin.index[-1], 60), method = 'pad')\nMonero = Monero.reindex(range(Monero.index[0], Monero.index[-1], 60), method = 'pad')\nTRON = TRON.reindex(range(TRON.index[0], TRON.index[-1], 60), method = 'pad')\nStellar = Stellar.reindex(range(Stellar.index[0], Stellar.index[-1], 60), method = 'pad')\nCardano = Cardano.reindex(range(Cardano.index[0], Cardano.index[-1], 60), method = 'pad')\nIOTA = IOTA.reindex(range(IOTA.index[0], IOTA.index[-1], 60), method = 'pad')\nMaker = Maker.reindex(range(Maker.index[0], Maker.index[-1], 60), method = 'pad')\nDogecoin = Dogecoin.reindex(range(Dogecoin.index[0], Dogecoin.index[-1], 60), method = 'pad')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:33.396425Z","iopub.execute_input":"2022-01-15T08:54:33.39667Z","iopub.status.idle":"2022-01-15T08:54:35.286125Z","shell.execute_reply.started":"2022-01-15T08:54:33.396637Z","shell.execute_reply":"2022-01-15T08:54:35.28535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ETH = Ethereum.copy()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:35.287529Z","iopub.execute_input":"2022-01-15T08:54:35.287768Z","iopub.status.idle":"2022-01-15T08:54:35.299031Z","shell.execute_reply.started":"2022-01-15T08:54:35.287734Z","shell.execute_reply":"2022-01-15T08:54:35.298315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ETH = ETH.Close.fillna(method = 'pad')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:35.300578Z","iopub.execute_input":"2022-01-15T08:54:35.300912Z","iopub.status.idle":"2022-01-15T08:54:35.307121Z","shell.execute_reply.started":"2022-01-15T08:54:35.300876Z","shell.execute_reply":"2022-01-15T08:54:35.306253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **4. Data Preprocessing for Deep Learning Model**\n## Split the train_data and test_data\nI prepare the bit_train_data and bit_test_data.\nthe first 8 of 19 is bit_train_data, remaining the data as bit_test_data\n","metadata":{}},{"cell_type":"code","source":"import math\nETH_Target = ETH.values\ntrain_len = math.ceil(len(ETH_Target) * 0.8)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:35.308832Z","iopub.execute_input":"2022-01-15T08:54:35.309564Z","iopub.status.idle":"2022-01-15T08:54:35.315889Z","shell.execute_reply.started":"2022-01-15T08:54:35.309525Z","shell.execute_reply":"2022-01-15T08:54:35.315153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ETH_Target = ETH_Target.reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:35.318242Z","iopub.execute_input":"2022-01-15T08:54:35.318965Z","iopub.status.idle":"2022-01-15T08:54:35.324485Z","shell.execute_reply.started":"2022-01-15T08:54:35.31893Z","shell.execute_reply":"2022-01-15T08:54:35.323799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scaler the train, test data\n\nI ues the MinMaxScaler method to scale the data","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nNormalize = MinMaxScaler(feature_range=(0, 1))\nETH_Target = Normalize.fit_transform(ETH_Target)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:35.327915Z","iopub.execute_input":"2022-01-15T08:54:35.328127Z","iopub.status.idle":"2022-01-15T08:54:35.336497Z","shell.execute_reply.started":"2022-01-15T08:54:35.328091Z","shell.execute_reply":"2022-01-15T08:54:35.335848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ETH_train_data, ETH_test_data = ETH_Target[0:train_len, :], ETH_Target[train_len:len(ETH_Target), :1]","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:35.337842Z","iopub.execute_input":"2022-01-15T08:54:35.338206Z","iopub.status.idle":"2022-01-15T08:54:35.342676Z","shell.execute_reply.started":"2022-01-15T08:54:35.338105Z","shell.execute_reply":"2022-01-15T08:54:35.341922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare the Time series data\nThe data from No.0 to No.99 is x_train and predicting No.100's data is y_train","metadata":{}},{"cell_type":"code","source":"x_train = []\ny_train = []\nfor i in range(300, len(ETH_train_data)):\n   x_train.append(ETH_train_data[i-300:i, 0])\n   y_train.append(ETH_train_data[i, 0])\nx_train, y_train = np.array(x_train), np.array(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:35.343988Z","iopub.execute_input":"2022-01-15T08:54:35.344767Z","iopub.status.idle":"2022-01-15T08:54:36.224896Z","shell.execute_reply.started":"2022-01-15T08:54:35.344731Z","shell.execute_reply":"2022-01-15T08:54:36.224016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.reshape(x_train.shape[0], x_train.shape[1], 1)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:36.229558Z","iopub.execute_input":"2022-01-15T08:54:36.231539Z","iopub.status.idle":"2022-01-15T08:54:36.237474Z","shell.execute_reply.started":"2022-01-15T08:54:36.231491Z","shell.execute_reply":"2022-01-15T08:54:36.236771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The data from No.0 to No.99 is x_test and predicting No.100's data is y_test","metadata":{}},{"cell_type":"code","source":"x_test = []\ny_test = []\nfor i in range(300, len(ETH_test_data)):\n   x_test.append(ETH_test_data[i-300:i, 0])\n   y_test.append(ETH_test_data[i, 0])\nx_test, y_test = np.array(x_test), np.array(y_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:36.239025Z","iopub.execute_input":"2022-01-15T08:54:36.239619Z","iopub.status.idle":"2022-01-15T08:54:36.451518Z","shell.execute_reply.started":"2022-01-15T08:54:36.239583Z","shell.execute_reply":"2022-01-15T08:54:36.450797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test = x_test.reshape(x_test.shape[0], x_test.shape[1], 1)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:36.452812Z","iopub.execute_input":"2022-01-15T08:54:36.453053Z","iopub.status.idle":"2022-01-15T08:54:36.458818Z","shell.execute_reply.started":"2022-01-15T08:54:36.453021Z","shell.execute_reply":"2022-01-15T08:54:36.458032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-15T08:54:36.460276Z","iopub.execute_input":"2022-01-15T08:54:36.461277Z","iopub.status.idle":"2022-01-15T08:54:36.468208Z","shell.execute_reply.started":"2022-01-15T08:54:36.461238Z","shell.execute_reply":"2022-01-15T08:54:36.467257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **5. Deep Learning LSTM Model**","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, LSTM, Dropout, Conv1D, MaxPooling1D, TimeDistributed, Flatten\nfrom tensorflow.keras.utils import plot_model\nmodel = Sequential()\nmodel.add(LSTM(40, return_sequences=True, input_shape=(x_train.shape[1], 1)))\nmodel.add(LSTM(40, return_sequences=True))\nmodel.add(LSTM(40))\nmodel.add(Dense(30))\nmodel.add(Dense(1))\nmodel.compile(loss = 'mse', optimizer = 'adam')\nhistory = model.fit(x_train,y_train, batch_size = 512 , epochs = 20)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:01:03.856715Z","iopub.execute_input":"2022-01-15T09:01:03.856977Z","iopub.status.idle":"2022-01-15T09:15:43.266177Z","shell.execute_reply.started":"2022-01-15T09:01:03.856948Z","shell.execute_reply":"2022-01-15T09:15:43.265405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set()\ndf_history = pd.DataFrame(history.history)\nsns.lineplot(x = df_history.index, y = df_history.loss)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:21:47.891659Z","iopub.execute_input":"2022-01-15T09:21:47.891929Z","iopub.status.idle":"2022-01-15T09:21:48.18983Z","shell.execute_reply.started":"2022-01-15T09:21:47.891899Z","shell.execute_reply":"2022-01-15T09:21:48.189152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **6. The Comparsion between Reality and Pridiction data**\n\n#### After MinMaxScaler comparsion between prediction(y_pred) and reality(y_test)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ny_pred = model.predict(x_test)\nplt.scatter(y_test, y_pred)\nplt.plot([x for x in range(2)], [x for x in range(2)], color = 'r')\nplt.xlabel(\"Reality MinMax\")\nplt.ylabel(\"Predicted MinMax\")\nplt.title('ETH')\nplt.show()\nplt.clf()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:21:50.48243Z","iopub.execute_input":"2022-01-15T09:21:50.48295Z","iopub.status.idle":"2022-01-15T09:22:21.766891Z","shell.execute_reply.started":"2022-01-15T09:21:50.482913Z","shell.execute_reply":"2022-01-15T09:22:21.766165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### No MinMaxScaler comparsion between prediction(y_pred) price and reality(y_test) price","metadata":{}},{"cell_type":"code","source":"y_pred = model.predict(x_test)\ny_test = y_test.reshape(-1, 1)\ny_pred = Normalize.inverse_transform(y_pred)\ny_test = Normalize.inverse_transform(y_test)\ny_pred = y_pred.reshape(len(y_pred), 1)\ny_test = y_test.reshape(len(y_pred), 1)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:22:21.768554Z","iopub.execute_input":"2022-01-15T09:22:21.768873Z","iopub.status.idle":"2022-01-15T09:23:02.89083Z","shell.execute_reply.started":"2022-01-15T09:22:21.768835Z","shell.execute_reply":"2022-01-15T09:23:02.889996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(y_test, y_pred)\nplt.plot([1000*x for x in range(0, 8)], [1000*x for x in range(0, 8)], color = 'r')\nplt.xlabel(\"Reality Prices\")\nplt.ylabel(\"Predicted prices\")\nplt.title('ETH')\nplt.show()\nplt.clf()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:23:02.891997Z","iopub.execute_input":"2022-01-15T09:23:02.892257Z","iopub.status.idle":"2022-01-15T09:23:03.281792Z","shell.execute_reply.started":"2022-01-15T09:23:02.892225Z","shell.execute_reply":"2022-01-15T09:23:03.280898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred = pd.DataFrame(y_pred, columns = ['Prediction'], index = ETH.index[train_len +300:])\ndf_pred['Reality'] = y_test","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:00:53.653234Z","iopub.status.idle":"2022-01-15T09:00:53.653846Z","shell.execute_reply.started":"2022-01-15T09:00:53.653601Z","shell.execute_reply":"2022-01-15T09:00:53.653626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:00:53.654997Z","iopub.status.idle":"2022-01-15T09:00:53.6556Z","shell.execute_reply.started":"2022-01-15T09:00:53.655335Z","shell.execute_reply":"2022-01-15T09:00:53.655382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def log_return(close):\n    return np.log(close).diff(periods = 15)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:00:53.656728Z","iopub.status.idle":"2022-01-15T09:00:53.657333Z","shell.execute_reply.started":"2022-01-15T09:00:53.657072Z","shell.execute_reply":"2022-01-15T09:00:53.657099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ETH_Target[train_len:len(ETH_Target), :1]","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:00:53.658519Z","iopub.status.idle":"2022-01-15T09:00:53.659073Z","shell.execute_reply.started":"2022-01-15T09:00:53.658842Z","shell.execute_reply":"2022-01-15T09:00:53.658866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred['Log_return_prediction'] = log_return(df_pred['Prediction'])[15: ]\ndf_pred['Log_return_reality'] = log_return(df_pred['Reality'])[15: ]","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:00:53.660213Z","iopub.status.idle":"2022-01-15T09:00:53.660819Z","shell.execute_reply.started":"2022-01-15T09:00:53.660574Z","shell.execute_reply":"2022-01-15T09:00:53.6606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,10))\nplt.plot(df_pred['Log_return_prediction'], color = 'r' , label = 'Prediction', alpha = 1)\nplt.plot(df_pred['Log_return_reality'], color = 'g', label = 'Reality', alpha = 0.7)\nplt.title('Log_Return Between the Prediction and Reality')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:00:53.661956Z","iopub.status.idle":"2022-01-15T09:00:53.662565Z","shell.execute_reply.started":"2022-01-15T09:00:53.662294Z","shell.execute_reply":"2022-01-15T09:00:53.66232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### the Bitcoin prediction and reality comparsion","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (30, 15))\nsns.lineplot(x = ETH.index[-100000:], y = ETH.values[-100000:], label = f'The ETH Close')\nsns.lineplot(x = df_pred.index, y = df_pred.Prediction, label = 'Prediction', color = 'r')\nsns.lineplot(x = df_pred.index, y = df_pred.Reality, label = 'Reality', color = 'g')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T09:00:53.663706Z","iopub.status.idle":"2022-01-15T09:00:53.664289Z","shell.execute_reply.started":"2022-01-15T09:00:53.664044Z","shell.execute_reply":"2022-01-15T09:00:53.664069Z"},"trusted":true},"execution_count":null,"outputs":[]}]}