{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport seaborn as sns\nfrom pandas.plotting import register_matplotlib_converters\n\nregister_matplotlib_converters()\nsns.set_style(\"darkgrid\")\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nRUN = False\nBASE_PATH = '/kaggle/input/g-research-crypto-forecasting/'\nNROWS = 1e8\n\nif RUN:\n    train = pd.read_csv(f'{BASE_PATH}train.csv')\nelse:\n    train = pd.read_csv(f'{BASE_PATH}train.csv', nrows=NROWS)\n    \ndetails   = pd.read_csv(f'{BASE_PATH}asset_details.csv')\nsup_train = pd.read_csv(f'{BASE_PATH}supplemental_train.csv')\nexp_test  = pd.read_csv(f'{BASE_PATH}example_test.csv')\nsub       = pd.read_csv(f'{BASE_PATH}example_sample_submission.csv')\n\nimport statsmodels.api as sm\nfrom statsmodels.tsa.seasonal import seasonal_decompose, STL\nfrom statsmodels.tsa.api import ExponentialSmoothing, SimpleExpSmoothing, Holt\nfrom statsmodels.tsa.arima.model import ARIMA \n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error as MAE\n\nimport matplotlib.pyplot as plt\n\n\nimport xgboost as xgb\n\nimport tensorflow as tf\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-28T02:04:21.926572Z","iopub.execute_input":"2021-12-28T02:04:21.927365Z","iopub.status.idle":"2021-12-28T02:04:24.676046Z","shell.execute_reply.started":"2021-12-28T02:04:21.927329Z","shell.execute_reply":"2021-12-28T02:04:24.675241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TODO\n\n- Eliminate some of the features, too large\n- Write out a input file\n- LSTM\n- working time-series model\n- XGBoost","metadata":{}},{"cell_type":"markdown","source":"## Augmenting Features\n\nAdding some columns using the existing ones to increase the number of features. Some of these columns might be deleted in case they don't seem usefull.","metadata":{}},{"cell_type":"code","source":"# Features added for all the coins\ntrain['Close/Open']   = train['Close'] / train['Open']\ntrain['Close - Open'] = train['Close'] - train['Open']\ntrain['High/Low']     = train['High'] / train['Low']\ntrain['High - Low']   = train['High'] - train['Low']\ntrain['High - Close'] = train['High'] - train['Close']\ntrain['High - Open']  = train['High'] - train['Open']","metadata":{"execution":{"iopub.status.busy":"2021-12-28T02:04:24.677913Z","iopub.execute_input":"2021-12-28T02:04:24.678155Z","iopub.status.idle":"2021-12-28T02:04:24.689760Z","shell.execute_reply.started":"2021-12-28T02:04:24.678123Z","shell.execute_reply":"2021-12-28T02:04:24.689090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utilities\n\n- coin: Gets a specific crypto based on its ID.\n- normalize_col: Normalizes a column etiher based on max or std.\n- xgb_importance_plot: Plots the importance of features while using xgb, useful for feature pruning.","metadata":{}},{"cell_type":"code","source":"# Useful functions\ndef normalize_col(col, T='std'):\n    if T == 'max': return (col.max() - col) / (col.max() - col.min())\n    return (col - col.mean()) / col.std()\n\ndef xgb_importance_plot(xgb_model):\n    xgb.plot_importance(xgb_model, max_num_features=18, importance_type='gain'  ,title='gain: average gain for splits', show_values=False)\n    xgb.plot_importance(xgb_model, max_num_features=18, importance_type='cover' ,title='cover: # of samples affected by it', show_values=False)\n    xgb.plot_importance(xgb_model, max_num_features=18, importance_type='weight',title='weight: # of appearance', show_values=False)\n\ndef coin(coind_id):\n    ''' Gets a specific crypto based on its ID. '''\n    coin = train[train['Asset_ID'] == coind_id] # Get the coin\n    coin['time'] = pd.to_datetime(coin['timestamp'], unit='s') # Change the time format\n    # Uncommented to see if the STL will work or not?\n#     coin.set_index('time', inplace=True) # Set the index\n    coin['Target'].fillna(method='ffill', inplace=True) # Fill the missing values using forward filling\n    \n    # Some columns have high means, and makes sense to push them into 0 to 1 interval\n    # (Min-Max) Normalizing \n    col_norm = ['High', 'Low', 'Open', 'Close', 'VWAP', 'Volume']\n\n    for col in col_norm:\n        coin.loc[:, col] = normalize_col(coin.loc[:, col], 'max')\n    \n    # (Std) Normalization\n    col_std = ['Count', 'Close - Open', 'Close/Open', 'High/Low', 'High - Low', 'High - Open', 'High - Close']\n    \n    for col in col_std:\n        coin[f'{col}_z'] = normalize_col(coin.loc[:, col]) \n    \n    # Adding the Lag features\n    coin.loc[:, 'vwap_lag']    = coin.loc[:,'VWAP'].shift(1).fillna(0)\n    coin.loc[:, 'vol_lag']     = coin.loc[:,'Volume'].shift(1).fillna(0)\n    coin.loc[:, 'count_lag']   = coin.loc[:,'Count'].shift(1).fillna(0)\n    coin.loc[:, 'count_z_lag'] = coin.loc[:,'Count_z'].shift(1).fillna(0)\n    \n    coin['target_z'] = normalize_col(coin.loc[:, 'Target'])\n    \n    return coin\n\n\"\"\" BTC and ETH are the main players, so the analysis\nwill be done on these two crptos. \"\"\"\nbtc = coin(0)\n# btcc = coin(2)\n# eth = coin(6)\n# dog = coin(4)\n# ada = coin(3)","metadata":{"execution":{"iopub.status.busy":"2021-12-28T02:04:24.691246Z","iopub.execute_input":"2021-12-28T02:04:24.691672Z","iopub.status.idle":"2021-12-28T02:04:24.727116Z","shell.execute_reply.started":"2021-12-28T02:04:24.691638Z","shell.execute_reply":"2021-12-28T02:04:24.726456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dese(df, day=1):\n    DAY = day\n    PERIOD = 60\n    SEASONAL = PERIOD * 24 * DAY + 1\n    TREND = PERIOD * 12 + 1\n    SEASON_JUMP = int(SEASONAL / 2)\n\n    s = STL(df['Target'], period=PERIOD, seasonal=SEASONAL, trend=TREND, seasonal_jump=SEASON_JUMP, robust=True)\n    r = s.fit()\n    \n    df[f't-ses-stl{DAY}'] = df['Target'] - r.seasonal\n    df[f't-trend-stl{DAY}'] = df['Target'] - r.trend\n    df[f't-ses-trend-stl{DAY}'] = df['Target'] - r.seasonal - r.trend","metadata":{"execution":{"iopub.status.busy":"2021-12-28T02:04:24.730057Z","iopub.execute_input":"2021-12-28T02:04:24.730348Z","iopub.status.idle":"2021-12-28T02:04:24.737210Z","shell.execute_reply.started":"2021-12-28T02:04:24.730317Z","shell.execute_reply":"2021-12-28T02:04:24.736446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dese(btc)\ndese(btc, 2)\ndese(btc, 3)","metadata":{"execution":{"iopub.status.busy":"2021-12-28T02:04:24.739468Z","iopub.execute_input":"2021-12-28T02:04:24.739905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# r = s.fit()\n# r.plot()\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc[btc.filter(regex='t-|Target').columns.to_list()].reset_index().to_csv('d.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('/kaggle/working/d.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc[btc.filter(regex='t-|Target').columns.to_list()].corr()['Target']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(8, 6), dpi=80)\n\n# plt.plot(btc['target_z'], label='Bitcoin')\n# plt.plot(eth['target_z'], label='Etherum')\n# plt.plot(dog['target_z'], label='Dodge')\n# plt.plot(ada['target_z'], label='Cardano')\n\n# plt.savefig('coin-dependence-on-coin.png')\n# plt.legend()\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# s = seasonal_decompose(p.loc[:60*12,'target_z'], model='add', period=60)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# s.plot()\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TMEAN = btc['Target'].mean()\n# TSTD  = btc['Target'].std()\n# TMIN = btc['Target'].min()\n# TMAX = btc['Target'].max()\n\n# # btc['Target'] = (TMAX - btc['Target']) / (TMAX - TMIN)\n# btc['Target'] = (btc['Target'] - TMEAN) / TSTD","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Try running LSTM model with seasonality and trend, then try two versions one without seasonality one without trend.\n- Don't use lag features when running on detime version of the data","metadata":{}},{"cell_type":"code","source":"def detime(df, day=1):\n    # 3 Day detrending\n    ranging = int(day * 24 * 60)\n    \n    # Initializing the columns\n    df[f'Target-trend{day}'] = df[f'Target-seasonal{day}'] = df[f'Target-seasonal-trend{day}'] = 0\n\n    num_intervals = int(df.shape[0] / ranging)\n    \n    for i in range(num_intervals):\n        # Start and End indices\n        si = i * ranging\n        ei = (i + 1) * ranging\n        \n        # Last interval\n        if i == num_intervals - 1: ei = df.shape[0]\n\n        s = seasonal_decompose(df.loc[si:ei, 'Target'], model='add', period=60)\n#         s.plot()\n        \n        df.loc[si:ei, f'Target-trend{day}']    = df.loc[si:ei, 'Target'] - s.trend.fillna(0)\n        df.loc[si:ei, f'Target-seasonal{day}'] = df.loc[si:ei, 'Target'] - s.seasonal.fillna(0)\n        df.loc[si:ei, f'Target-seasonal-trend{day}'] = df.loc[si:ei, 'Target'] - s.trend.fillna(0) - s.seasonal.fillna(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detime(btc, 3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axes = plt.subplots(1, 2, figsize=(15,4))\n\n# fig = sm.graphics.tsa.plot_acf(btc['Target'], lags=40, ax=axes[0])\n# fig = sm.graphics.tsa.plot_pacf(btc['Target'], lags=40, ax=axes[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Statitionary Test\n\nadf:\n- Null Hypothesis: The series has a unit root.\n- Alternate Hypothesis: The series has no unit root.\n- Note: If the null hypothesis in failed to be rejected, this test may provide evidence that the series is non-stationary.\n\nkpss:\n- Null Hypothesis: The process is trend stationary.\n- Alternate Hypothesis: The series has a unit root (series is not stationary).\n- Note: Oppostie of ADF\n\nBased on the test, data is not stationary.","metadata":{}},{"cell_type":"code","source":"# Reference: https://www.statsmodels.org/stable/examples/notebooks/generated/stationarity_detrending_adf_kpss.html\nfrom statsmodels.tsa.stattools import kpss\nfrom statsmodels.tsa.stattools import adfuller\n\ndef adf_test(timeseries):\n    print(\"Results of Dickey-Fuller Test:\")\n    dftest = adfuller(timeseries, autolag=\"AIC\")\n    dfoutput = pd.Series(\n        dftest[0:4],\n        index=[\n            \"Test Statistic\",\n            \"p-value\",\n            \"#Lags Used\",\n            \"Number of Observations Used\",\n        ],\n    )\n    for key, value in dftest[4].items():\n        dfoutput[\"Critical Value (%s)\" % key] = value\n    print(dfoutput)\n    \ndef kpss_test(timeseries):\n    print(\"Results of KPSS Test:\")\n    kpsstest = kpss(timeseries, regression=\"c\", nlags=\"auto\")\n    kpss_output = pd.Series(\n        kpsstest[0:3], index=[\"Test Statistic\", \"p-value\", \"Lags Used\"]\n    )\n    for key, value in kpsstest[3].items():\n        kpss_output[\"Critical Value (%s)\" % key] = value\n    print(kpss_output)\n    \n# adf_test(btc['Target'])\n# kpss_test(btc['Target'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering\n\nUsing XGBoost:\n- Which columns are important for each coin\n- Which columns are important in general when ran on all of the data","metadata":{}},{"cell_type":"code","source":"# X = btc.drop(['Target', 'Asset_ID', 'timestamp', 'time'], axis=1)\n# y = btc[\"Target\"]\n    \n# X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=24, shuffle=False)\n\n# dtrain = xgb.DMatrix(X_train, label=y_train)\n# dval   = xgb.DMatrix(X_val, label=y_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n    'eta': 1e-1,\n    'max_depth': 10,\n    'objective': 'reg:squarederror',\n    'subsample': 0.9, # Avoiding overfitting\n    'tree_method': 'gpu_hist',\n    'verbosity': 0\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# m = xgb.train(\n#     params, \n#     dtrain, \n#     num_boost_round=200, \n#     early_stopping_rounds=10, \n#     evals=[(dtrain, 'train'), (dval, 'eval')],\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgb_importance_plot(m)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = btc.drop(['Target', 'Target-trend3', 'Target-seasonal3', 'Target-seasonal-trend3','Asset_ID', 'timestamp', 'time'], axis=1)\n# y = btc[\"Target-seasonal-trend3\"]\n    \n# X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=24, shuffle=False)\n\n# linear = tf.keras.Sequential([\n#     tf.keras.layers.Dense(units=128),\n#     tf.keras.layers.Dense(units=128),\n#     tf.keras.layers.Dense(units=64),\n#     tf.keras.layers.Dense(units=1)\n# ])\n\n# linear.compile(loss=tf.losses.MeanSquaredError(),\n#               optimizer=tf.optimizers.Adam(1e-2),\n#               metrics=[tf.metrics.MeanAbsoluteError()])\n\n# history_linear = linear.fit(X_train, y_train, epochs=100, callbacks=[early_stopping], validation_data=(X_val, y_val))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.DataFrame(history_linear.history).plot(y=['loss', 'val_loss'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seq_len = 60 * 12\n# seg_stride = 60 * 6\n# batch_size= 32\n\n# train_data = tf.keras.preprocessing.timeseries_dataset_from_array(\n#           data=np.array(X_train),\n#           targets=np.array(y_train),\n#           sequence_length=seq_len,\n#           sequence_stride=seg_stride,\n#           shuffle=True,\n#           batch_size=batch_size,)\n\n# val_data = tf.keras.preprocessing.timeseries_dataset_from_array(\n#           data=np.array(X_val),\n#           targets=np.array(y_val),\n#           sequence_length=seq_len,\n#           sequence_stride=seg_stride,\n#           shuffle=True,\n#           batch_size=batch_size,)\n\n# test_data = tf.keras.preprocessing.timeseries_dataset_from_array(\n#           data=np.array(X_val),\n#     targets=None,\n#           sequence_length=seq_len,\n#           sequence_stride=seg_stride,\n#           shuffle=True,\n#           batch_size=batch_size,)\n\n# early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n#                                                     patience=30,\n#                                                     mode='min')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lstm = tf.keras.models.Sequential([\n#     # Shape [batch, time, features] => [batch, time, lstm_units]\n#     tf.keras.layers.LSTM(32, return_sequences=True),\n#     tf.keras.layers.LSTM(64, return_sequences=True),\n#     tf.keras.layers.LSTM(128, return_sequences=True),\n#     # Shape => [batch, time, features]\n#     tf.keras.layers.Dense(units=1)\n# ])\n\n# lstm.compile(loss=tf.losses.MeanSquaredError(),\n#               optimizer=tf.optimizers.Adam(),\n#               metrics=[tf.metrics.MeanAbsoluteError()])\n\n# history_lstm = lstm.fit(train_data, epochs=5, callbacks=[early_stopping], validation_data=val_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tf.keras.utils.plot_model(lstm,show_shapes=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"# import gresearch_crypto\n# env = gresearch_crypto.make_env()\n# iter_test = env.iter_test()\n# for (test_df, sample_prediction_df) in iter_test:\n#     sample_prediction_df['Target'] = 0\n#     env.predict(sample_prediction_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}