{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"No TA-Lib, so I will have to make functions to calculate technical indicators myself.","metadata":{}},{"cell_type":"markdown","source":"## Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.linear_model import LinearRegression\nfrom lightgbm import LGBMRegressor\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport gresearch_crypto","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-20T03:23:27.337533Z","iopub.execute_input":"2021-11-20T03:23:27.337802Z","iopub.status.idle":"2021-11-20T03:23:29.134288Z","shell.execute_reply.started":"2021-11-20T03:23:27.337736Z","shell.execute_reply":"2021-11-20T03:23:29.133354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"train_csv = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nasset_details_csv = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:23:29.135774Z","iopub.execute_input":"2021-11-20T03:23:29.136056Z","iopub.status.idle":"2021-11-20T03:23:29.141545Z","shell.execute_reply.started":"2021-11-20T03:23:29.136024Z","shell.execute_reply":"2021-11-20T03:23:29.140711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(train_csv)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:23:29.142674Z","iopub.execute_input":"2021-11-20T03:23:29.143012Z","iopub.status.idle":"2021-11-20T03:24:22.407924Z","shell.execute_reply.started":"2021-11-20T03:23:29.142980Z","shell.execute_reply":"2021-11-20T03:24:22.407306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:22.409938Z","iopub.execute_input":"2021-11-20T03:24:22.410493Z","iopub.status.idle":"2021-11-20T03:24:22.421463Z","shell.execute_reply.started":"2021-11-20T03:24:22.410460Z","shell.execute_reply":"2021-11-20T03:24:22.420480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(asset_details_csv)\ndf_asset_details.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:22.422885Z","iopub.execute_input":"2021-11-20T03:24:22.423127Z","iopub.status.idle":"2021-11-20T03:24:22.451673Z","shell.execute_reply.started":"2021-11-20T03:24:22.423098Z","shell.execute_reply":"2021-11-20T03:24:22.450768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = df_asset_details.sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:22.453111Z","iopub.execute_input":"2021-11-20T03:24:22.453428Z","iopub.status.idle":"2021-11-20T03:24:22.469900Z","shell.execute_reply.started":"2021-11-20T03:24:22.453379Z","shell.execute_reply":"2021-11-20T03:24:22.468279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Missing Data\n\n","metadata":{}},{"cell_type":"code","source":"df_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:22.471416Z","iopub.execute_input":"2021-11-20T03:24:22.472071Z","iopub.status.idle":"2021-11-20T03:24:22.949629Z","shell.execute_reply.started":"2021-11-20T03:24:22.472024Z","shell.execute_reply":"2021-11-20T03:24:22.948780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_dict = {}\nfor asset_id in df_asset_details['Asset_ID']:\n    asset_dict[asset_id] = df_train[df_train['Asset_ID']==asset_id].dropna()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:22.950939Z","iopub.execute_input":"2021-11-20T03:24:22.951200Z","iopub.status.idle":"2021-11-20T03:24:28.011176Z","shell.execute_reply.started":"2021-11-20T03:24:22.951158Z","shell.execute_reply":"2021-11-20T03:24:28.010128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in asset_dict.keys():\n    print('Missing Values for Asset '+str(key)+':', asset_dict[key].isna().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.012465Z","iopub.execute_input":"2021-11-20T03:24:28.012724Z","iopub.status.idle":"2021-11-20T03:24:28.472808Z","shell.execute_reply.started":"2021-11-20T03:24:28.012692Z","shell.execute_reply":"2021-11-20T03:24:28.472015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in asset_dict.keys():\n    print('Observations for Asset '+str(key)+':', asset_dict[key].shape[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.473995Z","iopub.execute_input":"2021-11-20T03:24:28.474235Z","iopub.status.idle":"2021-11-20T03:24:28.482517Z","shell.execute_reply.started":"2021-11-20T03:24:28.474206Z","shell.execute_reply":"2021-11-20T03:24:28.481888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering\n\nThe tutorial notebook has two suggested engineered features: upper_shadow and lower_shadow\n\nThese are the distance from the high(low) to the maximum(minumum) of the open or close price, like in a candlestick chart.\n\nWe are also going to engineer a daily_range feature, which is the difference between the high and low.","metadata":{}},{"cell_type":"code","source":"def upper_shadow(df):\n    return df['High'] - np.maximum(df['Open'], df['Close'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Open'], df['Close']) - df['Low']\n\ndef daily_range(df):\n    return df['High'] - df['Low']","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.483696Z","iopub.execute_input":"2021-11-20T03:24:28.483934Z","iopub.status.idle":"2021-11-20T03:24:28.492916Z","shell.execute_reply.started":"2021-11-20T03:24:28.483904Z","shell.execute_reply":"2021-11-20T03:24:28.492240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are going to use technical indicators, and I am going to start with Stochastic Indicator, RSI, ATR, and MACD. These are commonly used indicators, because they have been shown to be reliably useful. The group has two momentum indicators, a volatility indicator, and a trend indicator. Each of these has a hyperparameter that will need to be optimized.","metadata":{}},{"cell_type":"code","source":"test_df = asset_dict[0]\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.493919Z","iopub.execute_input":"2021-11-20T03:24:28.494236Z","iopub.status.idle":"2021-11-20T03:24:28.516877Z","shell.execute_reply.started":"2021-11-20T03:24:28.494206Z","shell.execute_reply":"2021-11-20T03:24:28.516106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def stochastic_indicator(high, low, close, window=14, smooth_window=3):\n    \"\"\" calculates stochastic oscillator for rolling window, smoothed by smooth_window SMA \"\"\"\n    def stoch_k(high, low, close, window):\n        window_high = high.rolling(window).max()\n        window_low = low.rolling(window).min()\n        return round((close - window_low) / (window_high - window_low) * 100)\n    \n    def stoch_d(stoch_k, smooth_window):\n        return stoch_k.rolling(smooth_window).mean()\n    \n    k = stoch_k(high, low, close, window)\n    return k, stoch_d(k, smooth_window)\n        \n\ndef add_stochastic_to_df(df, window=14, smooth_window=3):\n        df['STOCH'], df['STOCH_signal'] = stochastic_indicator(df['High'], df['Low'], df['Close'], \n                                                               window=window, smooth_window=smooth_window)\n        return df.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.520565Z","iopub.execute_input":"2021-11-20T03:24:28.521388Z","iopub.status.idle":"2021-11-20T03:24:28.530501Z","shell.execute_reply.started":"2021-11-20T03:24:28.521348Z","shell.execute_reply":"2021-11-20T03:24:28.529895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rsi(close, window):\n    close = close.diff()\n    upward_series = pd.Series([np.nan] * len(close), index=close.index)\n    upward_series[close > 0] = close[close > 0]\n    \n    downward_series = pd.Series([np.nan] * len(close), index=close.index)\n    downward_series[close < 0] = close[close < 0]\n    \n    rolling_up = upward_series.rolling(window, min_periods=1).mean()\n    rolling_down = downward_series.rolling(window, min_periods=1).mean()\n    \n    return 100 - ( 100 / ( 1 + (rolling_up / abs(rolling_down))  ) )\n    \n\ndef add_rsi_to_df(df, window=14):\n    df['RSI'] = rsi(df['Close'], window=window)\n    return df.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.531348Z","iopub.execute_input":"2021-11-20T03:24:28.531582Z","iopub.status.idle":"2021-11-20T03:24:28.546470Z","shell.execute_reply.started":"2021-11-20T03:24:28.531542Z","shell.execute_reply":"2021-11-20T03:24:28.545739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def atr(high, low, close, window):\n    high = high.values\n    low = low.values\n    close = close.values\n    \n    atr = [max([high[i] - low[i], abs( high[i] - close[i-1] ), abs( low[i] - close[i-1] )]) for i in range(1, len(high))]\n    atr.insert(0, high[0] - low[0])\n    atr[1:] = [(atr[i-1] * (window - 1) + atr[i]) / window for i in range(1, len(atr))]\n    return np.array(atr)\n\ndef add_atr_to_df(df, window=14):\n    df['ATR'] = atr(df['High'], df['Low'], df['Close'], window=window)\n    return df.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.547585Z","iopub.execute_input":"2021-11-20T03:24:28.547924Z","iopub.status.idle":"2021-11-20T03:24:28.559133Z","shell.execute_reply.started":"2021-11-20T03:24:28.547880Z","shell.execute_reply":"2021-11-20T03:24:28.558177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def macd(close, window_slow, window_fast, window_signal):\n    slow_ewm = close.ewm(span=window_slow, adjust=False).mean()\n    fast_ewm = close.ewm(span=window_fast, adjust=False).mean()\n    macd = slow_ewm - fast_ewm\n    macd_signal = macd.ewm(span=window_signal, adjust=False).mean()\n    return macd, macd_signal\n\ndef add_macd_to_df(df, window_slow=26, window_fast=12, window_signal=9):\n    df['MACD'], df['MACD_signal'] = macd(df['Close'], window_slow, window_fast, window_signal)\n    return df.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.560753Z","iopub.execute_input":"2021-11-20T03:24:28.561277Z","iopub.status.idle":"2021-11-20T03:24:28.571631Z","shell.execute_reply.started":"2021-11-20T03:24:28.561230Z","shell.execute_reply":"2021-11-20T03:24:28.570890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we need a function that processes the data to add the features and separate the targets.","metadata":{}},{"cell_type":"code","source":"def get_features(df, STOCH_window, STOCH_smooth_window, RSI_window, ATR_window, \n                            MACD_window_slow, MACD_window_fast, MACD_window_sign):\n    df_features = df[['Open', 'High', 'Low', 'Close', 'Count', 'Volume', 'VWAP']].copy()\n    df_features['upper_shadow'] = upper_shadow(df_features)\n    df_features['lower_shadow'] = lower_shadow(df_features)\n    df_features['daily_range'] = daily_range(df_features)\n    df_features = add_stochastic_to_df(df_features, STOCH_window, STOCH_smooth_window)\n    df_features = add_rsi_to_df(df_features, RSI_window)\n    df_features = add_atr_to_df(df_features, ATR_window)\n    df_features = add_macd_to_df(df_features, MACD_window_slow, MACD_window_fast, MACD_window_sign)\n    df_features = df_features.dropna()\n    return df_features.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.572764Z","iopub.execute_input":"2021-11-20T03:24:28.573167Z","iopub.status.idle":"2021-11-20T03:24:28.587293Z","shell.execute_reply.started":"2021-11-20T03:24:28.573130Z","shell.execute_reply":"2021-11-20T03:24:28.586638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_and_target(df, STOCH_window, STOCH_smooth_window, RSI_window, ATR_window, \n                            MACD_window_slow, MACD_window_fast, MACD_window_sign):\n    df_features = df[['Open', 'High', 'Low', 'Close', 'Count', 'Volume', 'VWAP', 'Target']].copy()\n    df_features['upper_shadow'] = upper_shadow(df_features)\n    df_features['lower_shadow'] = lower_shadow(df_features)\n    df_features['daily_range'] = daily_range(df_features)\n    df_features = add_stochastic_to_df(df_features, STOCH_window, STOCH_smooth_window)\n    df_features = add_rsi_to_df(df_features, RSI_window)\n    df_features = add_atr_to_df(df_features, ATR_window)\n    df_features = add_macd_to_df(df_features, MACD_window_slow, MACD_window_fast, MACD_window_sign)\n    df_features = df_features.dropna()\n    return df_features.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.589547Z","iopub.execute_input":"2021-11-20T03:24:28.590064Z","iopub.status.idle":"2021-11-20T03:24:28.600555Z","shell.execute_reply.started":"2021-11-20T03:24:28.590017Z","shell.execute_reply":"2021-11-20T03:24:28.599753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def separate_xy(df, STOCH_window, STOCH_smooth_window, RSI_window, ATR_window, \n                MACD_window_slow, MACD_window_fast, MACD_window_sign):\n    df = df.replace([np.inf, -np.inf], np.nan).dropna()\n    feats = get_features_and_target(df, STOCH_window, STOCH_smooth_window, RSI_window, ATR_window, \n                            MACD_window_slow, MACD_window_fast, MACD_window_sign).copy()\n    y = feats['Target'].copy()\n    X = feats.drop('Target', axis=1).copy()\n    return X, y","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.602031Z","iopub.execute_input":"2021-11-20T03:24:28.602864Z","iopub.status.idle":"2021-11-20T03:24:28.616043Z","shell.execute_reply.started":"2021-11-20T03:24:28.602797Z","shell.execute_reply":"2021-11-20T03:24:28.615313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binary_target(y):\n    return np.array([1 if i>0 else -1 for i in y])","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.617238Z","iopub.execute_input":"2021-11-20T03:24:28.618126Z","iopub.status.idle":"2021-11-20T03:24:28.628665Z","shell.execute_reply.started":"2021-11-20T03:24:28.618083Z","shell.execute_reply":"2021-11-20T03:24:28.627846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def fit_linreg(X,y):\n#    model = LinearRegression()\n#    model.fit(X,y)\n#    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.630193Z","iopub.execute_input":"2021-11-20T03:24:28.631244Z","iopub.status.idle":"2021-11-20T03:24:28.639239Z","shell.execute_reply.started":"2021-11-20T03:24:28.631189Z","shell.execute_reply":"2021-11-20T03:24:28.638138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_lgbm(X,y):\n    model = LGBMRegressor()\n    model.fit(X,y)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.640170Z","iopub.execute_input":"2021-11-20T03:24:28.640684Z","iopub.status.idle":"2021-11-20T03:24:28.650778Z","shell.execute_reply.started":"2021-11-20T03:24:28.640653Z","shell.execute_reply":"2021-11-20T03:24:28.649968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set Hyperparameters for Indicators","metadata":{}},{"cell_type":"code","source":"## hyperparameters for technical indicators\nSTOCH_window = 14\nSTOCH_smooth_window = 3\n\nRSI_window = 14\n\nATR_window = 14\n\nMACD_window_slow = 21\nMACD_window_fast = 12\nMACD_window_sign = 9","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.651970Z","iopub.execute_input":"2021-11-20T03:24:28.653366Z","iopub.status.idle":"2021-11-20T03:24:28.661968Z","shell.execute_reply.started":"2021-11-20T03:24:28.653313Z","shell.execute_reply":"2021-11-20T03:24:28.661066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit Linear Regression Models\n\nWe want to cycle through assets, in the asset_dict, and fit a linear regression models to each.","metadata":{}},{"cell_type":"code","source":"#Xs = {}\n#ys = {}\n#models = {}\n\n#for asset_id in asset_dict.keys():\n#    print(asset_id, \"Fitting Model for \", df_asset_details.loc[df_asset_details['Asset_ID']==asset_id, \"Asset_Name\"].values)\n#    asset_df = asset_dict[asset_id]\n#    X, y = separate_xy(asset_df, STOCH_window, STOCH_smooth_window, RSI_window, ATR_window, \n#                       MACD_window_slow, MACD_window_fast, MACD_window_sign)\n#    model = fit_linreg(X,y)\n#    \n#    Xs[asset_id] = X\n#    ys[asset_id] = y\n#    models[asset_id] = model","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.664467Z","iopub.execute_input":"2021-11-20T03:24:28.664906Z","iopub.status.idle":"2021-11-20T03:24:28.672201Z","shell.execute_reply.started":"2021-11-20T03:24:28.664855Z","shell.execute_reply":"2021-11-20T03:24:28.671379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit GBM Models\n\nSame as the LinearRegression() models, but using LGBMRegressor().","metadata":{}},{"cell_type":"code","source":"Xs = {}\nys = {}\navg_changes = {}\nmodels = {}\n\nfor asset_id in asset_dict.keys():\n    print(asset_id, \"Fitting Model for \", df_asset_details.loc[df_asset_details['Asset_ID']==asset_id, \"Asset_Name\"].values)\n    asset_df = asset_dict[asset_id]\n    X, y = separate_xy(asset_df, STOCH_window, STOCH_smooth_window, RSI_window, ATR_window, \n                       MACD_window_slow, MACD_window_fast, MACD_window_sign)\n    \n    Xs[asset_id] = X\n    ys[asset_id] = y\n    avg_changes[asset_id] = np.mean(abs(y))\n    y = binary_target(y)\n    model = fit_lgbm(X,y)\n    models[asset_id] = model","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:24:28.673313Z","iopub.execute_input":"2021-11-20T03:24:28.673985Z","iopub.status.idle":"2021-11-20T03:28:59.615494Z","shell.execute_reply.started":"2021-11-20T03:24:28.673953Z","shell.execute_reply":"2021-11-20T03:28:59.614715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_changes","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:28:59.616898Z","iopub.execute_input":"2021-11-20T03:28:59.617812Z","iopub.status.idle":"2021-11-20T03:28:59.626814Z","shell.execute_reply.started":"2021-11-20T03:28:59.617764Z","shell.execute_reply":"2021-11-20T03:28:59.626197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Predictions and Test Submission","metadata":{}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j, row in df_test.iterrows():\n        model = models[row['Asset_ID']]\n        avg_change = avg_changes[row['Asset_ID']]\n        \n        df = asset_dict[row['Asset_ID']]\n        df = df.loc[df['timestamp'] < row['timestamp'], :]\n        df = df.append(row.replace([np.inf, -np.inf], np.nan).dropna())\n        X_test = get_features(df, STOCH_window, STOCH_smooth_window, RSI_window, ATR_window, \n                            MACD_window_slow, MACD_window_fast, MACD_window_sign)\n        y_pred = model.predict(X_test.tail(1))\n        y_pred_scaled = y_pred * avg_change\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred_scaled\n        \n        if i==0 and j==0:\n            display(X_test)\n            \n    if i==0:\n        display(df_pred)\n        \n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-20T03:28:59.627940Z","iopub.execute_input":"2021-11-20T03:28:59.630302Z","iopub.status.idle":"2021-11-20T03:39:09.978834Z","shell.execute_reply.started":"2021-11-20T03:28:59.630250Z","shell.execute_reply":"2021-11-20T03:39:09.977870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}