{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport datatable as dt # Load data faster\nimport seaborn as sns \nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-30T14:22:00.288632Z","iopub.execute_input":"2022-01-30T14:22:00.291082Z","iopub.status.idle":"2022-01-30T14:22:01.525689Z","shell.execute_reply.started":"2022-01-30T14:22:00.290703Z","shell.execute_reply":"2022-01-30T14:22:01.524945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading the data:","metadata":{}},{"cell_type":"code","source":"asset_details = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv', index_col='Asset_ID')\nnames = asset_details.sort_index().Asset_Name.values\nasset_dfs = []\nfor name in names:\n    asset_df = dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-preprocessing-2/{name.lower().replace(' ', '_')}.jay\").to_pandas()\n    \n    # Needed for some feature calculations\n    asset_df['diffs'] = asset_df['Close'] - asset_df['Close'].shift(1)\n    asset_df['signs_diffs'] = np.sign(asset_df['VWAP'] - asset_df['VWAP'].shift(1))\n    \n    asset_dfs.append(asset_df)\n\nmarket = dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-preprocessing-2/market.jay\").to_pandas()\nasset_dfs[1].head(10) # Bitcoin","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:01.527019Z","iopub.execute_input":"2022-01-30T14:22:01.527779Z","iopub.status.idle":"2022-01-30T14:22:17.452262Z","shell.execute_reply.started":"2022-01-30T14:22:01.527743Z","shell.execute_reply":"2022-01-30T14:22:17.451364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=False):\n    for col in df.columns:\n        if df[col].dtype == 'int64':\n            datatypes = [np.int8, np.int16, np.int32]\n            # Go through all possible smaller datatypes and assign the smallest one that can fit all rows.\n            for datatype in datatypes:\n                min_int, max_int = np.iinfo(datatype).min, np.iinfo(datatype).max\n                if df[col].max() <= max_int and df[col].min() >= min_int:\n                    df[col] = df[col].astype(datatype, copy=False)\n                    if verbose:\n                        print(f'Convert {col} to {datatype}.')\n                    break\n        elif df[col].dtype == 'int32':\n            datatypes = [np.int8, np.int16]\n            # Go through all possible smaller datatypes and assign the smallest one that can fit all rows.\n            for datatype in datatypes:\n                min_int, max_int = np.iinfo(datatype).min, np.iinfo(datatype).max\n                if df[col].max() <= max_int and df[col].min() >= min_int:\n                    df[col] = df[col].astype(datatype, copy=False)\n                    if verbose:\n                        print(f'Convert {col} to {datatype}.')\n                    break\n        elif df[col].dtype == 'int16':\n            datatypes = [np.int8]\n            # Go through all possible smaller datatypes and assign the smallest one that can fit all rows.\n            for datatype in datatypes:\n                min_int, max_int = np.iinfo(datatype).min, np.iinfo(datatype).max\n                if df[col].max() <= max_int and df[col].min() >= min_int:\n                    df[col] = df[col].astype(datatype, copy=False)\n                    if verbose:\n                        print(f'Convert {col} to {datatype}.')\n                    break\n                    \n        elif df[col].dtype == 'float64':\n            datatypes = [np.float32]\n            # If minimum value of column is more than the approximate resolution of the data type (smallest value that remains precise), we can switch to it\n            for datatype in datatypes:\n                resolution = np.finfo(datatype).resolution\n                if df[col].min() > resolution:\n                    df[col] =df[col].astype(datatype, copy=False)\n                    if verbose:\n                        print(f'Convert {col} to {datatype}.')\n                    break","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:17.45381Z","iopub.execute_input":"2022-01-30T14:22:17.454112Z","iopub.status.idle":"2022-01-30T14:22:17.467691Z","shell.execute_reply.started":"2022-01-30T14:22:17.454073Z","shell.execute_reply":"2022-01-30T14:22:17.466835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some of the more complex features go into separate functions:","metadata":{}},{"cell_type":"code","source":"def ichimoku(df, period_length=60):\n    #Calculate the conversion line\n    high_9 = df['High'].rolling(window=9*period_length).max()\n    low_9 = df['Low'].rolling(window=9*period_length).min()\n    tenkan_sen = (high_9 + low_9) / 2\n    \n    #Calculate the base line\n    high_26 = df['High'].rolling(window=26*period_length).max()\n    low_26 = df['Low'].rolling(window=26*period_length).min()\n    kijun_sen = (high_26 + low_26) / 2\n    \n    #Calculate leading span A\n    senkou_span_a = ((tenkan_sen + kijun_sen) / 2).shift(26*period_length)\n\n    #Calculate leading span B\n    high_52 = df['High'].rolling(window=52*period_length).max()\n    low_52 = df['Low'].rolling(window=52*period_length).min()\n    senkou_span_b = ((high_52 + low_52) / 2).shift(26*period_length)\n\n    # Calculate lagging span\n    chikou_span = df['Close'].shift(-22*period_length) #sometimes -26 \n    \n    return tenkan_sen, kijun_sen, senkou_span_a, senkou_span_b, chikou_span","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:17.469184Z","iopub.execute_input":"2022-01-30T14:22:17.469672Z","iopub.status.idle":"2022-01-30T14:22:17.485856Z","shell.execute_reply.started":"2022-01-30T14:22:17.469613Z","shell.execute_reply":"2022-01-30T14:22:17.484616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Stochastic(df, periods=14):\n    #Calculate highs and lows of price over 14 periods\n    H_14 = df['Close'].rolling(window=periods).max()\n    L_14 = df['Close'].rolling(window=periods).min()\n    \n    #Calculate value of fast stochastic indicator\n    stochastic_k = ((df['Close'] - L_14)/(H_14 - L_14)) * 100\n    stochastic_k.fillna(100, inplace=True)\n    \n    #Calculate value of slow stochastic indicator\n    stochastic_d = stochastic_k.rolling(window = 3).mean()\n    \n    return stochastic_k, stochastic_d","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:17.489603Z","iopub.execute_input":"2022-01-30T14:22:17.48989Z","iopub.status.idle":"2022-01-30T14:22:17.500622Z","shell.execute_reply.started":"2022-01-30T14:22:17.489854Z","shell.execute_reply":"2022-01-30T14:22:17.499879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def MoneyFlow(df, periods=14):    \n    #Calculate moving average of typical price\n    TP = df['VWAP'].to_numpy()\n    \n    #Calculate the raw money flow\n    MF = df['signs_diffs'] * TP * df['Volume']\n    pos_MF = MF.clip(lower=0)\n    neg_MF = -1 * MF.clip(upper=0)\n    \n    #Determine money flow ratio\n    MFR = pos_MF.rolling(periods).sum()/neg_MF.rolling(periods).sum()\n    MFR.fillna(1, inplace=True)\n    #Determine money flow index\n    return 100 - (100 / (1 + MFR))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:40:17.446745Z","iopub.execute_input":"2022-01-30T14:40:17.44707Z","iopub.status.idle":"2022-01-30T14:40:17.45337Z","shell.execute_reply.started":"2022-01-30T14:40:17.447033Z","shell.execute_reply":"2022-01-30T14:40:17.452483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rsi(df, periods=14):\n    # periods: Amount of periods (minute long) the RSI will be calculated with\n    \n    # The gain and loss values relative strength uses, gain is simply the difference between close prices but 0 if it is negative, opposite for loss\n    gain = df['diffs'].clip(lower=0)\n    loss = -1 * df['diffs'].clip(upper=0)\n    mean = None\n    \n    def smma(window):            \n        nonlocal mean\n        # Smoothed moving average\n        if mean is not None:\n            mean = (mean * (len(window)-1) + window[-1]) / periods\n        else:\n            mean = window.mean()\n        return mean\n    \n    \n    # Now we have to calculate a SMMA over it to get the RS\n    gain_mean = gain.rolling(periods).apply(smma, raw=True)\n\n    mean = None\n    loss_mean = loss.rolling(periods).apply(smma, raw=True)\n    \n    rs = gain_mean/loss_mean\n    \n    rsi = (100 - 100/(1+rs))\n    return gain_mean, loss_mean, rsi.fillna(100)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:17.514526Z","iopub.execute_input":"2022-01-30T14:22:17.515001Z","iopub.status.idle":"2022-01-30T14:22:17.52536Z","shell.execute_reply.started":"2022-01-30T14:22:17.514968Z","shell.execute_reply":"2022-01-30T14:22:17.524343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def obv_change(df):\n    return df['signs_diffs'] * df['Volume']","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:17.526712Z","iopub.execute_input":"2022-01-30T14:22:17.527029Z","iopub.status.idle":"2022-01-30T14:22:17.544184Z","shell.execute_reply.started":"2022-01-30T14:22:17.526999Z","shell.execute_reply":"2022-01-30T14:22:17.543294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ad_line(df):\n    df['MFM'] = ((df['Close'] - df['Low']) - (df['High'] - df['Close'])) / (df['High'] - df['Low'])\n    return df['Volume'] * df['MFM']","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:17.545543Z","iopub.execute_input":"2022-01-30T14:22:17.546108Z","iopub.status.idle":"2022-01-30T14:22:17.554503Z","shell.execute_reply.started":"2022-01-30T14:22:17.546069Z","shell.execute_reply":"2022-01-30T14:22:17.553614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def beta(df, market_df, periods): # The beta as calculated in the tutorial for the target\n    close_col = df['Close']\n    numerator = (market_df['log_ret'] * np.log(close_col/close_col.shift(15))).rolling(periods).mean()\n    denominator = (market_df['log_ret']**2).rolling(periods).mean()\n    return numerator/denominator\n    ","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:32:27.287863Z","iopub.execute_input":"2022-01-30T14:32:27.288288Z","iopub.status.idle":"2022-01-30T14:32:27.293236Z","shell.execute_reply.started":"2022-01-30T14:32:27.288256Z","shell.execute_reply":"2022-01-30T14:32:27.292405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we define a function to engineer the features for each dataframe:","metadata":{}},{"cell_type":"code","source":"def engineer_features(df):\n    OBVs = obv_change(df)\n    AD_line = ad_line(df)    \n    df['ema_24'] = df['Close'].ewm(span=24).mean()\n    df['ema_52'] = df['Close'].ewm(span=52).mean()\n    df['MACD_signal'] = (df['ema_24'] - df['ema_52']).ewm(span=18).mean()\n    df['MACD_crossover_norm'] = (df['ema_24'] - df['ema_52'] - df['MACD_signal']) / df['MACD_signal']\n    df['gain_mean'], df['loss_mean'], df['RSI'] = rsi(df, periods=28)\n    df['stochastic_k'], df['stochastic_d'] = Stochastic(df)\n    df['stochastic_crossover'] = df['stochastic_k'] - df['stochastic_d']\n    df['log_ret1'] = np.log(df['Close'] / df['Close'].shift(1))\n    df['log_ret30'] = df['log_ret1'].rolling(30).sum()\n    df['log_ret240'] = df['log_ret1'].rolling(240).sum() # 4 hours\n    df['log_ret1440'] = df['log_ret1'].rolling(1440).sum() # 1 day\n    \n    df['mfi'] = MoneyFlow(df, periods=28)\n#     df['MFI_divergence'] = df['log_ret30'] - np.log(df['mfi'] / df['mfi'].shift(1)).rolling(30).sum() # Cum. Log returns compared to cum. log MFI change over 30 mins\n\n    #df['beta_3750'] = beta(df, market, 3750)\n#     df['sin_240'] = np.sin(np.linspace(0, 2*np.pi*(len(df)/240), len(df))) # 4 hours\n#     df['sin_1440'] = np.sin(np.linspace(0, 2*np.pi*(len(df)/240), len(df))) # A day\n    \n    df = df.drop(['Count', 'High', 'Low', 'Open', 'Close', 'Volume', 'VWAP', 'diffs', 'signs_diffs', 'MFM'], axis=1) \n        \n    # Remove the first n rows where n is the maximum lag period, otherwise we have nans for those.\n    return df.iloc[3750+15:]","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:40:59.507377Z","iopub.execute_input":"2022-01-30T14:40:59.507768Z","iopub.status.idle":"2022-01-30T14:40:59.518463Z","shell.execute_reply.started":"2022-01-30T14:40:59.507728Z","shell.execute_reply":"2022-01-30T14:40:59.51734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's test this on bitcoin data:","metadata":{}},{"cell_type":"code","source":"bitcoin = asset_dfs[1].copy()\nbitcoin = engineer_features(bitcoin)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:41:02.869915Z","iopub.execute_input":"2022-01-30T14:41:02.870347Z","iopub.status.idle":"2022-01-30T14:41:08.978561Z","shell.execute_reply.started":"2022-01-30T14:41:02.870316Z","shell.execute_reply":"2022-01-30T14:41:08.977635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bitcoin[bitcoin.isnull().values]","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:41:18.612401Z","iopub.execute_input":"2022-01-30T14:41:18.612759Z","iopub.status.idle":"2022-01-30T14:41:18.791567Z","shell.execute_reply.started":"2022-01-30T14:41:18.612717Z","shell.execute_reply":"2022-01-30T14:41:18.790808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bitcoin.head(50)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:41:20.684431Z","iopub.execute_input":"2022-01-30T14:41:20.685341Z","iopub.status.idle":"2022-01-30T14:41:20.728547Z","shell.execute_reply.started":"2022-01-30T14:41:20.685264Z","shell.execute_reply":"2022-01-30T14:41:20.727855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**One remark is that we now loop over all observations whereas in case of testing we will receive one observation after the other. In that case we do not want to loop over the whole dataset everytime but just compare the gotten observation with the previous one.**","metadata":{}},{"cell_type":"code","source":"# def obv(df):\n#     df['Obv'] = df['Volume'][0]\n#     for i in range(len(df)):\n#         if i > 0:\n#             j = i - 1\n#             if df['Close'][i] > df['Close'][j]:\n#                 df['Obv'][i] = df['Obv'][j] + df['Volume'][i]\n#             elif df['Close'][i] < df['Close'][j]:\n#                 df['Obv'][i] = df['Obv'][j] - df['Volume'][i]\n#             else:\n#                 df['Obv'][i] = df['Obv'][j]\n        \n#     return df\n            \n#obv_test = obv(asset_dfs[1])","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.518013Z","iopub.status.idle":"2022-01-30T14:22:23.518311Z","shell.execute_reply.started":"2022-01-30T14:22:23.518148Z","shell.execute_reply":"2022-01-30T14:22:23.518165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.set_theme()\n# plt.plot(bitcoin['24h OBV change MA'], label='24h obv change')\n# plt.legend()\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.519555Z","iopub.status.idle":"2022-01-30T14:22:23.520154Z","shell.execute_reply.started":"2022-01-30T14:22:23.519982Z","shell.execute_reply":"2022-01-30T14:22:23.520003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.plot(bitcoin['1h sine'][:5*60], label='1h sine')\n# plt.legend(loc='lower right')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.521314Z","iopub.status.idle":"2022-01-30T14:22:23.523023Z","shell.execute_reply.started":"2022-01-30T14:22:23.522732Z","shell.execute_reply":"2022-01-30T14:22:23.522764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.plot(bitcoin['MACD'], label='MACD')\n# plt.legend(loc='lower right')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.524076Z","iopub.status.idle":"2022-01-30T14:22:23.524908Z","shell.execute_reply.started":"2022-01-30T14:22:23.524698Z","shell.execute_reply":"2022-01-30T14:22:23.524728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.plot(bitcoin['beta_3750'], label='3750m beta')\n# plt.legend(loc='lower right')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:41:56.964635Z","iopub.execute_input":"2022-01-30T14:41:56.96501Z","iopub.status.idle":"2022-01-30T14:41:57.41976Z","shell.execute_reply.started":"2022-01-30T14:41:56.964968Z","shell.execute_reply":"2022-01-30T14:41:57.41881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def ad_line(df):\n#     df['MFM'] = ((df['Close'] - df['Low']) - (df['High'] - df['Close'])) / (df['High'] - df['Low'])\n#     df['MFV'] = df['Volume'] * df['MFM']\n#     df['AD_line'] = df['MFV'][0]\n#     for i in range(len(df)):\n#         if i > 0:\n#             df['AD_line'][i] = (df['AD_line'][i-1] + df['MFV'][i])\n    \n#     return df\n# #ad_line_test = ad_line(asset_dfs[1])","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.528158Z","iopub.status.idle":"2022-01-30T14:22:23.528485Z","shell.execute_reply.started":"2022-01-30T14:22:23.528332Z","shell.execute_reply":"2022-01-30T14:22:23.528349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.set_theme()\n# #plt.plot(ad_line_test['AD_line'], label='AD_line')\n# plt.legend()\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.52946Z","iopub.status.idle":"2022-01-30T14:22:23.529957Z","shell.execute_reply.started":"2022-01-30T14:22:23.529748Z","shell.execute_reply":"2022-01-30T14:22:23.529776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.set_theme()\n# plt.plot(bitcoin['HLCO ratio'], label='HLCO ratio')\n# plt.legend()\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.530829Z","iopub.status.idle":"2022-01-30T14:22:23.531108Z","shell.execute_reply.started":"2022-01-30T14:22:23.530963Z","shell.execute_reply":"2022-01-30T14:22:23.530979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(20, 10))\n# for period in periods:\n#     plt.plot(bitcoin[f'{period}h log returns'], label=f'{period}h log returns')\n# plt.legend(loc='lower right')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.532618Z","iopub.status.idle":"2022-01-30T14:22:23.533344Z","shell.execute_reply.started":"2022-01-30T14:22:23.533161Z","shell.execute_reply":"2022-01-30T14:22:23.533182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(20, 10))\n# plt.plot(bitcoin['MACD_crossover_norm'], label='MACD crossover')\n# plt.legend(loc='lower right')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.534686Z","iopub.status.idle":"2022-01-30T14:22:23.534988Z","shell.execute_reply.started":"2022-01-30T14:22:23.534839Z","shell.execute_reply":"2022-01-30T14:22:23.534855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for asset_id, df in enumerate(asset_dfs):\n    df = engineer_features(df).reset_index()\n    name = asset_details.loc[asset_details.index == asset_id, 'Asset_Name'][asset_id].lower().replace(' ', '_')\n    reduce_mem_usage(df, verbose=True)\n    dt.Frame(df).to_jay(f'{name}.jay')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T14:22:23.540869Z","iopub.status.idle":"2022-01-30T14:22:23.54136Z","shell.execute_reply.started":"2022-01-30T14:22:23.541088Z","shell.execute_reply":"2022-01-30T14:22:23.541113Z"},"trusted":true},"execution_count":null,"outputs":[]}]}