{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport datatable as dt # Fast data reading/writing\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-01T22:51:39.936158Z","iopub.execute_input":"2022-02-01T22:51:39.936955Z","iopub.status.idle":"2022-02-01T22:51:40.961068Z","shell.execute_reply.started":"2022-02-01T22:51:39.936862Z","shell.execute_reply":"2022-02-01T22:51:40.960354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, we load the asset details so that we can load the data in the order of their asset id. Filenames are based on asset names.","metadata":{}},{"cell_type":"code","source":"asset_details = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv', index_col='Asset_ID')\nnames = asset_details.sort_index().Asset_Name.values\nids = asset_details.sort_index().index","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:40.962415Z","iopub.execute_input":"2022-02-01T22:51:40.962619Z","iopub.status.idle":"2022-02-01T22:51:40.989827Z","shell.execute_reply.started":"2022-02-01T22:51:40.962595Z","shell.execute_reply":"2022-02-01T22:51:40.989065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since all the dataframes are too big to be stored, trained and evaluated at once, we define a function to do it in one go:","metadata":{}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr \nfrom glob import iglob\nfrom datetime import datetime\nparams = {'lambda_l1': 0.004498875792752676, 'lambda_l2': 0.03243290696956152, 'num_leaves': 60, \n              'max_depth': 6, 'min_data_in_leaf': 2496, 'learning_rate': 0.18502752618241153, 'n_estimators': 100,\n              'boosting_type': 'goss', 'random_state': 1}\nused_features = ['RSI', 'MACD_crossover_norm', 'stochastic_crossover', 'log_ret1', 'log_ret30', 'log_ret240', 'log_ret1440', 'mfi']\n\ndef train(asset_name):\n    df = dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/{asset_name.lower().replace(' ', '_')}.jay\").to_pandas() # Load asset data\n    df.drop('index', axis=1, inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    X, y = df.drop(['Target'], axis=1)[used_features], df.Target # Separate into features and labels\n\n    \n    # Training the model\n    model = LGBMRegressor(**params)\n    model.fit(X, y)\n    \n    print(f'Trained model for {asset_name}')\n    \n    return model\n\ndef global_train():\n    all_df = []\n    for filename in iglob(\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/*.jay\"):\n        all_df.append(dt.fread(filename).to_pandas()) # Load asset data\n    all_df = pd.concat(all_df)\n    X_all, y_all = all_df.drop('Target', axis=1)[used_features], all_df.Target\n    \n    global_model = LGBMRegressor(**params)\n    global_model.fit(X_all, y_all)\n    \n    return global_model","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:40.99105Z","iopub.execute_input":"2022-02-01T22:51:40.991248Z","iopub.status.idle":"2022-02-01T22:51:41.967171Z","shell.execute_reply.started":"2022-02-01T22:51:40.991224Z","shell.execute_reply":"2022-02-01T22:51:41.966435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature engineering functions","metadata":{}},{"cell_type":"code","source":"from collections import deque\n\nclass DynamicSimpleMovingAverage:\n    def __init__(self, column, window_size):\n        # Dynamically shift moving averages over column\n        # Column should be columns with values from training data\n        self.window_size = window_size\n        self.window = deque(column.iloc[-window_size:].to_numpy())\n        self.sum = np.sum(self.window)\n        \n    def get_sma(self, new_values):\n        mas = np.full(len(new_values), np.nan)\n        for i, value in enumerate(new_values):\n            self.sum -= self.window.popleft()\n            self.sum += value\n            self.window.append(value)\n            mas[i] = self.sum/self.window_size\n        return mas","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:41.968689Z","iopub.execute_input":"2022-02-01T22:51:41.968928Z","iopub.status.idle":"2022-02-01T22:51:41.974312Z","shell.execute_reply.started":"2022-02-01T22:51:41.968891Z","shell.execute_reply":"2022-02-01T22:51:41.97364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DynamicExponentialMovingAverage:\n    def __init__(self, column, window_size):\n        # Dynamically shift exponential moving averages over column\n        # Unlike simple moving average, instead of sums we need to keep track of only the previous EMA.\n        # Column should be column with previous emas from training data\n        self.prev_ema = column[-1]\n        self.alpha = 2/(1+window_size)\n        \n    def get_ema(self, new_values):\n        emas = np.full(len(new_values), np.nan)\n        for i in range(len(new_values)):\n            ema = self.alpha * new_values[i] + self.prev_ema * (1-self.alpha)\n            self.prev_ema = ema\n            emas[i] = ema\n        return emas","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:41.97632Z","iopub.execute_input":"2022-02-01T22:51:41.976588Z","iopub.status.idle":"2022-02-01T22:51:41.988326Z","shell.execute_reply.started":"2022-02-01T22:51:41.976557Z","shell.execute_reply":"2022-02-01T22:51:41.987685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Feature():\n    # An interface for dynamic feature computations\n    def __init__(self, name):\n        self.name = name\n        \n    def get(self, new_values):\n        # Compute the feature based on dataframe of new values\n        pass","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:41.989398Z","iopub.execute_input":"2022-02-01T22:51:41.99003Z","iopub.status.idle":"2022-02-01T22:51:41.9983Z","shell.execute_reply.started":"2022-02-01T22:51:41.989994Z","shell.execute_reply":"2022-02-01T22:51:41.997647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RSI(Feature):\n    def __init__(self, name, close_col, gain_mean, loss_mean, period):\n        super().__init__(name)\n        used_col = close_col.to_numpy()[-period:]\n        self.gain_mean = gain_mean[-1]\n        self.loss_mean = loss_mean[-1]\n        self.last_val = used_col[-1]\n        self.period = period\n    \n    def get_diff(self, new_value):\n        gain = 0\n        loss = 0\n        if new_value < self.last_val:\n            gain = 0\n            loss = abs(new_value-self.last_val)\n        else:\n            gain = new_value-self.last_val\n            loss = 0\n        self.last_val = new_value\n        \n        return gain, loss\n    \n    def get(self, new_values):\n        close_col = new_values['Close'].to_numpy()\n        \n        rsis = np.zeros(len(close_col))\n        for i in range(len(close_col)):\n            gain, loss = self.get_diff(close_col[i])\n            self.gain_mean = (self.gain_mean * (self.period-1) + gain)/self.period\n            self.loss_mean = (self.loss_mean * (self.period-1) + loss)/self.period\n            rs = self.gain_mean / self.loss_mean\n            rsis[i] = (100 - 100/(1+rs))\n        \n        return rsis","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:41.999536Z","iopub.execute_input":"2022-02-01T22:51:41.999815Z","iopub.status.idle":"2022-02-01T22:51:42.009646Z","shell.execute_reply.started":"2022-02-01T22:51:41.999781Z","shell.execute_reply":"2022-02-01T22:51:42.008859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MFI(Feature):\n    def __init__(self, name, close_col, volume_col, period):\n        super().__init__(name)\n        used_close = close_col.to_numpy()[-(period+1):]\n        used_vol = volume_col.to_numpy()[-(period+1):]\n        sign_diffs = np.sign(used_close[1:] - used_close[:-1])\n        self.money_flow = sign_diffs * used_close[1:] * used_vol[1:]\n        self.pos = self.money_flow.clip(min=0).sum()\n        self.neg = -1 * self.money_flow.clip(max=0).sum()\n        self.last_val = used_close[-1]\n        \n    \n    def get(self, new_values):\n        close_col = new_values['VWAP'].to_numpy()\n        vol_col = new_values['Volume'].to_numpy()\n        mfis = np.zeros(len(close_col))\n        \n        for i in range(len(close_col)):\n            # Remove first element of window in sums\n            self.pos -= self.money_flow[0].clip(min=0) \n            self.neg -= -1 * self.money_flow[0].clip(max=0)\n            \n            # Slide window\n            self.money_flow = np.roll(self.money_flow, -1)\n            \n            curr_close = close_col[i]\n            curr_vol = vol_col[i]\n            if curr_close < self.last_val:\n                self.money_flow[-1] = -1 * curr_close * curr_vol\n                self.neg += -1 * self.money_flow[-1]\n            else:\n                self.money_flow[-1] = curr_close * curr_vol\n                self.pos += self.money_flow[-1]\n            \n            if self.neg == 0: # to prevent div by zero\n                mfis[i] = 50\n            else:\n                mfis[i] = 100 - 100/(1 + self.pos/self.neg)\n\n        return mfis","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.010712Z","iopub.execute_input":"2022-02-01T22:51:42.011163Z","iopub.status.idle":"2022-02-01T22:51:42.023157Z","shell.execute_reply.started":"2022-02-01T22:51:42.011121Z","shell.execute_reply":"2022-02-01T22:51:42.022543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MACD(Feature):\n    def __init__(self, name, macd_long_period, macd_long_col, macd_short_period, macd_short_col, signal_period, macd_col):\n        super().__init__(name)\n        self.macd_long_ema = DynamicExponentialMovingAverage(macd_long_col.to_numpy(), macd_long_period)\n        self.macd_short_ema = DynamicExponentialMovingAverage(macd_short_col.to_numpy(), macd_short_period)\n        self.signal_ema = DynamicExponentialMovingAverage(macd_col.to_numpy(), signal_period)\n\n    def get(self, new_values):\n        close_col = new_values['Close'].to_numpy()\n        macd = self.macd_short_ema.get_ema(close_col) - self.macd_long_ema.get_ema(close_col)\n        signal = self.signal_ema.get_ema(macd)\n        macd_crossovers = (macd - signal) / signal # Normalize with signal \n        return macd_crossovers","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.02453Z","iopub.execute_input":"2022-02-01T22:51:42.025279Z","iopub.status.idle":"2022-02-01T22:51:42.036786Z","shell.execute_reply.started":"2022-02-01T22:51:42.025239Z","shell.execute_reply":"2022-02-01T22:51:42.036075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Stochastic(Feature):\n    def __init__(self, name, close_col, k_col, period):\n        super().__init__(name)\n        self.window = deque(close_col.iloc[-period:].to_numpy())\n        self.low = np.min(self.window)\n        self.high = np.max(self.window)\n        self.d_sma = DynamicSimpleMovingAverage(k_col, 3) # needs K% column of training data\n        \n    def get(self, new_values):\n        close_col = new_values['Close'].to_numpy()\n        \n        k = np.zeros(len(close_col))\n        for i in range(len(close_col)):\n            self.window.popleft()\n            self.window.append(close_col[i])\n            self.low = np.min(self.window)\n            self.high = np.max(self.window)\n            k[i] = (close_col[i] - self.low)/(self.high - self.low) * 100\n            \n        d = self.d_sma.get_sma(k)\n        return k-d","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.037752Z","iopub.execute_input":"2022-02-01T22:51:42.03795Z","iopub.status.idle":"2022-02-01T22:51:42.050469Z","shell.execute_reply.started":"2022-02-01T22:51:42.037926Z","shell.execute_reply":"2022-02-01T22:51:42.049835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CumLogReturns(Feature):\n    def __init__(self, name, close_col, period):\n        super().__init__(name)\n        used_col = close_col.to_numpy()[-(period+1):]\n        self.window = np.log(used_col[1:] / used_col[:-1])\n        self.sum = self.window.sum()\n        self.last_val = used_col[-1]\n        \n    def get(self, new_values):\n        close_col = new_values['Close'].to_numpy()\n        ret = np.zeros(len(close_col))\n        for i in range(len(close_col)):\n            self.sum -= self.window[0]\n            new_ret = np.log(close_col[i] / self.last_val)\n            self.sum += new_ret\n            self.window = np.roll(self.window, -1)\n            self.window[-1] = new_ret\n            self.last_val = close_col[i]\n            ret[i] = self.sum\n        return ret","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.051287Z","iopub.execute_input":"2022-02-01T22:51:42.052162Z","iopub.status.idle":"2022-02-01T22:51:42.060428Z","shell.execute_reply.started":"2022-02-01T22:51:42.052129Z","shell.execute_reply":"2022-02-01T22:51:42.059846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Functions for processing incoming data","metadata":{}},{"cell_type":"code","source":"def get_last_train_rows():\n    # Get rows of all assets in one dataframe, of the last timestamp seen \n    last_rows = []\n    for name in names:\n        last_rows.append(dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/{name.lower().replace(' ', '_')}.jay\").to_pandas().iloc[-1])\n    concat = pd.concat(last_rows)\n    concat.index = sorted(asset_details.index)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.061299Z","iopub.execute_input":"2022-02-01T22:51:42.062005Z","iopub.status.idle":"2022-02-01T22:51:42.073117Z","shell.execute_reply.started":"2022-02-01T22:51:42.061972Z","shell.execute_reply":"2022-02-01T22:51:42.072362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_window(asset_dfs, window_size):\n    # Get rows of the current window, as a list of dataframes for every asset\n    return [asset_df[:-window_size] for asset_df in asset_dfs]","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.07426Z","iopub.execute_input":"2022-02-01T22:51:42.074486Z","iopub.status.idle":"2022-02-01T22:51:42.081447Z","shell.execute_reply.started":"2022-02-01T22:51:42.074461Z","shell.execute_reply":"2022-02-01T22:51:42.080743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import timedelta\ndef interpolate(test_batch, prev_timestamp_rows):\n    prev_timestamp_rows['row_id'] = -1 # Add row_id column as dummy so they match columns\n    asset_dfs = {}\n    for asset_id in prev_timestamp_rows['Asset_ID'].unique(): \n        prev_row = prev_timestamp_rows[prev_timestamp_rows['Asset_ID'] == asset_id]\n        if asset_id not in test_batch['Asset_ID'].values:\n            # If this asset is not included in new data at all, create a new row filled with nans for it, to be interpolated later.\n            asset_df = pd.DataFrame(columns=prev_timestamp_rows.columns, index=[prev_timestamp_rows['Asset_ID'].index[0] + timedelta(minutes=1)])\n            asset_df['Asset_ID'] = asset_id\n        else:\n            asset_df = test_batch.loc[test_batch['Asset_ID'] == asset_id, :]\n#         if asset_df.index.value_counts()[0] > 1:\n#             return {} # In case of some weird event where the timestamp is the same as the previous iteration, just return an empty dict to skip this iteration entirely\n\n\n        asset_df.replace([np.inf, -np.inf], 0, inplace=True) # Replace infs with zeros\n        asset_df.loc[asset_df.Volume == 0.0, 'Volume'] = np.nan # Zero volume seems unlikely, so interpolate this instead\n        if (asset_df.index[0] == prev_row.index[0]):\n            asset_df.reset_index(inplace=True)\n            asset_df['timestamp'][0] += timedelta(minutes=1) # If somehow the timestamp remain the same, add 1 minute to it so asfreq() doesnt break\n            asset_df.set_index('timestamp', inplace=True)\n        if (asset_df.index[0] >= prev_row.index[0]): \n            asset_df = pd.concat([prev_row, asset_df]).asfreq(freq='60S') # Adds nans to missing minutes using previous row\n            asset_df['row_id'] = asset_df['row_id'].fillna(-1) # So that we can recognize interpolated rows and skip them for prediction\n            asset_df['Asset_ID'] = asset_df['Asset_ID'].fillna(asset_id) # This should not be interpolated\n            asset_df = asset_df.interpolate(method='linear', axis=0) # Interpolate and forward fill potential missing values at the end\n            asset_df = asset_df.iloc[1:] # Remove the previous row again\n        asset_dfs[asset_id] = asset_df.fillna(method='ffill')\n    return asset_dfs","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.083826Z","iopub.execute_input":"2022-02-01T22:51:42.084163Z","iopub.status.idle":"2022-02-01T22:51:42.093989Z","shell.execute_reply.started":"2022-02-01T22:51:42.084136Z","shell.execute_reply":"2022-02-01T22:51:42.093313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def engineer_features(batch_assets, features):\n    engineered = {}\n    start_time = time.time()\n    for asset_id in batch_assets.keys():\n        df = batch_assets[asset_id]\n        init_timestamp = df.index[0]        \n        for feature in features[asset_id]:\n            df[f'{feature.name}'] = feature.get(df)\n        df = df.drop(['Count', 'High', 'Low', 'Open', 'Close', 'Volume', 'VWAP'], axis=1)\n#         engineered[asset_id] = window.loc[init_timestamp:]\n        engineered[asset_id] = df\n#     print(f'Engineering took {time.time()-start_time} seconds.')\n    return engineered","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.09504Z","iopub.execute_input":"2022-02-01T22:51:42.095626Z","iopub.status.idle":"2022-02-01T22:51:42.106243Z","shell.execute_reply.started":"2022-02-01T22:51:42.095598Z","shell.execute_reply":"2022-02-01T22:51:42.10558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_new_windows(old_windows, curr_batch, window_size):\n    # Get the new window to be the last rows that fit in the window\n    if len(curr_batch) == 0:\n        return old_windows\n    else:\n        return {asset_id: pd.concat([old_windows[asset_id], curr_batch[asset_id]]).iloc[-window_size:] for asset_id in old_windows.keys()} ","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.107309Z","iopub.execute_input":"2022-02-01T22:51:42.107979Z","iopub.status.idle":"2022-02-01T22:51:42.117867Z","shell.execute_reply.started":"2022-02-01T22:51:42.10795Z","shell.execute_reply":"2022-02-01T22:51:42.117332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_targets(asset_dfs, models, global_model, global_weight=0.5):\n    targets = []\n    for asset_id in asset_dfs.keys():\n        asset_df = asset_dfs[asset_id]\n        model = models[asset_id]\n        features = asset_df.drop(['row_id', 'Asset_ID', 'Target', 'group_num'], axis=1, errors='ignore').to_numpy()\n        targets.extend((zip(asset_df['row_id'].to_numpy(), \n                            global_weight*global_model.predict(features)+(1-global_weight)*model.predict(features))))\n    targets = sorted(filter((lambda tup: tup[0] >= 0), targets), key=(lambda tup: tup[0])) # Remove interpolated rows and sort by id\n    return list(map((lambda tup: tup[1]), targets)) # Get the target values and add them to the predictions.","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.119013Z","iopub.execute_input":"2022-02-01T22:51:42.119415Z","iopub.status.idle":"2022-02-01T22:51:42.126875Z","shell.execute_reply.started":"2022-02-01T22:51:42.119363Z","shell.execute_reply":"2022-02-01T22:51:42.126261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training the models:","metadata":{}},{"cell_type":"code","source":"models = {}\nfor asset_id, asset_name in zip(ids, names):\n    models[asset_id] = train(asset_name)\n    \nglobal_model = global_train()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:51:42.128109Z","iopub.execute_input":"2022-02-01T22:51:42.128563Z","iopub.status.idle":"2022-02-01T22:56:24.683126Z","shell.execute_reply.started":"2022-02-01T22:51:42.128522Z","shell.execute_reply":"2022-02-01T22:56:24.682104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# example_rsi = [[26.9, 2.7, -7.5/37_595.2]] # Bitcoin - 29 jan 18:36 UTC\n# prediction = models[1].predict(example_rsi, pred_contrib = True)\n# print(prediction)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:56:24.685262Z","iopub.execute_input":"2022-02-01T22:56:24.68559Z","iopub.status.idle":"2022-02-01T22:56:24.690258Z","shell.execute_reply.started":"2022-02-01T22:56:24.685549Z","shell.execute_reply":"2022-02-01T22:56:24.689448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window_size = 1441\n# Retrieve the first window\nwindows = {}\nfeatures = {}\nsma_dict = {}\nfor asset_name, asset_id in zip(names, ids):\n    engineered_window = dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/{asset_name.lower().replace(' ', '_')}.jay\").to_pandas()\n    engineered_window = engineered_window.set_index('timestamp').iloc[-window_size:]\n    preprocessed_window = dt.fread(f\"../input/crypto-challenge-mlii-project-preprocessing-2/{asset_name.lower().replace(' ', '_')}.jay\").to_pandas()\n    preprocessed_window = preprocessed_window.set_index('timestamp').iloc[-window_size:]\n    windows[asset_id] = preprocessed_window\n    close_col = preprocessed_window['Close']\n    vwap_col = preprocessed_window['VWAP']\n    vol_col = preprocessed_window['Volume']\n    asset_features = [RSI('RSI', close_col, engineered_window['gain_mean'], engineered_window['loss_mean'], 28),\n                      MACD('MACD_crossover_norm', 52, engineered_window['ema_52'], 24, engineered_window['ema_24'], 18, engineered_window['MACD_signal']), \n                      Stochastic('stochastic_crossover', close_col, engineered_window['stochastic_k'], 28),\n                      CumLogReturns('log_ret1', close_col, 1),\n                      CumLogReturns('log_ret30', close_col, 30),\n                      CumLogReturns('log_ret240', close_col, 240),\n                      CumLogReturns('log_ret1440', close_col, 1440),\n                      MFI('mfi', vwap_col, vol_col, 28)\n                     ]\n    features[asset_id] = asset_features","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:56:24.691654Z","iopub.execute_input":"2022-02-01T22:56:24.692093Z","iopub.status.idle":"2022-02-01T22:56:50.47217Z","shell.execute_reply.started":"2022-02-01T22:56:24.692055Z","shell.execute_reply":"2022-02-01T22:56:50.471426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:56:50.473415Z","iopub.execute_input":"2022-02-01T22:56:50.473701Z","iopub.status.idle":"2022-02-01T22:56:50.505737Z","shell.execute_reply.started":"2022-02-01T22:56:50.473651Z","shell.execute_reply":"2022-02-01T22:56:50.504872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For testing without using API\n\n# copy1 = dummy_test.copy()\n# copy2 = dummy_test.copy()\n# copy2['timestamp'] += 60\n# copy2.drop(copy2.index[copy2['Asset_ID'] == 2], inplace=True)\n# copy3 = dummy_test.copy()\n# copy3['timestamp'] += 180\n# copy3.drop(copy3.index[copy3['Asset_ID'] == 2], inplace=True) # Test what happens when assets are not all provided\n# test_data = [(copy1, pd.DataFrame()), (copy2, pd.DataFrame()), (copy3, pd.DataFrame())]\n# for i, (test_batch, sample_preds) in enumerate(test_data):\n#     start_time = time.time()\n#     test_batch['timestamp'] = pd.to_datetime(test_batch['timestamp'], unit='s')\n#     test_batch.set_index('timestamp', inplace=True)\n#     #TODO last rows should include interpolated ones\n#     last_rows = pd.concat([asset.iloc[-1:] for asset in windows.values()]) # Slice [-1:] so we get a DataFrame instead of Series\n#     asset_dfs = interpolate(test_batch, last_rows) # Use the final rows from the previous time to determine if there are any gaps\n#     engineered_dfs = engineer_features(asset_dfs, features, sma_dict)\n#     print(engineered_dfs[1])\n#     sns.lineplot(data=engineered_dfs[3], x='timestamp', y='RSI')\n#     windows = get_new_windows(windows, asset_dfs, window_size)\n#     targets = predict_targets(engineered_dfs, models)\n#     sample_preds['Target'] = predict_targets(engineered_dfs, models)\n#     print(f'Predicted {len(test_batch)} values! Took {time.time()-start_time} seconds.')","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:56:50.506926Z","iopub.execute_input":"2022-02-01T22:56:50.50713Z","iopub.status.idle":"2022-02-01T22:56:50.511394Z","shell.execute_reply.started":"2022-02-01T22:56:50.507104Z","shell.execute_reply":"2022-02-01T22:56:50.510628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TODO: train on new data\nimport time\n#dummy_test = None\nfor test_batch, sample_preds in iter_test:\n    start_time = time.time()\n    test_batch['timestamp'] = pd.to_datetime(test_batch['timestamp'], unit='s')\n    test_batch.set_index('timestamp', inplace=True)\n    test_batch.index = test_batch.index.ceil('min') # Round up to nearest minute\n    \n    last_rows = pd.concat([asset.iloc[-1:] for asset in windows.values()]) # Slice [-1:] so we get a DataFrame instead of Series\n    asset_dfs = interpolate(test_batch, last_rows) # Use the final rows from the previous time to determine if there are any gaps\n    engineered_dfs = engineer_features(asset_dfs, features)\n    windows = get_new_windows(windows, asset_dfs, 1)\n    targets = predict_targets(engineered_dfs, models, global_model)\n    targets = np.clip(np.nan_to_num(targets), -0.99, 0.99)\n    sample_preds['Target'] = targets\n    env.predict(sample_preds) # Call the predict function to pass it through the API.\n    #print(f'Predicted {len(test_batch)} values! Took {time.time()-start_time} seconds.')","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:56:50.5125Z","iopub.execute_input":"2022-02-01T22:56:50.512949Z","iopub.status.idle":"2022-02-01T22:56:51.337328Z","shell.execute_reply.started":"2022-02-01T22:56:50.512917Z","shell.execute_reply":"2022-02-01T22:56:51.336659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# start = datetime.fromtimestamp(1623542340)\n# end = datetime.fromtimestamp(1623542520)\n# df = dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/bitcoin.jay\").to_pandas().set_index('timestamp').loc[start:end]\n# display(df)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:56:51.338618Z","iopub.execute_input":"2022-02-01T22:56:51.338806Z","iopub.status.idle":"2022-02-01T22:56:51.342366Z","shell.execute_reply.started":"2022-02-01T22:56:51.338782Z","shell.execute_reply":"2022-02-01T22:56:51.341591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from datetime import datetime\n# from datatable import dt\n# import pandas as pd\n# display(dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-preprocessing-2/bitcoin.jay\").to_pandas().set_index('timestamp').loc[datetime.fromtimestamp(1623540000):datetime.fromtimestamp(1623542520)])\n# display(dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/bitcoin.jay\").to_pandas().set_index('timestamp').loc[datetime.fromtimestamp(1623540000):datetime.fromtimestamp(1623542520)])\n# orig = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-02-01T22:56:51.344223Z","iopub.execute_input":"2022-02-01T22:56:51.344558Z","iopub.status.idle":"2022-02-01T22:57:43.060803Z","shell.execute_reply.started":"2022-02-01T22:56:51.34452Z","shell.execute_reply":"2022-02-01T22:57:43.059891Z"},"trusted":true},"execution_count":null,"outputs":[]}]}