{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This code is based on  https://www.kaggle.com/alexfir/recreating-target</br>\nFormulas:\n$$R^a(t) = log (P^a(t+16)\\ /\\ P^a(t+1))$$\n$$M(t) = \\frac{\\sum_a w^a R^a(t)}{\\sum_a w^a}$$\n$$\\beta^a = \\frac{\\langle M \\cdot R^a \\rangle}{\\langle M^2 \\rangle}$$\n$$\\text{Target}^a(t) = R^a(t) - \\beta^a M(t)$$\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime as dt\nimport lightgbm as lgb\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.metrics import mean_squared_log_error, mean_squared_error\n\nasset_details = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\nasset_details = asset_details.sort_values('Asset_ID')\n\nids = list(asset_details.Asset_ID)\nweights = np.array(list(asset_details.Weight))\n\ndf = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')\ndf.index = pd.to_datetime(df['timestamp'], unit='s')\ndf = df.sort_index()\n\nsupplemental_df = pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv')\nsupplemental_df.index = pd.to_datetime(supplemental_df['timestamp'], unit='s')\nsupplemental_df = supplemental_df.sort_index()\n\ndf = df.append(supplemental_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:09:32.778010Z","iopub.execute_input":"2022-01-31T23:09:32.778651Z","iopub.status.idle":"2022-01-31T23:10:38.860865Z","shell.execute_reply.started":"2022-01-31T23:09:32.778604Z","shell.execute_reply":"2022-01-31T23:10:38.858964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_price = df.groupby([df.index, 'Asset_ID'])['Close'].last().unstack()\n\n#price 16-min returns\n#df_price_rets = df_price_rets.shift(periods=-16)/df_price_rets.shift(periods=-1) - 1\n\n#daily data for prediction\n#df_price_daily = df.copy()\n#df_price_daily.index = df_price_daily.index.normalize()\n#df_price_daily = df_price_daily.groupby([df_price_daily.index, 'Asset_ID'])['Close'].last().unstack()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:38.865378Z","iopub.execute_input":"2022-01-31T23:10:38.866249Z","iopub.status.idle":"2022-01-31T23:10:55.387763Z","shell.execute_reply.started":"2022-01-31T23:10:38.866213Z","shell.execute_reply":"2022-01-31T23:10:55.387036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(df_price/df_price.shift(1)-1).iloc[:30,:5].plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:55.388969Z","iopub.execute_input":"2022-01-31T23:10:55.389469Z","iopub.status.idle":"2022-01-31T23:10:56.074843Z","shell.execute_reply.started":"2022-01-31T23:10:55.389411Z","shell.execute_reply":"2022-01-31T23:10:56.074091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_price","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.076863Z","iopub.execute_input":"2022-01-31T23:10:56.077591Z","iopub.status.idle":"2022-01-31T23:10:56.102958Z","shell.execute_reply.started":"2022-01-31T23:10:56.077551Z","shell.execute_reply":"2022-01-31T23:10:56.101987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.104558Z","iopub.execute_input":"2022-01-31T23:10:56.104986Z","iopub.status.idle":"2022-01-31T23:10:56.119348Z","shell.execute_reply.started":"2022-01-31T23:10:56.104937Z","shell.execute_reply":"2022-01-31T23:10:56.118222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\n\n#Error: You can only iterate over `iter_test()` once.\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.120652Z","iopub.execute_input":"2022-01-31T23:10:56.121407Z","iopub.status.idle":"2022-01-31T23:10:56.180048Z","shell.execute_reply.started":"2022-01-31T23:10:56.121270Z","shell.execute_reply":"2022-01-31T23:10:56.179329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Possible solutions to this competition\n\n1. Naive - replicate the test price 16 times and there calc target\n2. ARIMA - https://towardsdatascience.com/multi-step-time-series-forecasting-with-arima-lightgbm-and-prophet-cc9e3f95dfb0\n3. LGBM  - https://towardsdatascience.com/a-lightgbm-autoregressor-using-sktime-6402726e0e7b\n3. ML  - predict 16 future values with SVR/H2O/LGBM+StackingRegressor. \n4. DNN - predict 16 future values with windowed DNN/CNN. \n5. Build model based on the daily data? Then how to predict minute intervals?\n\nFor 2/3:<br/>\nThe X will be current prices and the Y will be 16 future prices (multilabel).<br/>\nWe will have to rebuild the model at every iteration.<br/>\nThe data can be limited to the last 5000 records.<br/>\nWe can try to add function `calculate_target` inside tensorflow model and predict the final targets.</br>\nInstead of the prices we can predict returns and use them when calculating targets:<br/>\n`ret = price.shift(periods=-16)/price.shift(periods=-1) - 1`","metadata":{}},{"cell_type":"code","source":"def calculate_target(data):\n    all_timestamps = data.index.unique()\n    targets = pd.DataFrame(index=all_timestamps)\n\n    for id in ids:\n        targets[id] = (data[id].shift(periods=-16)/data[id].shift(periods=-1)) - 1\n    \n    targets['m'] = np.average(targets.fillna(0), axis=1, weights=weights)\n    \n    m = targets['m']\n    num = targets.multiply(m.values, axis=0).rolling(3750).mean().values\n    denom = m.multiply(m.values, axis=0).rolling(3750).mean().values\n    beta = np.nan_to_num(num.T / denom, nan=0., posinf=0., neginf=0.)\n\n    targets = targets - (beta * m.values).T\n    targets.drop('m', axis=1, inplace=True)\n    \n    return targets\n\n\n#Naive prediction\ndef predict_values_naive(df_price_uplimit: pd.DataFrame, num_preds):\n    new_df = df_price_uplimit.iloc[-1:]\n    for id_lag in range(num_preds):\n        new_df.index = new_df.index + dt.timedelta(minutes=1)\n        df_price_uplimit = df_price_uplimit.append(new_df)\n        \n    return df_price_uplimit\n\n\ndef create_sliding_windows(data: np.array, window_size):\n    wx = [data[i:-window_size+i] for i in range(window_size)]\n    wx = np.dstack(wx)[0]\n    wy = data[window_size:]\n    return wx, wy\n\n\ndef train_model(df_price: pd.DataFrame, window_size):\n    sliced_data = df_price.iloc[-dataset_size:].copy().fillna(method='ffill')\n    #sliced_data = sliced_data.pct_change().iloc[1:]\n    #sliced_data.index = np.arange(sliced_data.index.shape[0])\n    #sliced_data = (df_price.shift(periods=-16)/df_price.shift(periods=-1) - 1).dropna()\n\n    test_data = sliced_data.iloc[-window_size:].values\n    regressors = []\n    for i in range(test_data.shape[1]):\n        x, y = create_sliding_windows(sliced_data[i].values, window_size)     #device='gpu',num_leaves=128,learning_rate=0.01\n        regressor = Pipeline([('scaler', StandardScaler()),('lgb', lgb.LGBMRegressor(device='gpu',num_leaves=128,learning_rate=0.01))]).fit(x, y)\n        \"\"\"regressor = Pipeline([\n            ('scaler', StandardScaler()), \n            ('regr', VotingRegressor([\n                ('lgb', lgb.LGBMRegressor(device='gpu')), \n                ('svr', SVR(C=0.5))\n            ]))\n        ]).fit(x, y)\"\"\"\n        \n        regressors.append(regressor)\n    return regressors, test_data\n    \n\ndef predict_values(regressors, test_data: np.array, num_preds):\n    pred_data = []\n    for _ in range(num_preds):\n        y_pred = [regressors[i].predict([test_data[:,i]])[0] for i in range(len(regressors))]\n        test_data = np.append(test_data[1:], [y_pred], axis=0)\n        pred_data.append(y_pred)\n    return pred_data\n\n\ndef weighted_correlation(a, b, weights):\n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n    return corr","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.181503Z","iopub.execute_input":"2022-01-31T23:10:56.181750Z","iopub.status.idle":"2022-01-31T23:10:56.200456Z","shell.execute_reply.started":"2022-01-31T23:10:56.181717Z","shell.execute_reply":"2022-01-31T23:10:56.199786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#debug: create_sliding_windows\ncreate_sliding_windows([1,2,3,4,5,6,7,8,9,10,11,12,13], 4)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.202000Z","iopub.execute_input":"2022-01-31T23:10:56.202477Z","iopub.status.idle":"2022-01-31T23:10:56.214912Z","shell.execute_reply.started":"2022-01-31T23:10:56.202439Z","shell.execute_reply":"2022-01-31T23:10:56.214149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Autocorrelation (ACF) plot can be used to find if time series is stationarity. It also can be helpful to find the order of moving average part in ARIMA model.<br/> \nPartial autocorrelation (PACF) plot is useful to identify the order of autoregressive part in ARIMA model. <br/>\nAugmented Dickey–Fuller unit test examines if the time series is non-stationary. <br/>\nThe null hypothesis is that the series is non-stationary, hence if the p-value is small, it implies the time series is NOT non-stationary.","metadata":{}},{"cell_type":"code","source":"%%script echo skipping\n#debug: checking data stationarity\n\nimport matplotlib.pyplot as plt\nimport statsmodels.graphics.tsaplots as smt\nimport statsmodels.api as sm\n\ny = df_price[0].iloc[-500:].fillna(method='ffill')\ny.index = np.arange(y.index.shape[0])\ny = (y.shift(periods=-16)/y.shift(periods=-1) - 1).dropna()\n\nprint('Min:',y.min(), 'Max:',y.max())\nprint('Less STD is more stationary:', y.rolling(60*24).std().std())\n\nlags = None\nlayout = (2, 2)\nfig = plt.figure(figsize=(12, 7))\nts_ax = plt.subplot2grid(layout, (0, 0), colspan=2)\nacf_ax = plt.subplot2grid(layout, (1, 0))\npacf_ax = plt.subplot2grid(layout, (1, 1))\n\ny.plot(ax=ts_ax)\np_value = sm.tsa.stattools.adfuller(y)[1]\nts_ax.set_title('Time Series Analysis Plots\\n Dickey-Fuller: p={0:.5f}'.format(p_value))\nsmt.plot_acf(y, lags=lags, ax=acf_ax)\nsmt.plot_pacf(y, lags=lags, ax=pacf_ax)\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.216356Z","iopub.execute_input":"2022-01-31T23:10:56.216601Z","iopub.status.idle":"2022-01-31T23:10:56.376335Z","shell.execute_reply.started":"2022-01-31T23:10:56.216569Z","shell.execute_reply":"2022-01-31T23:10:56.375307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#debug: checking LGBM + sktime forecast\n\n!pip install sktime pmdarima\n\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import mean_absolute_percentage_error\nfrom sktime.forecasting.compose import make_reduction, TransformedTargetForecaster\nfrom sktime.forecasting.model_selection import ExpandingWindowSplitter, ForecastingGridSearchCV\nimport lightgbm as lgb\n\n\ndef plot_forecast(series_train, series_test, forecast, forecast_int=None):\n    mae = mean_absolute_error(series_test, forecast)\n    mape = mean_absolute_percentage_error(series_test, forecast)\n\n    plt.figure(figsize=(12, 6))\n    plt.title(f\"MAE: {mae:.2f}, MAPE: {mape:.3f}\", size=18)\n    series_train.plot(label=\"train\", color=\"b\")\n    series_test.plot(label=\"test\", color=\"g\")\n    forecast.index = series_test.index\n    forecast.plot(label=\"forecast\", color=\"r\")\n    if forecast_int is not None:\n        plt.fill_between(series_test.index,forecast_int[\"lower\"],forecast_int[\"upper\"],alpha=0.2,color=\"dimgray\")\n        \n    plt.legend(prop={\"size\": 16})\n    plt.show()\n    return mae, mape\n\ndef create_forecaster():\n    # creating forecaster with LightGBM\n    regressor = lgb.LGBMRegressor()\n    forecaster = make_reduction(regressor, window_length=5, strategy=\"recursive\")\n    return forecaster\n\ndef grid_serch_forecaster(train, test, forecaster, param_grid):\n    # Grid search on window_length\n    cv = ExpandingWindowSplitter(initial_window=int(len(train) * 0.7))\n    gscv = ForecastingGridSearchCV(forecaster, strategy=\"refit\", cv=cv, param_grid=param_grid)\n    gscv.fit(train)\n    print(f\"best params: {gscv.best_params_}\")\n    \n    # forecasting\n    fh=np.arange(len(test))+1\n    y_pred = gscv.predict(fh=fh)\n    mae, mape = plot_forecast(train, test, y_pred)\n    return mae, mape, y_pred\n    \nparam_grid = {\"window_length\": [20]} # parameter set to be grid searched\nforecaster = create_forecaster()\n\ny = df_price[0].iloc[-1000:-150].fillna(method='ffill')\ny.index = np.arange(y.index.shape[0])\n#y = (y.shift(periods=-16)/y.shift(periods=-1) - 1).dropna()\n\ntest_len = 50 #int(len(y) * 0.2)\nsun_train, sun_test = y.iloc[:-test_len], y.iloc[-test_len:]\nsun_lgb_mae, sun_lgb_mape, y_pred = grid_serch_forecaster(sun_train, sun_test, forecaster, param_grid)\nsun_lgb_mae, sun_lgb_mape","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.380118Z","iopub.execute_input":"2022-01-31T23:10:56.380359Z","iopub.status.idle":"2022-01-31T23:10:56.496005Z","shell.execute_reply.started":"2022-01-31T23:10:56.380329Z","shell.execute_reply":"2022-01-31T23:10:56.495216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\nsliced_data.plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.497704Z","iopub.execute_input":"2022-01-31T23:10:56.498214Z","iopub.status.idle":"2022-01-31T23:10:56.610879Z","shell.execute_reply.started":"2022-01-31T23:10:56.498169Z","shell.execute_reply":"2022-01-31T23:10:56.610084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\nvalid_df = pd.DataFrame(valid_data)\nvalid_df['pred'] = pd.Series(pred_data)\nvalid_df.plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.612722Z","iopub.execute_input":"2022-01-31T23:10:56.615006Z","iopub.status.idle":"2022-01-31T23:10:56.728005Z","shell.execute_reply.started":"2022-01-31T23:10:56.614971Z","shell.execute_reply":"2022-01-31T23:10:56.727229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#targets that have to be predicted\ndf_target = df.groupby([df.index, 'Asset_ID'])['Target'].last().unstack()\n\ndf_test, df_pred = next(iter_test)\n#env.predict(df_pred)\n\ndf_test['datetime'] = pd.to_datetime(df_test['timestamp'], unit='s')\ndf_test = df_test.set_index('datetime').drop('timestamp', axis=1)\ndf_test = df_test.groupby([df_test.index, 'Asset_ID'])['Close'].last().unstack()\ndf_test","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.729687Z","iopub.execute_input":"2022-01-31T23:10:56.731201Z","iopub.status.idle":"2022-01-31T23:10:56.848545Z","shell.execute_reply.started":"2022-01-31T23:10:56.731156Z","shell.execute_reply":"2022-01-31T23:10:56.847668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#for debugging public leaderboard\ndf_target.loc[df_test.index[0]]","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.850238Z","iopub.execute_input":"2022-01-31T23:10:56.850526Z","iopub.status.idle":"2022-01-31T23:10:56.967830Z","shell.execute_reply.started":"2022-01-31T23:10:56.850485Z","shell.execute_reply":"2022-01-31T23:10:56.967032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\n#for debugging public leaderboard\ndf_price_uplimit = df_price.iloc[:df_price.index.get_loc(df_test.index[0], method='nearest')+17]\ncalculate_target(df_price_uplimit.iloc[-5000:]).dropna().iloc[-1]","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:56.969572Z","iopub.execute_input":"2022-01-31T23:10:56.970112Z","iopub.status.idle":"2022-01-31T23:10:57.082256Z","shell.execute_reply.started":"2022-01-31T23:10:56.970064Z","shell.execute_reply":"2022-01-31T23:10:57.081451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#debug: create dummy \"iter_test\"\n\ntest_size = 250\n#ids = [0,1]\n#weights = weights[ids]\n\ndf_price = df.groupby([df.index, 'Asset_ID'])['Close'].last().unstack()[ids]\ndf_target = df.groupby([df.index, 'Asset_ID'])['Target'].last().unstack()[ids]\n\ncutoff_index = df_price.index.get_loc('2022-01-05 00:00:00', method='nearest')\n#iter_test = [(df_price.iloc[cutoff_index+i].to_frame().T, df_target.iloc[cutoff_index+i].to_frame().T) for i in range(10)]\niter_test = [(df_price.iloc[cutoff_index+i].to_frame().T, None) for i in range(test_size)]\n\ntest_price = df_price.iloc[cutoff_index:cutoff_index+test_size].values\ntest_target = df_target.iloc[cutoff_index:cutoff_index+test_size].values\ndf_price = df_price.iloc[:cutoff_index]","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:57.083985Z","iopub.execute_input":"2022-01-31T23:10:57.084601Z","iopub.status.idle":"2022-01-31T23:10:57.198911Z","shell.execute_reply.started":"2022-01-31T23:10:57.084556Z","shell.execute_reply":"2022-01-31T23:10:57.198100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#debug: directly predict targets\n\ni = 0\nk = 0\nregressor = None\nwindow_size = 360\nretrain_size = 0\ndataset_size = 1500\npred_price = []\npred_target = []\n\nfor df_test, df_pred in iter_test:\n    new_df = df_test.copy()\n    #new_df.index = pd.to_datetime(new_df['timestamp'], unit='s')\n    #new_df = new_df.groupby([new_df.index, 'Asset_ID'])['Close'].last().unstack()\n    #print('df_test: ',new_df[0].values)\n\n    if new_df.index[0]<=df_price.index[-1]:\n        last_index = df_price.index.get_loc(new_df.index[0], method='nearest')\n        df_price_uplimit = df_price.iloc[:last_index]\n    else:\n        df_price = df_price.append(new_df)\n        df_price_uplimit = df_price\n        \n    df_price_uplimit = calculate_target(df_price_uplimit.iloc[-5000:]).dropna()\n\n    #if i==0 or regressor is None:\n    regressor, test_data = train_model(df_price_uplimit, window_size)\n    #else:\n    #    test_data = df_price_uplimit.fillna(method='ffill').iloc[-window_size:].values\n\n    print('test_data', i, test_data.shape)\n    pred_data = predict_values(regressor, test_data, 1)\n    target = pred_data[0]\n    i = i+1 if i < retrain_size else 0\n    #pred_price.append(pred_data[0])\n    \n    #rmse_price = np.sqrt(mean_squared_error(pred_price, test_price[:len(pred_price)], multioutput='raw_values'))\n    pred_target.append(weighted_correlation(target, test_target[i], weights))\n    print(k, 'last metric:', pred_target[-1], 'mean metric:', np.mean(pred_target))\n    k += 1\n    \n    try:\n        for _, row in df_test.iterrows():\n            df_pred.loc[df_pred['row_id']==row['row_id'], 'Target'] = target[row['Asset_ID']]\n    except: \n        pass\n    \n    #env.predict(df_pred.fillna(0))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:57.201090Z","iopub.execute_input":"2022-01-31T23:10:57.201363Z","iopub.status.idle":"2022-01-31T23:10:57.320166Z","shell.execute_reply.started":"2022-01-31T23:10:57.201323Z","shell.execute_reply":"2022-01-31T23:10:57.319345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nk = 0\nregressor = None\nwindow_size = 360\nretrain_size = 10\ndataset_size = 1500\npred_price = []\npred_target = []\n\nfor df_test, df_pred in iter_test:\n    new_df = df_test.copy()\n    new_df.index = pd.to_datetime(new_df['timestamp'], unit='s')\n    new_df = new_df.groupby([new_df.index, 'Asset_ID'])['Close'].last().unstack()\n    #print('df_test: ',new_df[0].values)\n\n    num_preds = 17\n    if new_df.index[0]<=df_price.index[-1]:\n        last_index = df_price.index.get_loc(new_df.index[0], method='nearest') + num_preds\n        num_preds = last_index - (df_price.index.shape[0]-1)\n        df_price_uplimit = df_price if num_preds>=0 else df_price.iloc[:last_index]\n        num_preds = min(max(num_preds, 0), 17)\n    else:\n        df_price = df_price.append(new_df)\n        df_price_uplimit = df_price\n\n    print('num_preds: ',num_preds)\n    #we have to predict 17 steps above the limit\n    if num_preds>0:\n        if i==0 or regressor is None:\n            regressor, test_data = train_model(df_price_uplimit, window_size)\n        else:\n            test_data = np.append(test_data[1:], new_df.values, axis=0)\n            \n        print('test_data', i, test_data.shape)\n        pred_data = predict_values(regressor, test_data, num_preds)\n        df_price_uplimit = df_price_uplimit.append(pred_data)\n        i = i+1 if i < retrain_size else 0\n        #pred_price.append(pred_data[0])\n    \n    target = calculate_target(df_price_uplimit.iloc[-5000:]).dropna().iloc[-1]\n    \n    #rmse_price = np.sqrt(mean_squared_error(pred_price, test_price[:len(pred_price)], multioutput='raw_values'))\n    #pred_target.append(weighted_correlation(target.values, test_target[i], weights))\n    #print(k, 'last metric:', pred_target[-1], 'mean metric:', np.mean(pred_target))\n    #k += 1\n    \n    try:\n        for _, row in df_test.iterrows():\n            df_pred.loc[df_pred['row_id']==row['row_id'], 'Target'] = target[row['Asset_ID']]\n    except: \n        pass\n    \n    env.predict(df_pred.fillna(0))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:57.323357Z","iopub.execute_input":"2022-01-31T23:10:57.323891Z","iopub.status.idle":"2022-01-31T23:10:57.692502Z","shell.execute_reply.started":"2022-01-31T23:10:57.323856Z","shell.execute_reply":"2022-01-31T23:10:57.691830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n\ndtsize = len(pred_target)\npd.DataFrame(data={'pred':np.vstack(pred_target).T[0],'test':test_target[:dtsize,0]}).plot()\npd.DataFrame(data={'pred':np.vstack(pred_price).T[0], 'test':test_price[:dtsize,0]}).plot()\n#pd.DataFrame(data={'pred':pred_price[1], 'test':test_price[1]}).plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:57.693820Z","iopub.execute_input":"2022-01-31T23:10:57.694484Z","iopub.status.idle":"2022-01-31T23:10:57.805667Z","shell.execute_reply.started":"2022-01-31T23:10:57.694437Z","shell.execute_reply":"2022-01-31T23:10:57.804783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#debug: naive flow\n\nfor df_test, df_pred in iter_test:\n    new_df = df_test.copy()\n    new_df.index = pd.to_datetime(new_df['timestamp'], unit='s')\n    new_df = new_df.groupby([new_df.index, 'Asset_ID'])['Close'].last().unstack()\n    print('df_test index: ',new_df.index[0])\n\n    num_preds = 17\n    if new_df.index[0]<=df_price.index[-1]:\n        last_index = df_price.index.get_loc(new_df.index[0], method='nearest') + num_preds\n        num_preds = last_index - (df_price.index.shape[0]-1)\n        df_price_uplimit = df_price if num_preds>=0 else df_price.iloc[:last_index]\n        num_preds = min(max(num_preds, 0), 17)\n    else:\n        df_price = df_price.append(new_df)\n        df_price_uplimit = df_price\n\n    #we have to predict 17 steps above the limit\n    if num_preds>0:\n        df_price_uplimit = predict_values_naive(df_price_uplimit.copy(), num_preds)\n\n    target = calculate_target(df_price_uplimit.iloc[-5000:]).dropna().iloc[-1]\n    print('target index: ',target.name)\n    \n    try:\n        for _, row in df_test.iterrows():\n            df_pred.loc[df_pred['row_id']==row['row_id'], 'Target'] = target[row['Asset_ID']]\n    except: \n        pass\n    \n    env.predict(df_pred.fillna(0))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T23:10:57.807664Z","iopub.execute_input":"2022-01-31T23:10:57.808178Z","iopub.status.idle":"2022-01-31T23:10:57.920209Z","shell.execute_reply.started":"2022-01-31T23:10:57.808133Z","shell.execute_reply":"2022-01-31T23:10:57.919117Z"},"trusted":true},"execution_count":null,"outputs":[]}]}