{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### References: \n\n#### (1) [Tutorial to the G-Research Crypto Competition](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)\n#### (2) [Detailed API Introduction](https://www.kaggle.com/sohier/detailed-api-introduction)\n#### (3) [Recreating Target](https://www.kaggle.com/alexfir/recreating-target)\n#### (4) [Starter LGBM Pipeline](https://www.kaggle.com/julian3833/g-research-starter-lgbm-pipeline)\n#### (5) [How to submit lagged features](https://www.kaggle.com/tomforbes/gresearch-submitting-lagged-features-via-api)","metadata":{}},{"cell_type":"markdown","source":"## Import and load dfs","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport time\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import LinearRegression\nimport gresearch_crypto\nfrom pathlib import Path\nimport gc\nimport shutil\nfrom itertools import product\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\ndef read_csv_strict(file_name='/kaggle/input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    # embargo by 3 days\n    df = df[df['datetime'] < '2021-06-10 00:00:00']\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-31T03:53:49.972646Z","iopub.execute_input":"2022-01-31T03:53:49.973113Z","iopub.status.idle":"2022-01-31T03:53:52.149211Z","shell.execute_reply.started":"2022-01-31T03:53:49.972963Z","shell.execute_reply":"2022-01-31T03:53:52.148251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = read_csv_strict()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T03:53:52.151045Z","iopub.execute_input":"2022-01-31T03:53:52.151295Z","iopub.status.idle":"2022-01-31T03:54:52.415193Z","shell.execute_reply.started":"2022-01-31T03:53:52.151264Z","shell.execute_reply":"2022-01-31T03:54:52.41421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-31T03:54:52.416607Z","iopub.execute_input":"2022-01-31T03:54:52.416959Z","iopub.status.idle":"2022-01-31T03:54:52.438976Z","shell.execute_reply.started":"2022-01-31T03:54:52.416913Z","shell.execute_reply":"2022-01-31T03:54:52.438059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"### fill gaps in timestamps\n\ndfs = []\n\nts = df_train.timestamp.unique().tolist()\n\nfor name, g in df_train[['timestamp', 'Asset_ID']].set_index('timestamp').groupby('Asset_ID'):\n    # clearly target is not calculated with method='pad' on price variables\n    dfs += [g.reindex(range(g.index[0], g.index[-1]+60, 60), method='pad').reset_index()]\n    \ndf_train = pd.concat(dfs).sort_values('timestamp', kind='mergesort').reset_index(drop=True).merge(df_train, how='left', on =['Asset_ID', 'timestamp'])\ndf_train = df_train.loc[df_train.timestamp.isin(ts)].reset_index(drop=True)\n\ndel dfs, ts\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T03:54:52.440336Z","iopub.execute_input":"2022-01-31T03:54:52.440649Z","iopub.status.idle":"2022-01-31T03:55:16.21365Z","shell.execute_reply.started":"2022-01-31T03:54:52.440607Z","shell.execute_reply":"2022-01-31T03:55:16.212694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### start lag target calculations here\n\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)\n\ndf_train['lr15'] = df_train.groupby('Asset_ID')['Close'].apply(lambda x: log_return(x, 15))\ndf_train['lr16'] = df_train.groupby('Asset_ID')['Close'].apply(lambda x: log_return(x, 16))\ndf_train = df_train.merge(df_asset_details[['Asset_ID','Weight']], how='left', on = 'Asset_ID')\ndf_train['m'] = df_train['lr15']*df_train['Weight']\ndf_train['m'] = df_train.groupby('timestamp')['m'].transform('sum') / np.sum(df_asset_details['Weight'])\n\ndf_train = df_train.drop('Weight', axis=1)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T03:55:16.216365Z","iopub.execute_input":"2022-01-31T03:55:16.216692Z","iopub.status.idle":"2022-01-31T03:55:48.112719Z","shell.execute_reply.started":"2022-01-31T03:55:16.216648Z","shell.execute_reply":"2022-01-31T03:55:48.111647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)\n\n# # A utility function to build features around lags.\n# def get_features_hist(df, row=False):\n    \n#     ### features to consider. See potential features...\n#     ### note that we predicting returns 15 minutes ahead. This minutes price data is therefore not sufficient. We must roll the variables, 15, 30, 90, 250, 1250\n       \n#     df_feat = df[['Open', 'High', 'Low', 'Close', 'Volume', 'Count', 'VWAP', 'lr15', 'm', 'lr16', 'Asset_ID']].copy()\n\n#     if df_feat.shape[0]<3750:\n#         df_feat['beta_num'] = np.nan\n#         df_feat['m2'] = np.nan\n#     else:\n#         df_feat['beta_num'] = (df_feat['lr15']*df_feat['m']).rolling(3750).mean().values\n#         df_feat['m2'] = (df_feat['m']*df_feat['m']).rolling(3750).mean().values\n        \n#     if row:\n#         # first .iloc as far back as we need, compute feature, then downsize until .iloc[-1]\n#         df_feat = df_feat.iloc[-1]\n#         # Grab columns to divide\n#         divcols = ['Open', 'High', 'Low', 'Close', 'VWAP']\n#         # divide every element of divcols with every element of divcols\n#         df_feat = pd.concat([df_feat, pd.Series((df_feat[divcols].to_numpy()[:,np.newaxis] / df_feat[divcols].to_numpy()).reshape(len(divcols) * len(divcols)),\n#                                         index=map('/'.join, (product(divcols, divcols)))).drop(['Open/Open', 'High/High', 'Low/Low', 'Close/Close', 'VWAP/VWAP'])])\n#     else:\n#         # Grab columns to divide\n#         divcols = ['Open', 'High', 'Low', 'Close', 'VWAP']\n#         # divide every element of divcols with every element of divcols\n#         df_feat = pd.concat([df_feat.reset_index(drop=True), pd.DataFrame((df_feat[divcols].to_numpy()[..., None] / df_feat[divcols].to_numpy()[:, None]\n#                                                                            ).reshape((len(df_feat),len(divcols) * len(divcols))),\n#                                                                             columns=map('/'.join, (product(divcols, divcols))))], axis=1)\n#         df_feat = df_feat.drop(['Asset_ID','Open/Open', 'High/High', 'Low/Low', 'Close/Close', 'VWAP/VWAP'], axis=1)\n        \n#     df_feat['beta'] = np.nan_to_num(df_feat['beta_num'] / df_feat['m2'], nan=0., posinf=0., neginf=0.)\n#     df_feat['target_lag'] = df_feat['lr15'] - df_feat['beta']*df_feat['m']\n#     df_feat['volume2count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    \n#     return df_feat.replace([np.inf, -np.inf], np.nan)\n\n\n\n# A utility function to build features around lags.\ndef get_features_hist(df, row=False):\n    \n    ### features to consider. See potential features...\n    ### note that we predicting returns 15 minutes ahead. This minutes price data is therefore not sufficient. We must roll the variables, 15, 30, 90, 250, 1250\n       \n    df_feat = df[['Open', 'High', 'Low', 'Close', 'Volume', 'Count', 'VWAP', 'lr15', 'm', 'lr16', 'Asset_ID']].copy()\n\n    if df_feat.shape[0]<3750:\n        df_feat['beta_num'] = np.nan\n        df_feat['m2'] = np.nan\n    else:\n        df_feat['beta_num'] = (df_feat['lr15']*df_feat['m']).rolling(3750).mean().values\n        df_feat['m2'] = (df_feat['m']*df_feat['m']).rolling(3750).mean().values\n        \n    if row:\n        # first .iloc as far back as we need, compute feature, then downsize until .iloc[-1]\n        df_feat = df_feat.iloc[-1]\n#         mean_price = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n#         med_price = df_feat[['Open', 'High', 'Low', 'Close']].median()\n        df_feat['upper_shadow'] = df_feat['High'] / df_feat[['Close', 'Open']].max()\n        df_feat['lower_shadow'] = df_feat[['Close', 'Open']].min() / df_feat['Low']\n    else:\n#         mean_price = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n#         med_price = df_feat[['Open', 'High', 'Low', 'Close']].median(axis=1)\n        df_feat['upper_shadow'] = df_feat['High'] / df_feat[['Close', 'Open']].max(axis=1)\n        df_feat['lower_shadow'] = df_feat[['Close', 'Open']].min(axis=1) / df_feat['Low']\n        df_feat = df_feat.drop('Asset_ID', axis=1)\n        \n    df_feat['beta'] = np.nan_to_num(df_feat['beta_num'] / df_feat['m2'], nan=0., posinf=0., neginf=0.)\n    df_feat['target_lag'] = df_feat['lr15'] - df_feat['beta']*df_feat['m']\n\n        ### Sense checks\n#         print((df_feat['Target'] - df_feat.groupby('Asset_ID')['target_lag'].shift(-16)).abs().mean())\n#         print(df_feat.loc[(df_feat.Target.isnull())&(df_feat.target_lag.notnull())].shape)\n#         print(df_feat.loc[(df_feat.Target.notnull())&(df_feat.target_lag.isnull())].shape)        \n        \n    df_feat['open2close'] = df_feat['Close'] / df_feat['Open']\n    df_feat['high2low'] = df_feat['High'] / df_feat['Low']\n           \n#     df_feat['high2mean'] = df_feat['High'] / mean_price\n#     df_feat['low2mean'] = df_feat['Low'] / mean_price\n#     df_feat['high2median'] = df_feat['High'] / med_price\n#     df_feat['low2median'] = df_feat['Low'] / med_price\n    df_feat['volume2count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    df_feat['close2vwap'] = df_feat['Close'] / df_feat['VWAP']\n    \n    return df_feat.replace([np.inf, -np.inf], np.nan)\n\n# this function prevents leakage of data into other splits, not within split...\n# Adapted from numerai tournament\ndef get_time_series_cross_val_splits(data, cv = 5, embargo = 3750):\n    all_train_timestamps = data['timestamp'].unique()\n    len_split = len(all_train_timestamps) // cv\n    test_splits = [all_train_timestamps[i * len_split:(i + 1) * len_split] for i in range(cv)]\n    # fix the last test split to have all the last timestamps, in case the number of timestamps wasn't divisible by cv\n    rem = len(all_train_timestamps) - len_split*cv\n    if rem>0:\n        test_splits[-1] = np.append(test_splits[-1], all_train_timestamps[-rem:])\n\n    train_splits = []\n    for test_split in test_splits:\n        test_split_max = int(np.max(test_split))\n        test_split_min = int(np.min(test_split))\n        # get all of the timestamps that aren't in the test split\n        train_split_not_embargoed = [e for e in all_train_timestamps if not (test_split_min <= int(e) <= test_split_max)]\n        # embargo the train split so we have no leakage. Note timestamps are expressed in seconds, so multiply by 60\n        embargo_sec = 60*embargo\n        train_split = [e for e in train_split_not_embargoed if\n                       abs(int(e) - test_split_max) > embargo_sec and abs(int(e) - test_split_min) > embargo_sec]\n        train_splits.append(train_split)\n\n    # convenient way to iterate over train and test splits\n    train_test_zip = zip(train_splits, test_splits)\n    return train_test_zip\n\nMODEL_FOLDER = \"models\"\n\ndef save_model(model, name):\n    try:\n        Path(MODEL_FOLDER).mkdir(exist_ok=True, parents=True)\n    except Exception as ex:\n        pass\n    pd.to_pickle(model, f\"{MODEL_FOLDER}/{name}.pkl\")\n\n\ndef load_model(name):\n    path = Path(f\"{MODEL_FOLDER}/{name}.pkl\")\n    if path.is_file():\n        model = pd.read_pickle(f\"{MODEL_FOLDER}/{name}.pkl\")\n    else:\n        path = Path(f\"/kaggle/input/lgbm-embargocv-weightedpearson-lagtarget/{MODEL_FOLDER}/{name}.pkl\")\n        if path.is_file():\n            model = pd.read_pickle(f\"/kaggle/input/lgbm-embargocv-weightedpearson-lagtarget/{MODEL_FOLDER}/{name}.pkl\")\n        else:\n            model = False\n    return model\n\n\ndef get_model_and_valid_preds_per_asset(df, asset_id):\n    \n    df_proc = get_features_hist(df.copy())\n    df_proc['y'] = df['Target'].values\n    df_proc['timestamp'] = df['timestamp'].values\n    feature_cols = df_proc.drop(['timestamp', 'y'], axis=1).columns\n    impfeats = np.zeros(len(feature_cols))\n    \n    model_params = {\"n_estimators\": 100,\n                    \"learning_rate\": 0.01,\n                    \"max_depth\": 4,\n                    \"num_leaves\": (2 ** 4 - 1),\n                    \"colsample_bytree\": 0.8,\n                    \"subsample\": 0.8}\n    # keep track of some prediction columns\n#     ensemble_cols = set()\n#     pred_cols = set()\n    \n    ncv = 5\n    train_test_zip = get_time_series_cross_val_splits(df_proc, cv = ncv)\n    # get out of sample training preds via embargoed time series cross validation\n    print(\"entering time series cross validation loop\")\n    for split, train_test_split in enumerate(train_test_zip):\n        gc.collect()\n        print(f\"doing split {split+1} out of {ncv}\")\n        train_split, test_split = train_test_split\n        train_split_index = df_proc['timestamp'].isin(train_split)\n        test_split_index = df_proc['timestamp'].isin(test_split)\n\n        # train a model on the training split (and save it for future use)\n        split_model_name = f\"model_asset{asset_id}_split{split+1}_cv{ncv}\"\n        split_model = load_model(split_model_name)\n        if not split_model:\n            print(f\"training model: {split_model_name}\")\n            split_model = LGBMRegressor(**model_params)\n            split_model.fit(df_proc.loc[train_split_index, feature_cols], df_proc.loc[train_split_index,'y'])\n            save_model(split_model, split_model_name)\n        \n        impfeats += split_model.feature_importances_ / ncv\n        # now we can predict on the test part of the split\n        expected_features = split_model.booster_.feature_name()\n        print(f\"predicting {split_model_name}\")\n        df_proc.loc[test_split_index, 'preds'] = split_model.predict(df_proc.loc[test_split_index, expected_features])\n        \n        # lets try adjusting for negative correlations... simple pearson for now\n        check_corrs = df_proc.loc[test_split_index, 'preds'].corr(df_proc.loc[test_split_index, 'y'])\n        print(check_corrs)\n        neglist = []\n        if check_corrs<0:\n            df_proc.loc[test_split_index, 'preds'] = -df_proc.loc[test_split_index, 'preds']\n            neglist+=[split_model_name]\n       \n        # sense check\n        print(np.count_nonzero(np.isnan(df_proc.loc[test_split_index, 'preds'])))\n\n        # remember that we made all of these different pred columns\n#         pred_cols.add(f\"preds_{split_model_name}\")\n    # sense check\n    print(np.count_nonzero(np.isnan(df_proc['preds'])))\n        # To Do:  add neutralization of most riskiest features. 10  should be fine...\n    display(pd.DataFrame({'Value': impfeats, 'Feature': feature_cols}).sort_values(by=\"Value\", ascending=False).head(5))    \n#     print(\"creating ensemble\")\n#     df_proc[\"ensemble_all\"] = df_proc[pred_cols].mean(axis=1)\n#     ensemble_cols.add(\"ensemble_all\")        \n        \n    return df_proc['preds'], neglist","metadata":{"execution":{"iopub.status.busy":"2022-01-31T03:59:22.631391Z","iopub.execute_input":"2022-01-31T03:59:22.631783Z","iopub.status.idle":"2022-01-31T03:59:22.674224Z","shell.execute_reply.started":"2022-01-31T03:59:22.631738Z","shell.execute_reply":"2022-01-31T03:59:22.673102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = 5\npath = Path(f\"/kaggle/input/lgbm-embargocv-weightedpearson-lagtarget/{MODEL_FOLDER}/model_asset13_split1_cv{cv}.pkl\")\npath2 = Path(f\"{MODEL_FOLDER}/model_asset13_split1_cv{cv}.pkl\")\nif path2.is_file()|path.is_file():\n    print(\"models already exist\")\n    expected_features = load_model(f\"model_asset13_split1_cv{cv}\").booster_.feature_name()\n    acond = df_train[\"Asset_ID\"] == 0\n    feature_cols = get_features_hist(df_train.loc[acond].iloc[-3750:]).columns\n    if set(expected_features) != set(feature_cols):\n        print(f\"New features are available! Might want to retrain the model.\")\n        shutil.rmtree(MODEL_FOLDER)\nelse:\n    weights=[]\n    corrdf = pd.DataFrame()\n    neglist = []\n    for asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n        print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n        acond = df_train[\"Asset_ID\"] == asset_id\n        val_preds, neglist_asset = get_model_and_valid_preds_per_asset(df_train.loc[acond], asset_id)\n        neglist += neglist_asset\n        val_df = pd.DataFrame(val_preds.values, columns=['preds'])\n        val_df['Target'] = df_train.loc[acond, 'Target'].values\n        val_df['timestamp'] = df_train.loc[acond, 'timestamp'].values\n        val_df['Weight'] = df_asset_details.loc[df_asset_details[\"Asset_ID\"] == asset_id, 'Weight'].values[0]\n        corrdf = pd.concat([corrdf, val_df])\n\n    with open('neglist.txt', 'w') as filehandle:\n        for listitem in neglist:\n            filehandle.write('%s\\n' % listitem)\n        \n    corrdf['wpreds'] = corrdf['Weight']*corrdf['preds']\n    corrdf['wpreds_mean'] = corrdf.groupby('timestamp')['wpreds'].transform('sum') / corrdf.groupby('timestamp')['Weight'].transform('sum')\n    corrdf['preds_mean'] = corrdf.groupby('timestamp')['preds'].transform('mean')\n    \n    corrdf = corrdf.dropna()\n    print(corrdf.shape)\n    \n    # adjusting predictions\n    corrdf['preds2'] = 0.8*corrdf['preds'] + 0.2*corrdf['preds_mean']\n    corrdf['preds3'] = 0.9*corrdf['preds'] + 0.1*corrdf['preds_mean']\n    corrdf['preds4'] = 0.8*corrdf['preds'] + 0.2*corrdf['wpreds_mean']\n    corrdf['preds5'] = 0.9*corrdf['preds'] + 0.1*corrdf['wpreds_mean']\n    \n    def weighted_pearson(a,b):\n        w = corrdf.Weight.values\n        sumw = w.sum()\n        ma = (w*a).sum() / sumw\n        mb = (w*b).sum() / sumw\n\n        # updated in line with competition guidelines\n        covab = (a*b*w).sum() / sumw - ma*mb #(w*(a-ma)*(b-mb)).sum()/sumw\n        covaa = (w*(a-ma)*(a-ma)).sum() / sumw\n        covbb = (w*(b-mb)*(b-mb)).sum() / sumw\n\n        return covab/np.sqrt(covaa*covbb)\n\n    display(corrdf[['Target','preds', 'preds2', 'preds3', 'preds4', 'preds5']].corr(method=weighted_pearson))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T03:59:27.0856Z","iopub.execute_input":"2022-01-31T03:59:27.085938Z","iopub.status.idle":"2022-01-31T04:04:15.383381Z","shell.execute_reply.started":"2022-01-31T03:59:27.085901Z","shell.execute_reply":"2022-01-31T04:04:15.382361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model interface\nrowtrain = df_train.iloc[-1]\nrowhist = df_train.loc[df_train.Asset_ID==rowtrain['Asset_ID']].iloc[-3750:]\nx = get_features_hist(rowhist, row=True).drop('Asset_ID')\ncv = 5\ny_pred = 0\nfor split in range(cv):\n    split_model_name = f\"model_asset{rowtrain['Asset_ID']}_split{split+1}_cv{cv}\"\n    split_model = load_model(split_model_name)\n    y_pred += split_model.predict([x])[0] / cv\nprint(y_pred)\n\n# define an empty list\nneglist = []\n# open file and read the content in a list\nif path2.is_file():\n    with open('neglist.txt', 'r') as filehandle:\n        for line in filehandle:\n            # remove linebreak which is the last character of the string\n            currentName = line[:-1]\n            # add item to the list\n            neglist.append(currentName)\nelif path.is_file():\n    with open('/kaggle/input/lgbm-embargocv-weightedpearson-lagtarget/neglist.txt', 'r') as filehandle:\n        for line in filehandle:\n            # remove linebreak which is the last character of the string\n            currentName = line[:-1]\n            # add item to the list\n            neglist.append(currentName)\nelse:\n    with open('neglist.txt', 'r') as filehandle:\n        for line in filehandle:\n            # remove linebreak which is the last character of the string\n            currentName = line[:-1]\n            # add item to the list\n            neglist.append(currentName)\n\ndel x, df_train, split_model, rowtrain, rowhist\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T04:07:19.496517Z","iopub.execute_input":"2022-01-31T04:07:19.496843Z","iopub.status.idle":"2022-01-31T04:07:20.089504Z","shell.execute_reply.started":"2022-01-31T04:07:19.496811Z","shell.execute_reply":"2022-01-31T04:07:20.088512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nhistory = pd.DataFrame()\nlenassets = len(df_asset_details)\nhistmax = 3750*lenassets\nretmax = 16*lenassets\n\nsumweights = np.sum(df_asset_details['Weight'])\n\ncv = 5\nrangecv = range(cv)\nsplit_models = {}\nfor r in range(14):\n    for split in rangecv:\n        split_model_name = f\"model_asset{r}_split{split+1}_cv{cv}\"\n        split_models[split_model_name] = load_model(split_model_name)\n\ndef pred_func(df):\n    df_feat = get_features_hist(df, row=True)\n    asset_id = int(df_feat.Asset_ID)\n    df_feat = df_feat.drop('Asset_ID')\n    negswitch = []\n    for i in range(5):\n        if f\"model_asset{asset_id}_split{i+1}_cv5\" in neglist:\n            negswitch+=[-1]\n        else:\n            negswitch+=[1]\n            \n    y_pred = (negswitch[0]*split_models[f\"model_asset{asset_id}_split1_cv5\"].predict([df_feat])[0] + negswitch[1]*split_models[\n                f\"model_asset{asset_id}_split2_cv5\"].predict([df_feat])[0] + negswitch[2]*split_models[f\"model_asset{asset_id}_split3_cv5\"].predict(\n                [df_feat])[0] + negswitch[3]*split_models[f\"model_asset{asset_id}_split4_cv5\"].predict([df_feat])[0] + negswitch[4]*split_models[\n                f\"model_asset{asset_id}_split5_cv5\"].predict([df_feat])[0]) / cv\n    return y_pred","metadata":{"execution":{"iopub.status.busy":"2022-01-31T04:07:31.479406Z","iopub.execute_input":"2022-01-31T04:07:31.479738Z","iopub.status.idle":"2022-01-31T04:07:31.720313Z","shell.execute_reply.started":"2022-01-31T04:07:31.479706Z","shell.execute_reply":"2022-01-31T04:07:31.712867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict & submit\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T20:57:49.349459Z","iopub.status.idle":"2021-11-02T20:57:49.349757Z","shell.execute_reply.started":"2021-11-02T20:57:49.349596Z","shell.execute_reply":"2021-11-02T20:57:49.349613Z"}}},{"cell_type":"code","source":"for (df_test, df_pred) in iter_test:\n#     t0 = time.time()\n\n    # filling gaps with missing assets\n    history = pd.concat([history, df_test[['Asset_ID', 'Close', 'row_id', 'Open', 'High', 'Low', 'Volume', 'Count', \n                                           'VWAP']].set_index('Asset_ID').reindex(df_asset_details['Asset_ID'].values).reset_index()], ignore_index=True)\n    \n    test_rows = history.row_id.isin(df_test.row_id)\n    test_assets = history.Asset_ID.isin(df_test.Asset_ID)\n    weights = df_asset_details.loc[df_asset_details.Asset_ID.isin(df_test.Asset_ID), 'Weight'].values\n       \n    lenhist = history.shape[0]\n    \n    # ensuring sufficient history for each asset\n    if lenhist>histmax:\n        history = history.iloc[-histmax:]\n\n#     # because 'm' requires the whole group of assets, it must be computed prior to the loop below... \n    history_test = history.loc[test_rows, ['Asset_ID', 'Close']].copy()\n    \n    if lenhist<(retmax+1):\n        history_test['lr15'] = np.nan\n        history_test['lr16'] = np.nan\n    else:\n        # .take([-16,-17]).loc[history_test.Asset_ID]\n        takes = history.loc[test_assets].groupby('Asset_ID')['Close'].take([-16,-17]).groupby(level=0)\n        history_test['lr15'] = np.log(history_test['Close'].values / takes.take([0]).values)\n        history_test['lr16'] = np.log(history_test['Close'].values / takes.take([1]).values)\n        \n    history_test['m'] = (history_test['lr15'].values*weights).sum() / sumweights\n    history.loc[test_rows, ['lr15', 'm', 'lr16']] = history_test[['lr15', 'm', 'lr16']].values\n    \n    df_pred['Target'] = history.loc[test_assets].groupby('Asset_ID').apply(pred_func).reindex(df_test['Asset_ID'].values).values\n    \n    # try average by weight instead, but careful as some assets are not avaialble\n#     df_pred.loc['Target'] = 0.8*df_pred.loc['Target'] + 0.2*df_pred.loc['Target'].mean()\n\n    # Send submissions\n    env.predict(df_pred)\n#     print(time.time()-t0)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T04:07:33.871002Z","iopub.execute_input":"2022-01-31T04:07:33.871818Z","iopub.status.idle":"2022-01-31T04:07:34.554666Z","shell.execute_reply.started":"2022-01-31T04:07:33.871772Z","shell.execute_reply":"2022-01-31T04:07:34.553879Z"},"trusted":true},"execution_count":null,"outputs":[]}]}