{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G-Research Crypto - Starter XGB Pipeline\n![](https://storage.googleapis.com/kaggle-competitions/kaggle/30894/logos/header.png)\n\n\n### Just a simple pipeline going from zero to a valid submission\n\n\n","metadata":{}},{"cell_type":"markdown","source":"# Import and load dfs\n\nReferences: [Tutorial to the G-Research Crypto Competition](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\nimport xgboost as xgb\nimport traceback\nimport datetime\nimport matplotlib.pyplot as plt\nfrom scipy.stats import pearsonr\n\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"execution":{"iopub.status.busy":"2021-11-15T16:48:19.532871Z","iopub.execute_input":"2021-11-15T16:48:19.533701Z","iopub.status.idle":"2021-11-15T16:48:22.435628Z","shell.execute_reply.started":"2021-11-15T16:48:19.533609Z","shell.execute_reply":"2021-11-15T16:48:22.434779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype.name\n\n        if col_type not in ['object', 'category', 'datetime64[ns, UTC]']:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-15T16:48:26.539931Z","iopub.execute_input":"2021-11-15T16:48:26.540194Z","iopub.status.idle":"2021-11-15T16:48:26.553740Z","shell.execute_reply.started":"2021-11-15T16:48:26.540165Z","shell.execute_reply":"2021-11-15T16:48:26.552580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = 'GPU'\n\ndf = pd.read_csv(TRAIN_CSV)\ndf.replace([np.inf, -np.inf], np.nan)\n\n#df_train.fillna(-999, inplace=True)\n# df = df[df['Target'].notna()]\n# df.interpolate(method='linear', inplace=True)\ndf = df.dropna(how=\"any\")\n\n# sorting data into groups of days\ndf['date'] = pd.to_datetime(df['timestamp'], unit = 's')\ndf = df.sort_values('date')\ngroups = pd.factorize(df['date'].dt.day.astype(str) + '_' + df['date'].dt.month.astype(str) + '_' + df['date'].dt.year.astype(str))[0]\n\n# reduce memory usage\ndf.drop(columns = 'date', inplace = True)\ntarget = df['Target'].copy()\ndf.drop(columns = 'Target', inplace = True)\ndf = reduce_mem_usage(df)\ndf['Target'] = target\ndf['groups'] = groups\n\n# getting rid of data overlap\ndf_train = df[df['timestamp'] < 1623542400]\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T16:48:26.746830Z","iopub.execute_input":"2021-11-15T16:48:26.747040Z","iopub.status.idle":"2021-11-15T16:51:17.509013Z","shell.execute_reply.started":"2021-11-15T16:48:26.747014Z","shell.execute_reply":"2021-11-15T16:51:17.507066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-11-15T16:51:17.511069Z","iopub.execute_input":"2021-11-15T16:51:17.511308Z","iopub.status.idle":"2021-11-15T16:51:17.548172Z","shell.execute_reply.started":"2021-11-15T16:51:17.511272Z","shell.execute_reply":"2021-11-15T16:51:17.547252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility functions to train a model for one asset","metadata":{}},{"cell_type":"code","source":"\nfrom pandas import DataFrame\nfrom pandas import concat\n \ndef series_to_supervised(data, n_in=1, n_out=1, dropnan=False, interpolate = False):\n    \"\"\"\n    Frame a time series as a supervised learning dataset.\n    Arguments:\n        data: Sequence of observations as a list or NumPy array.\n        n_in: Number of lag observations as input (X).\n        n_out: Number of observations as output (y).\n        dropnan: Boolean whether or not to drop rows with NaN values.\n    Returns:\n        Pandas DataFrame of series framed for supervised learning.\n    \"\"\"\n    n_vars = 1 if type(data) is list else data.shape[1]\n    df = DataFrame(data)\n    cols, names = list(), list()\n    # input sequence (t-n, ... t-1)\n    for i in range(n_in, 0, -1):\n        cols.append(df.shift(i))\n        names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]\n    # forecast sequence (t, t+1, ... t+n)\n    for i in range(0, n_out):\n        cols.append(df.shift(-i))\n        if i == 0:\n            names += [('var%d(t)' % (j+1)) for j in range(n_vars)]\n        else:\n            names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]\n    # put it all together\n    agg = concat(cols, axis=1)\n    agg.columns = names\n    # drop rows with NaN values\n    if dropnan:\n        agg.dropna(inplace=True)\n    elif interpolate:\n        agg.interpolate(method='linear', inplace=True)\n    else:\n        agg.fillna(-999,inplace=True)\n    return agg\n ","metadata":{"execution":{"iopub.status.busy":"2021-11-15T17:53:41.869359Z","iopub.execute_input":"2021-11-15T17:53:41.869642Z","iopub.status.idle":"2021-11-15T17:53:41.879723Z","shell.execute_reply.started":"2021-11-15T17:53:41.869612Z","shell.execute_reply":"2021-11-15T17:53:41.878709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df, lag = 1, shuffle = False, less_features = False, diff = True):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    \n    #imputer\n#     df_feat = df_feat.interpolate()\n    df_feat.fillna(-999,inplace=True)\n    #less features\n    if less_features:\n        df_feat = df_feat[['Close', 'Volume','Count']]\n    if diff:\n        df_feat = df_feat.diff()\n    # add lagged observations\n    if lag > 0:\n        df_feat = series_to_supervised(df_feat, n_in = lag)\n    df_feat['Upper_Shadow'] = upper_shadow(df)\n    df_feat['Lower_Shadow'] = lower_shadow(df)\n    if shuffle is True:\n        df_feat = df_feat.sample(frac=1)\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2021-11-15T17:53:42.943600Z","iopub.execute_input":"2021-11-15T17:53:42.944308Z","iopub.status.idle":"2021-11-15T17:53:42.954267Z","shell.execute_reply.started":"2021-11-15T17:53:42.944269Z","shell.execute_reply":"2021-11-15T17:53:42.953310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Building the model","metadata":{}},{"cell_type":"code","source":"# Build pipeline here, imputer might not be necessary as all nans were dropped in the beginning.\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import roc_auc_score, mean_absolute_error\nfrom sklearn import set_config\nset_config(display='diagram') \ndef XGB_pipeline():\n#     imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')\n    scaler = StandardScaler()\n#     scaler = MinMaxScaler()\n\n    #parameters from here: https://www.kaggle.com/sarvesh278/xgboost-trading-classifier\n    #note: original is a classifier\n    XGB_model = xgb.XGBRegressor(\n        n_estimators=508,\n#         learning_rate=0.05,\n#         max_depth=12,\n#         subsample=0.9,\n#         colsample_bytree=0.7,\n        #colsample_bylevel=0.75,\n        missing=-999,\n        max_depth= 8,\n        learning_rate = 0.06992689459063349,\n        subsample = 0.8012753784867586,\n        colsample_bytree = 0.8419498887494685,\n#         gamma = 9,\n#         reg_lambda = 5,\n        random_state=1111,\n        tree_method='gpu_hist'  \n        )\n\n    pipe = Pipeline(steps=[\n#         ('imputer', imp_mean),\n        ('scaler', scaler),\n        ('linear', XGB_model)\n    ])\n    \n    return pipe","metadata":{"execution":{"iopub.status.busy":"2021-11-15T17:53:48.670348Z","iopub.execute_input":"2021-11-15T17:53:48.670622Z","iopub.status.idle":"2021-11-15T17:53:48.677516Z","shell.execute_reply.started":"2021-11-15T17:53:48.670592Z","shell.execute_reply":"2021-11-15T17:53:48.676811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = df_train[df_train[\"Asset_ID\"] == 10]\n# df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T17:53:51.504805Z","iopub.execute_input":"2021-11-15T17:53:51.505589Z","iopub.status.idle":"2021-11-15T17:53:51.510833Z","shell.execute_reply.started":"2021-11-15T17:53:51.505511Z","shell.execute_reply":"2021-11-15T17:53:51.510131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = df_train[df_train[\"Asset_ID\"] == 0]\n# #impute y\n# df['Target'] = df.Target.interpolate(method='slinear')\n# df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T17:53:51.756082Z","iopub.execute_input":"2021-11-15T17:53:51.756588Z","iopub.status.idle":"2021-11-15T17:53:51.760709Z","shell.execute_reply.started":"2021-11-15T17:53:51.756528Z","shell.execute_reply":"2021-11-15T17:53:51.759677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Check the model interface\n# x = get_features(df_train.iloc[1])\n# #y_pred = models[0].predict([x])\n# #y_pred[0]\n# y_pred = models[0].predict(pd.DataFrame([x]))\n# y_pred[0]","metadata":{"execution":{"iopub.status.busy":"2021-11-15T17:53:51.981723Z","iopub.execute_input":"2021-11-15T17:53:51.981954Z","iopub.status.idle":"2021-11-15T17:53:51.986554Z","shell.execute_reply.started":"2021-11-15T17:53:51.981929Z","shell.execute_reply":"2021-11-15T17:53:51.985606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross Validation","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.utils.validation import _deprecate_positional_args\nfrom sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\n\n# modified code for group gaps; source\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass PurgedGroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n    Allows for a gap in groups to avoid potentially leaking info from\n    train into test if the model has windowed or lag features.\n    Provides train/test indices to split time series data samples\n    that are observed at fixed time intervals according to a\n    third-party provided group.\n    In each split, test indices must be higher than before, and thus shuffling\n    in cross validator is inappropriate.\n    This cross-validation object is a variation of :class:`KFold`.\n    In the kth split, it returns first k folds as train set and the\n    (k+1)th fold as test set.\n    The same group will not appear in two different folds (the number of\n    distinct groups has to be at least equal to the number of folds).\n    Note that unlike standard cross-validation methods, successive\n    training sets are supersets of those that come before them.\n    Read more in the :ref:`User Guide <cross_validation>`.\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_group_size : int, default=Inf\n        Maximum group size for a single training set.\n    group_gap : int, default=None\n        Gap between train and test\n    max_test_group_size : int, default=Inf\n        We discard this number of groups from the end of each train split\n    \"\"\"\n\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_group_size=np.inf,\n                 max_test_group_size=np.inf,\n                 group_gap=None,\n                 verbose=False\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_group_size = max_train_group_size\n        self.group_gap = group_gap\n        self.max_test_group_size = max_test_group_size\n        self.verbose = verbose\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        group_gap = self.group_gap\n        max_test_group_size = self.max_test_group_size\n        max_train_group_size = self.max_train_group_size\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n\n        group_test_size = min(n_groups // n_folds, max_test_group_size)\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n\n            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n                train_array_tmp = group_dict[train_group_idx]\n                \n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n\n            train_end = train_array.size\n \n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n\n            test_array  = test_array[group_gap:]\n            \n            \n            if self.verbose > 0:\n                    pass\n                    \n            yield [int(i) for i in train_array], [int(i) for i in test_array]\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\n\n# this is code slightly modified from the sklearn docs here:\n# https://scikit-learn.org/stable/auto_examples/model_selection/plot_cv_indices.html#sphx-glr-auto-examples-model-selection-plot-cv-indices-py\ndef plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10):\n    \"\"\"Create a sample plot for indices of a cross-validation object.\"\"\"\n    \n    cmap_cv = plt.cm.coolwarm\n\n    jet = plt.cm.get_cmap('jet', 256)\n    seq = np.linspace(0, 1, 256)\n    _ = np.random.shuffle(seq)   # inplace\n    cmap_data = ListedColormap(jet(seq))\n\n    # Generate the training/testing visualizations for each CV split\n    for ii, (tr, tt) in enumerate(cv.split(X=X, y=y, groups=group)):\n        # Fill in indices with the training/test groups\n        indices = np.array([np.nan] * len(X))\n        indices[tt] = 1\n        indices[tr] = 0\n\n        # Visualize the results\n        ax.scatter(range(len(indices)), [ii + .5] * len(indices),\n                   c=indices, marker='_', lw=lw, cmap=cmap_cv,\n                   vmin=-.2, vmax=1.2)\n\n    # Plot the data classes and groups at the end\n    ax.scatter(range(len(X)), [ii + 1.5] * len(X),\n               c=y, marker='_', lw=lw, cmap=plt.cm.Set3)\n\n    ax.scatter(range(len(X)), [ii + 2.5] * len(X),\n               c=group, marker='_', lw=lw, cmap=cmap_data)\n\n    # Formatting\n    yticklabels = list(range(n_splits)) + ['target', 'day']\n    ax.set(yticks=np.arange(n_splits+2) + .5, yticklabels=yticklabels,\n           xlabel='Sample index', ylabel=\"CV iteration\",\n           ylim=[n_splits+2.2, -.2], xlim=[0, len(y)])\n    ax.set_title('{}'.format(type(cv).__name__), fontsize=15)\n    return ax","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-15T16:51:17.648528Z","iopub.execute_input":"2021-11-15T16:51:17.649613Z","iopub.status.idle":"2021-11-15T16:51:17.697799Z","shell.execute_reply.started":"2021-11-15T16:51:17.649574Z","shell.execute_reply":"2021-11-15T16:51:17.695808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, ax = plt.subplots()\n\n# feature_names = [i for i in df_train.columns if i not in ['Target', 'date', 'timestamp', 'VWAP', 'Asset_ID', 'groups']]\n# cv = PurgedGroupTimeSeriesSplit(\n#     n_splits=3,\n# #     max_train_group_size=150,\n#     group_gap=20,\n# #     max_test_group_size=60\n# )\n\n# plot_cv_indices(\n#     cv,    \n#     df_train.loc[df_train[\"Asset_ID\"] == 1][feature_names].values,\n#     df_train.loc[df_train[\"Asset_ID\"] == 1]['Target'].values > np.nanmean(df_train.loc[df_train[\"Asset_ID\"] == 1]['Target'].values),\n#     df_train.loc[df_train[\"Asset_ID\"] == 1]['groups'].values,\n#     ax,\n#     5,\n#     lw=20\n# );","metadata":{"execution":{"iopub.status.busy":"2021-11-15T16:51:17.703637Z","iopub.execute_input":"2021-11-15T16:51:17.706616Z","iopub.status.idle":"2021-11-15T16:51:17.713759Z","shell.execute_reply.started":"2021-11-15T16:51:17.706577Z","shell.execute_reply":"2021-11-15T16:51:17.712759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna\n\n# get_features_params = {'lag':1,'less_features': True, 'shuffle':False, 'diff':True}\nfeature_names = [i for i in df_train.columns if i not in ['Target', 'date', 'timestamp' 'Asset_ID', 'groups']]\n# Testing with Asset_ID == 0 for now\nasset_df = df_train[df_train['Asset_ID'] == 0]\ny_labels = asset_df['Target'].values\n# X_train = asset_df[feature_names].values\n# X_train = get_features(asset_df[feature_names], **get_features_params).values\ngroups = asset_df['groups'].values\n\ncv = PurgedGroupTimeSeriesSplit(\n    n_splits=3,\n    max_train_group_size=160,\n    group_gap=5,\n    max_test_group_size=60\n)\n\ndef objective(trial, cv=cv, cv_fold_func=np.average):\n    \n    # Optuna suggest params for feature engineering\n    feature_params = {\n        'lag':trial.suggest_int('lag', 0, 20),\n        'less_features': trial.suggest_int('less_features',0, 1), \n        'shuffle': trial.suggest_int('shuffle',0, 1), \n        'diff': trial.suggest_int('diff',0, 1)\n        }\n    X_train = get_features(asset_df[feature_names], **feature_params).values\n    # Optuna suggest params for model\n    params = {\n        'n_estimators': trial.suggest_int('n_estimators', 100, 1000),\n        'max_depth': trial.suggest_int('max_depth', 3, 10),\n        'learning_rate': trial.suggest_uniform('learning_rate', 0.01, 0.10),\n        'subsample': trial.suggest_uniform('subsample', 0.50, 0.90),\n        'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.50, 0.90),\n#         'gamma': trial.suggest_int('gamma', 0, 1),\n        'missing': -999,        \n        }\n    \n    if DEVICE == 'GPU': params['tree_method'] = 'gpu_hist'  \n    # setup the pipeline\n#     imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')\n    scaler = StandardScaler()\n    clf = xgb.XGBRegressor(**params)\n\n    pipe = Pipeline(steps=[\n#         ('imputer', imp_mean),\n        ('scaler', scaler),\n        ('xgb', clf)\n    ])\n\n\n    # fit for all folds and return composite MAE score\n    maes = []\n    correlations = []\n    for i, (train_idx, valid_idx) in enumerate(cv.split(\n        X_train,\n        y_labels,\n        groups=groups)):\n        \n        train_data = X_train[train_idx, :], y_labels[train_idx]\n        valid_data = X_train[valid_idx, :], y_labels[valid_idx]\n        \n#         display(X_train[valid_idx, :])\n        pipe.fit(X_train[train_idx, :], y_labels[train_idx])\n        preds = pipe.predict(X_train[valid_idx, :])\n        mae = mean_absolute_error(y_labels[valid_idx], preds)\n        correlation = pearsonr(y_labels[valid_idx], preds)[0]\n#         print(preds)\n        maes.append(mae)\n        correlations.append(correlation)\n    \n    print(f'Trial done: mae values on folds: {maes}, correlation: {correlations}')\n#     return -1.0 * cv_fold_func(maes)\n    return cv_fold_func(correlations)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-15T18:21:21.717125Z","iopub.execute_input":"2021-11-15T18:21:21.717407Z","iopub.status.idle":"2021-11-15T18:21:21.893014Z","shell.execute_reply.started":"2021-11-15T18:21:21.717378Z","shell.execute_reply":"2021-11-15T18:21:21.892136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nFIT_XGB = True\n\nn_trials = 60\n\nif FIT_XGB:\n    study = optuna.create_study(direction=\"maximize\")\n    study.optimize(objective, n_trials=n_trials)\n\n    print(\"Number of finished trials: {}\".format(len(study.trials)))\n\n    print(\"Best trial:\")\n    trial = study.best_trial\n\n    print(\"  Value: {}\".format(trial.value))\n\n    print(\"  Params: \")\n    for key, value in trial.params.items():\n        print(\"    {}: {}\".format(key, value))\n    best_params = trial.params        \nelse: best_params = {}","metadata":{"execution":{"iopub.status.busy":"2021-11-15T18:21:23.683886Z","iopub.execute_input":"2021-11-15T18:21:23.684407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# maes = []\n# correlations = []\n# for i, (train_idx, valid_idx) in enumerate(cv.split(\n#     X_train,\n#     y_labels,\n#     groups=groups)):\n\n#     train_data = X_train[train_idx, :], y_labels[train_idx]\n#     valid_data = X_train[valid_idx, :], y_labels[valid_idx]\n\n# #         display(X_train[valid_idx, :])\n#     pipe = XGB_pipeline()\n#     pipe.fit(X_train[train_idx, :], y_labels[train_idx])\n#     preds = pipe.predict(X_train[valid_idx, :])\n#     mae = mean_absolute_error(y_labels[valid_idx], preds)\n#     correlation = pearsonr(y_labels[valid_idx], preds)[0]\n# #         print(preds)\n#     maes.append(mae)\n#     correlations.append(correlation)\n# correlations","metadata":{"execution":{"iopub.status.busy":"2021-11-15T17:29:21.876359Z","iopub.execute_input":"2021-11-15T17:29:21.876626Z","iopub.status.idle":"2021-11-15T17:29:43.063127Z","shell.execute_reply.started":"2021-11-15T17:29:21.876595Z","shell.execute_reply":"2021-11-15T17:29:43.062390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_train, asset_id, lag = 1, shuffle = False, less_features = False, diff = True):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    #impute y, not working\n#     df['Target'] = df.Target.interpolate(method='slinear')\n    # TODO: Try different features here!\n    df_proc = get_features(df, lag = lag, shuffle = shuffle, less_features = less_features, diff = diff)\n    df_proc['y'] = df['Target']\n    #df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    # TODO: Try different models here!\n    #model = LGBMRegressor(random_state=1111, n_estimators=1200)\n    #model.fit(X, y)\n    #return X, y, model\n    \n    model = XGB_pipeline()\n    \n    model.fit(X, y)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-15T10:29:26.826630Z","iopub.execute_input":"2021-11-15T10:29:26.827468Z","iopub.status.idle":"2021-11-15T10:29:26.837784Z","shell.execute_reply.started":"2021-11-15T10:29:26.827349Z","shell.execute_reply":"2021-11-15T10:29:26.836605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loop over all assets","metadata":{}},{"cell_type":"code","source":"# training models, change params with the 'params' variable.\nparams = {'lag':0,'less_features': True, 'shuffle':True, 'diff':False}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    try:\n        model = get_Xy_and_model_for_asset(df_train, asset_id, **params)    \n        models[asset_id] = model\n    except KeyboardInterrupt:\n        break\n    except: \n        traceback.print_exc()\n        models[asset_id] = None ","metadata":{"execution":{"iopub.status.busy":"2021-11-15T10:29:26.842265Z","iopub.execute_input":"2021-11-15T10:29:26.843115Z","iopub.status.idle":"2021-11-15T10:31:27.008632Z","shell.execute_reply.started":"2021-11-15T10:29:26.843064Z","shell.execute_reply":"2021-11-15T10:31:27.007595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing","metadata":{}},{"cell_type":"code","source":"# testing the model, prints out correlation every 1000 iterations.\n\nparams['shuffle'] = False\nfrom tqdm import tqdm\n\ny_test = df[df['timestamp'] > 1623542400].Target\nX_test = df[df['timestamp'] > 1623542400].iloc[:-1]\n\ndict_tmp = {}\ncorrelations = []\npred = []\ncounter = 1\nfor j , row in tqdm(X_test.iterrows()):\n    counter += 1\n    if models[row['Asset_ID']] is not None:\n        try:\n            model = models[row['Asset_ID']]\n            # putting test data in the correct format, not optimized\n            if row['Asset_ID'] not in dict_tmp:\n                dict_tmp[row['Asset_ID']] = pd.DataFrame()\n            dict_tmp[row['Asset_ID']] = dict_tmp[row['Asset_ID']].append(row)\n            if len(dict_tmp[row['Asset_ID']]) > 50:\n                dict_tmp[row['Asset_ID']] = dict_tmp[row['Asset_ID']].tail(50)\n            x_test = get_features(dict_tmp[row['Asset_ID']],**params).tail(1)\n            if len(x_test) < 1 :\n                pred.append(0)\n                continue\n#             y_pred = model.predict(x_test)[0]\n            y_pred = model.predict(x_test)[0]\n            pred.append(y_pred)\n        except KeyboardInterrupt:\n            break\n        except:\n            pred.append(0)\n            traceback.print_exc()\n    else: \n        print('no model found')\n        pred.append(0)\n    if counter%1000 == 0:\n        correlations.append(pearsonr(y_test[:len(pred)], pred)[0])\n        print(f'Correlation: {pearsonr(y_test[:len(pred)], pred)[0]}, p-value = {pearsonr(y_test[:len(pred)], pred)[1]}')\n    if counter%20000 == 0:\n        break\n","metadata":{"execution":{"iopub.status.busy":"2021-11-15T10:31:27.018227Z","iopub.execute_input":"2021-11-15T10:31:27.019375Z","iopub.status.idle":"2021-11-15T10:34:42.866598Z","shell.execute_reply.started":"2021-11-15T10:31:27.019329Z","shell.execute_reply":"2021-11-15T10:34:42.865308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_test)\npred[:10]","metadata":{"execution":{"iopub.status.busy":"2021-11-15T10:34:42.868560Z","iopub.execute_input":"2021-11-15T10:34:42.869142Z","iopub.status.idle":"2021-11-15T10:34:42.886132Z","shell.execute_reply.started":"2021-11-15T10:34:42.869091Z","shell.execute_reply":"2021-11-15T10:34:42.884544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This assumes that we have minute-by-minute data for each asset","metadata":{"execution":{"iopub.status.busy":"2021-11-15T10:34:42.887670Z","iopub.execute_input":"2021-11-15T10:34:42.888333Z","iopub.status.idle":"2021-11-15T10:34:42.894732Z","shell.execute_reply.started":"2021-11-15T10:34:42.888284Z","shell.execute_reply":"2021-11-15T10:34:42.893214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## these numbers are all an artifact of that fact that predictions are all very close to 0.\n\n# fixed lag, new XBGoost params:\n    #params = {'lag':5,'less_features': False, 'shuffle':True} test_size = 10000, 0.0051152419695546245 (peak = 0.068)\n    #params = {'lag':0,'less_features': True, 'shuffle':True, 'diff': False} test_size = 10000, 0.0051361107196150665 (peak = 0.06)\n    #params = {'lag':0,'less_features': True, 'shuffle':True } test_size = 10000, 0.004907722808215038 (peak = 0.06)\n    #params = {'lag':0,'less_features': True, 'shuffle':False } test_size = 10000, 0.004716585631229205 (peak = 0.06)\n\n# previous code didn't run correctly, get_features(training) returns all nans except for the two shadow features.\n\n# with scaling:\n    # params = {'lag':10,'less_features': True, 'shuffle':False }, test_size = 10000, 0.03236088315425213\n    \n#with diff, no scaling & model = xgb.XGBRegressor(\n#         n_estimators=500 (i think),\n#         missing=-999,\n#         random_state=1111,\n#         tree_method='gpu_hist'  \n#         ):\n    # params = {'lag':5,'less_features': True, 'shuffle':False }, test_size = 10000, <0\n    # params = {'lag':8,'less_features': True, 'shuffle':False }, test_size = 10000, <0\n    # params = {'lag':10,'less_features': True, 'shuffle':False }, test_size = 10000, 0.059430976177124534\n    # params = {'lag':10,'less_features': True, 'shuffle':True }, test_size = 10000, 0.007976245597496146\n    # params = {'lag':20,'less_features': True, 'shuffle':True }, test_size = 10000, <0\n\n    # with lagged features, lag = 3, test_size = 6000, shuffled, correlation for all assets: -0.025720808709807898\n\n#with n_estimators = 1000:\n\n    # params = {'lag':10,'less_features': True, 'shuffle':False }, test_size = 10000, 0.038023285612531356 (but high variance)\n\n# all shit\n    # with lagged features, lag = 3, test_size = 4000, correlation for all assets: -0.025720808709807898\n    # with lagged features, lag = 3, test_size = 5000, correlation for all assets: 0.010013542572384845\n    # with lagged features, lag = 3, test_size = 6000, correlation for all assets: 0.005208812545830533\n    # with lagged features, lag = 1, test_size = 6000, correlation for asset_ID = 1: 0.012588759721880852\n    # with lagged features,lag = 1, test_size = 6000, correlation for asset_ID = 0: -0.006493023259016858\n    # with lagged features, lag = 3, test_size = 6000, correlation for asset_ID = 0: 0.025999920635118752\n    # without lagged features, test_size = 6000, correlation for asset_ID = 1: <0\n    # without lagged features, test_size = 6000, correlation for asset_ID = 0: <0","metadata":{"execution":{"iopub.status.busy":"2021-11-15T10:34:42.896315Z","iopub.execute_input":"2021-11-15T10:34:42.896739Z","iopub.status.idle":"2021-11-15T10:34:42.908735Z","shell.execute_reply.started":"2021-11-15T10:34:42.896693Z","shell.execute_reply":"2021-11-15T10:34:42.906958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict & submit\n\nReferences: [Detailed API Introduction](https://www.kaggle.com/sohier/detailed-api-introduction)\n\nSomething that helped me understand this iterator was adding a pdb checkpoint inside of the for loop:\n\n```python\nimport pdb; pdb.set_trace()\n```\n\nSee [Python Debugging With Pdb](https://realpython.com/python-debugging-pdb/) if you want to use it and you don't know how to.\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T20:57:49.349459Z","iopub.status.idle":"2021-11-02T20:57:49.349757Z","shell.execute_reply.started":"2021-11-02T20:57:49.349596Z","shell.execute_reply":"2021-11-02T20:57:49.349613Z"}}},{"cell_type":"code","source":"# env = gresearch_crypto.make_env()\n# iter_test = env.iter_test()\n\n# for i, (df_test, df_pred) in enumerate(iter_test):\n#     for j , row in df_test.iterrows():\n        \n#         if models[row['Asset_ID']] is not None:\n#             try:\n#                 model = models[row['Asset_ID']]\n#                 x_test = get_features(row)\n#                 y_pred = model.predict(pd.DataFrame([x_test]))[0]\n#                 df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n#             except:\n#                 df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n#                 traceback.print_exc()\n#         else: \n#             df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n        \n#     env.predict(df_pred)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-15T10:34:42.910191Z","iopub.execute_input":"2021-11-15T10:34:42.910618Z","iopub.status.idle":"2021-11-15T10:34:43.168937Z","shell.execute_reply.started":"2021-11-15T10:34:42.910568Z","shell.execute_reply":"2021-11-15T10:34:43.165410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}