{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport dateutil\nimport os\nimport time\nimport datetime\nimport xgboost as xgb\nimport time\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:02:01.686539Z","iopub.execute_input":"2022-01-13T21:02:01.687074Z","iopub.status.idle":"2022-01-13T21:02:02.964254Z","shell.execute_reply.started":"2022-01-13T21:02:01.686970Z","shell.execute_reply":"2022-01-13T21:02:02.963421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 0. Functions","metadata":{}},{"cell_type":"code","source":"def todatetime(t):\n    \"\"\"\n    Convert Unix timestamp to datetime.\n    \"\"\"\n    return datetime.datetime.fromtimestamp(t)\n\ndef reindex_by_date(df):\n    \"\"\"\n    Reindex Time Series. \n    \"\"\"\n    dates = range(df.index[0], df.index[-1]+60,60)\n    return df.reindex(dates, method = 'pad')\n\ndef apply_ta_strategy(data, ta_strategy ):\n    \"\"\" \n    Apply Technical Indicators Strategy\n    \"\"\"\n\n    dfs = []\n \n    for asset in data['Asset_ID'].unique():\n        \n        #print(\" Calculating TA for asset \", asset)\n        _df_temp = data[data['Asset_ID'] == asset].copy(deep = True)\n        _df_temp.ta.strategy(ta_strategy)\n        dfs.append(_df_temp)\n\n    return pd.concat(dfs, ignore_index= True)\n\ndef build_fourier_time_features(df : pd.DataFrame, \n                                time_levels: list, \n                                max_levels: list, \n                                drop_columns = False):\n    \"\"\"\n    Transform time featuers in Fourier transformation\n    \"\"\"\n\n    for time_level, max_level in zip(time_levels, max_levels):\n       \n        print(\"Generating Transformation for\", time_level)\n\n        if time_level == 'month':\n            df.loc[:,f\"{time_level}_sin\"] = (df['timestamp']\n                                    .dt.month\n                                    ).apply(\n                                    lambda x: np.sin( \n                                        2 * np.pi + x/max_level))\n        if time_level == 'day':\n            df.loc[:,f\"{time_level}_sin\"] = (df['timestamp']\n                                    .dt.day\n                                    ).apply(\n                                    lambda x: np.sin( \n                                        2 * np.pi + x/max_level))\n        if time_level == 'hour': \n            df.loc[:,f\"{time_level}_sin\"] = (df['timestamp']\n                                    .dt.hour\n                                    ).apply(\n                                    lambda x: np.sin( \n                                        2 * np.pi + x/max_level))\n        if time_level == 'minute':\n            df.loc[:,f\"{time_level}_sin\"] = (df['timestamp']\n                                    .dt.minute\n                                    ).apply(\n                                    lambda x: np.sin( \n                                        2 * np.pi + x/max_level))\n    \ndef lag_features(data, n_lags, ref_variable):\n    \"\"\"\n    Generate lag Features\n    \"\"\"\n    \n    lags_features = []\n\n    if n_lags is not None:\n        \n        for lag in n_lags:\n\n            columns_name = f'{ref_variable}_lag_{lag}'\n\n            data.loc[:,columns_name] = (data\n                                        .groupby(['Asset_ID'])\n                                        [ref_variable]\n                                        .transform(\n                                        lambda x : x.shift(lag)))\n\n            lags_features.append(columns_name) \n                \n    return data, lags_features\n\n\ndef calculate_returns(data, variable, lags, binary_lags, outlier_cutoff):\n    \"\"\"\n    Calculate Returns on a variable.\n    \"\"\"\n\n    returns = []\n\n    data.set_index(['timestamp', 'Asset_ID'], inplace = True)\n\n    for lag in lags:\n        if binary_lags:\n            _return = returns.append(data[variable]\n                        .sort_index() # Sort by Date\n                        .pct_change(lag) # Calculate percentage change of the respective lag value\n                        .pipe(lambda x: x.clip(lower=x.quantile(outlier_cutoff),\n                                                upper=x.quantile(1-outlier_cutoff))) # Cutoff outliers\n                        .add(1) # add 1 to the returns\n                        .pow(1/lag) # apply n root for n = lag\n                        .sub(1) #substract 1\n                        .apply(lambda x: 1 if x > 0 else 0)\n                        .to_frame(f'{variable}_return_{lag}m')\n                        \n                        )\n\n        else:\n            _return = returns.append(data[variable]\n                    .sort_index() # Sort by Date\n                    .pct_change(lag) # Calculate percentage change of the respective lag value\n                    .pipe(lambda x: x.clip(lower=x.quantile(outlier_cutoff),\n                                            upper=x.quantile(1-outlier_cutoff))) # Cutoff outliers\n                    .add(1) # add 1 to the returns\n                    .pow(1/lag) # apply n root for n = lag\n                    .sub(1) #substract 1\n                    .to_frame(f'{variable}_return_{lag}m')\n                    \n                )\n\n    returns.append(_return)\n        \n    returns = pd.concat(returns, axis = 1)\n    #returns.info(null_counts=True)\n\n    #data = data.set_index(['timestamp']).join(returns).dropna()\n    data = data.join(returns).dropna()\n    data.reset_index(inplace = True)\n\n    return data\n\ndef rsi(x: pd.Series, periods = 13):\n    \n    # differentiation 1\n    delta = x.diff(1)\n    # Gains\n    up = delta.clip(lower=0)\n    # Losses\n    down = delta.clip(upper=0).abs()\n    \n    # Exponential Weigted Average\n    # alpha = 1/(1+com)\n    ema_up = up.ewm(com=periods -1, min_periods = periods, adjust=False).mean()\n    ema_down = down.ewm(com=periods -1, min_periods = periods, adjust=False).mean()\n    \n    rs = abs(ema_up/ema_down)\n\n\n    return 100 - 100/(1+rs)\n\ndef calculate_rolling_features(df: pd.DataFrame, \n                            target_variable: str, \n                            windows: list, \n                            alpha: int = 0.8,\n                            indicator = [], ):\n    \"\"\"Function to calculate rolling feature for a target variable in \n    a data frame. \n    \n    Args:\n        df (pandas.Dataframe): df input data frame\n        target_variable (str): df containing the training samples\n        windows (list): windows \n        alpha (float): alpha decay parameter for WMA features\n        indicator (list): 'sma', 'std', 'bbands' and 'cv'\n    \n    Returns:\n        pandas.Dataframe: original data frame containing the calculated features\n    \"\"\"\n\n    if 'sma' in indicator:\n        for w in windows:\n            df[f'{target_variable}_sma_{w}'] = (df.groupby(['Asset_ID'])\n                                                        [target_variable]\n                                                        .transform(lambda x: \n                                                        x.rolling(window = w)\n                                                        .mean()))\n            \n    if 'std' in indicator:\n        for w in windows:\n            df[f'{target_variable}_std_{w}'] = (df.groupby(['Asset_ID'])\n                                                        [target_variable]\n                                                        .transform(lambda x: \n                                                        x.rolling(window = w)\n                                                        .std()))\n\n    \n    if 'wma_mean' in indicator:\n        df[f'{target_variable}_wma_mean'] = (df.groupby(['Asset_ID'])\n                                                    [target_variable]\n                                                    .transform(lambda x: \n                                                    x.ewm(alpha = alpha)\n                                                    .mean()))\n    \n    if 'wma_std' in indicator:\n        df[f'{target_variable}_wma_std'] = (df.groupby(['Asset_ID'])\n                                                    [target_variable]\n                                                    .transform(lambda x: \n                                                    x.ewm(alpha = alpha)\n                                                    .std()))\n\n\n\n    if 'cv' in indicator:\n        # TODO Reduce code\n        if (not ('sma' in indicator)) or (not ('std' in indicator)):\n            for w in windows:\n                df[f'{target_variable}_sma_{w}'] = (df.groupby(['Asset_ID'])\n                                                            [target_variable]\n                                                            .transform(lambda x: \n                                                            x.rolling(window = w)\n                                                            .mean()))\n\n                df[f'{target_variable}_std_{w}'] = (df.groupby(['Asset_ID'])\n                                                            [target_variable]\n                                                            .transform(lambda x: \n                                                            x.rolling(window = w)\n                                                            .std()))\n        for w in windows: \n            df[f'{target_variable}_cv_{w}'] = (df[f'{target_variable}_std_{w}'] \n                                                    / df[f'{target_variable}_sma_{w}'] ) \n       \n    \n    if 'bbands' in indicator:\n         \n        # Calculate Typical Price\n        # Ref: https://www.investopedia.com/terms/b/bollingerbands.asp\n        target_variable = 'Typical_Price'\n        df[target_variable] = ((df['High'] + df['Low'] + df['Close']) / 3)\n            \n        for w in windows:\n            df[f'{target_variable}_sma_{w}'] = (df.groupby(['Asset_ID'])\n                                                        [target_variable]\n                                                        .transform(lambda x: \n                                                        x.rolling(window = w)\n                                                        .mean()))\n\n            df[f'{target_variable}_std_{w}'] = (df.groupby(['Asset_ID'])\n                                                        [target_variable]\n                                                        .transform(lambda x: \n                                                        x.rolling(window = w)\n                                                        .std()))\n\n\n            df[f'bblow_{w}'] = (df[f'{target_variable}_sma_{w}'] - 2 * \n                                                  df[f'{target_variable}_std_{w}'])\n\n            df[f'bbhigh_{w}'] = (df[f'{target_variable}_sma_{w}'] + 2 * \n                                                    df[f'{target_variable}_std_{w}'])\n\n\n    return df \n\ndef momemtum_features(data, variable_returns, return_lags):\n    \"\"\"\n    Generate momentum features\n    \"\"\"\n    \n    for lag in return_lags:\n        if lag > return_lags[0]:\n            data['momentum_{}_{}'.format( return_lags[0], lag)] = (data[f'{variable_returns}_return_{lag}m']\n                                                            .sub(data['{}_return_{}m'\n                                                            .format(variable_returns, return_lags[0])]))\n        if lag > return_lags[1]:\n            data['momentum_{}_{}'.format( return_lags[1], lag)] = (data[f'{variable_returns}_return_{lag}m']\n                                                            .sub(data['{}_return_{}m'\n                                                            .format(variable_returns, return_lags[1])]))\n            \n    return data\n\nclass MultipleTimeSeriesCV:\n    \"\"\"\n    Generates tuples of train_idx, test_idx pairs.\n    Assumes the MultiIndex contains levels 'symbol' and 'date'.\n    Purges overlapping outcomes.\n    \"\"\"\n    \n    def __init__(self,\n                n_splits=3,\n                train_period_length=126,\n                test_period_length=21,\n                lookahead=None,\n                date_idx = 'date',\n                shuffle=False):\n        self.n_splits = n_splits\n        self.lookahead = lookahead\n        self.test_length = test_period_length\n        self.train_length = train_period_length\n        self.shuffle = shuffle\n        self.date_idx = date_idx\n    \n    def split(self, X, y=None, groups=None):\n        unique_dates = X.index.get_level_values(self.date_idx).unique()\n        days=sorted(unique_dates, reverse=True)\n        split_idx = []\n        for i in range(self.n_splits):\n            test_end_idx = i * self.test_length\n            test_start_idx = test_end_idx + self.test_length\n            train_end_idx = test_start_idx + self.lookahead -1\n            train_start_idx = train_end_idx + self.train_length + self.lookahead -1\n            split_idx.append([train_start_idx, train_end_idx,\n                             test_start_idx, test_end_idx])\n        \n        dates = X.reset_index()[[self.date_idx]]\n        for train_start, train_end, test_start, test_end in split_idx:\n            train_idx = dates[(dates[self.date_idx] > days[train_start])\n                             & (dates[self.date_idx] <= days[train_end])].index\n            test_idx = dates[(dates[self.date_idx]> days[test_start])\n                            & (dates[self.date_idx] <= days[test_end])].index\n            \n            if self.shuffle:\n                np.random.shuffle(list(train_idx))\n            yield train_idx.to_numpy(), test_idx.to_numpy()\n    \n    def get_n_splits(self, X, y, groups=None):\n        return self.n_splits\n    \ndef feature_selection(df, train_range, test_range, X_columns, forecast_variable):\n    \"\"\"\n    Feature selection.\n    \"\"\"\n\n    df.reset_index(inplace = True)\n    \n    #Filter selected asset\n    train = (df[(df['timestamp'] >= train_range[0]) & \n                    (df['timestamp'] < train_range[1])]\n                    .copy(deep = True)\n                    .set_index(['timestamp', 'Asset_ID'])\n                    )\n\n    test = (df[(df['timestamp'] >= test_range[0]) \n                & (df['timestamp'] <= test_range[1])]\n                .copy(deep = True)\n                .set_index(['timestamp', 'Asset_ID']))\n\n    X_train = train.filter(X_columns)\n    X_test = test.filter(X_columns)\n\n\n    Y_train = train.filter([forecast_variable])\n    Y_test = test.filter([forecast_variable])\n\n    return X_train, X_test, Y_train, Y_test\n\ndef convert_to_float32(df):\n    \n    for col in df.columns:\n        df.loc[:,col] = df[col].astype('float32')\n        \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:02:02.966042Z","iopub.execute_input":"2022-01-13T21:02:02.966458Z","iopub.status.idle":"2022-01-13T21:02:03.030541Z","shell.execute_reply.started":"2022-01-13T21:02:02.966409Z","shell.execute_reply":"2022-01-13T21:02:03.029294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preprocessing","metadata":{}},{"cell_type":"code","source":"data_path = \"/kaggle/input/g-research-crypto-forecasting/\"\n\ndf_weights = pd.read_csv(os.path.join(data_path, 'asset_details.csv'), usecols=[0,1]).set_index('Asset_ID')\ndf_asset_details = pd.read_csv(os.path.join(data_path,\"asset_details.csv\"))\ndf_supp_train = pd.read_csv(os.path.join(data_path, \"supplemental_train.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:02:03.032213Z","iopub.execute_input":"2022-01-13T21:02:03.032872Z","iopub.status.idle":"2022-01-13T21:02:09.063268Z","shell.execute_reply.started":"2022-01-13T21:02:03.032830Z","shell.execute_reply":"2022-01-13T21:02:09.062155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv(os.path.join(data_path, \"train.csv\"))\ndf_test = pd.read_csv(os.path.join(data_path,\"example_test.csv\"))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-13T21:02:09.065608Z","iopub.execute_input":"2022-01-13T21:02:09.065904Z","iopub.status.idle":"2022-01-13T21:03:17.075413Z","shell.execute_reply.started":"2022-01-13T21:02:09.065871Z","shell.execute_reply":"2022-01-13T21:03:17.073781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The test dataset validates models on Jun 13, 2021 data. Therefore models must be trained with data before that date. ","metadata":{}},{"cell_type":"code","source":"# Test data dates\nprint(pd.Series(df_test['timestamp'].unique()).apply(todatetime))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:03:17.078446Z","iopub.execute_input":"2022-01-13T21:03:17.079422Z","iopub.status.idle":"2022-01-13T21:03:17.099517Z","shell.execute_reply.started":"2022-01-13T21:03:17.079364Z","shell.execute_reply":"2022-01-13T21:03:17.098323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Complete missing timestamps\ndata = (data.set_index('timestamp')\n            .groupby(['Asset_ID'], \n            group_keys= False)\n            .apply(reindex_by_date)\n            .reset_index())\n\n# Convert Unix timestamp to datetime \ndata['timestamp'] = data['timestamp'].apply(todatetime)\n\n# Reduced training data on filter\n## latest_date = data['timestamp'].max()\nhistorical_days = 5 # Historical days for model training\n#latest_date = datetime.datetime(2021, 6,13) \n#date_filter = latest_date - dateutil.relativedelta.relativedelta(days = historical_days)\n\n# Test Range\nlatest_date = datetime.datetime(2021, 11,1) \ndate_filter = datetime.datetime(2021, 6,13) - dateutil.relativedelta.relativedelta(days = historical_days)\n\n# Set date filter for historical data\ndata = data[(data['timestamp'] >= date_filter) & \n           (data['timestamp'] < latest_date)].copy(deep = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:03:17.101414Z","iopub.execute_input":"2022-01-13T21:03:17.101683Z","iopub.status.idle":"2022-01-13T21:04:04.314890Z","shell.execute_reply.started":"2022-01-13T21:03:17.101646Z","shell.execute_reply":"2022-01-13T21:04:04.313910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['timestamp'].min()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:04.316379Z","iopub.execute_input":"2022-01-13T21:04:04.317402Z","iopub.status.idle":"2022-01-13T21:04:04.336642Z","shell.execute_reply.started":"2022-01-13T21:04:04.317358Z","shell.execute_reply":"2022-01-13T21:04:04.335808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.groupby('Asset_ID').agg({'Close':len})","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:04.337882Z","iopub.execute_input":"2022-01-13T21:04:04.338190Z","iopub.status.idle":"2022-01-13T21:04:04.428480Z","shell.execute_reply.started":"2022-01-13T21:04:04.338157Z","shell.execute_reply":"2022-01-13T21:04:04.427376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Historical Data\ndf_hist = data.copy(deep = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:04.430770Z","iopub.execute_input":"2022-01-13T21:04:04.431029Z","iopub.status.idle":"2022-01-13T21:04:04.495843Z","shell.execute_reply.started":"2022-01-13T21:04:04.430999Z","shell.execute_reply":"2022-01-13T21:04:04.494745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Feature Engineering","metadata":{}},{"cell_type":"code","source":"def get_technical_indicators(data, technical_indicators):\n    \n    # Calculate Technical Indicators\n    # Calculate rsi\n    for ti in technical_indicators.keys():\n        \n        if ti == 'rsi':\n            \n            target_variable = technical_indicators[ti]['variable']\n            periods = technical_indicators[ti]['periods']\n            \n            data[f'rsi_{periods}'] = (data.groupby(['Asset_ID'])\n                                    [target_variable]\n                                    .transform(lambda x: rsi(x, periods = periods)))\n            \n    return data\n\ndef get_features(data, technical_indicators, n_lags, \n                ref_variable_lags, outlier_cutoff, \n                return_lags, binary_lags, variable_returns, \n                rolling_features_params):\n    \"\"\"\n    Get Features\n    \"\"\"\n    \n    # Time Features\n    data['minute'] = data['timestamp'].dt.minute\n    data['hour'] = data['timestamp'].dt.hour\n    \n    # Get Technical Indicators\n    data = get_technical_indicators(data, technical_indicators)\n    \n    # Calculate Lag features\n    data, lags_features = lag_features(data, n_lags, ref_variable_lags)\n\n    # Calculate Returns Features\n    data = calculate_returns(data, variable_returns, return_lags, binary_lags, outlier_cutoff)\n\n    # Calculate Rolling Features\n    for target_variable in rolling_features_params.keys():\n\n        #print(\"Calculating rolling Features for \", target_variable)\n\n        windows = rolling_features_params[target_variable]['windows']\n        alpha = rolling_features_params[target_variable]['alpha']\n        indicator = rolling_features_params[target_variable]['indicator']\n\n        data = calculate_rolling_features(df = data, \n                                target_variable = target_variable, \n                                windows = windows,\n                                alpha = alpha,\n                                indicator = indicator)\n\n    # Calculate momentum features\n    data = momemtum_features(data, variable_returns, return_lags)\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:04.498906Z","iopub.execute_input":"2022-01-13T21:04:04.499159Z","iopub.status.idle":"2022-01-13T21:04:04.511354Z","shell.execute_reply.started":"2022-01-13T21:04:04.499129Z","shell.execute_reply":"2022-01-13T21:04:04.510697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"technical_indicators = {'rsi': {'periods': 15, 'variable': 'Close'}}\n\n# Parameters for lag features\nn_lags = [5, 15, 30]\nref_variable_lags = 'Close'\n\n# Paramaters for return Features\noutlier_cutoff = 0.01\nreturn_lags = [1, 15, 30, 60] \nbinary_lags = False\nvariable_returns = 'Close'\n\n# Parameters for rolling features\nrolling_features_params = {\n    'Close':{\n            'windows':[15],\n            'alpha' : 0.8,\n            'indicator' : ['wma_mean', 'wma_std','bbands']}, \n\n    'Volume':{\n            'windows': None,\n            'alpha' : 0.8,\n            'indicator' : ['wma_mean', 'wma_std']}, \n\n    'VWAP': {\n            'windows': None,\n            'alpha' : 0.8,\n            'indicator' : ['wma_mean', 'wma_std']}}","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:04.512501Z","iopub.execute_input":"2022-01-13T21:04:04.513134Z","iopub.status.idle":"2022-01-13T21:04:04.534872Z","shell.execute_reply.started":"2022-01-13T21:04:04.513097Z","shell.execute_reply":"2022-01-13T21:04:04.533867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = get_features(data, technical_indicators, n_lags, \n                ref_variable_lags, outlier_cutoff, \n                return_lags, binary_lags, variable_returns, \n                rolling_features_params)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:04.536325Z","iopub.execute_input":"2022-01-13T21:04:04.537468Z","iopub.status.idle":"2022-01-13T21:04:40.142814Z","shell.execute_reply.started":"2022-01-13T21:04:04.537427Z","shell.execute_reply":"2022-01-13T21:04:40.141870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:40.144243Z","iopub.execute_input":"2022-01-13T21:04:40.144542Z","iopub.status.idle":"2022-01-13T21:04:40.151832Z","shell.execute_reply.started":"2022-01-13T21:04:40.144504Z","shell.execute_reply":"2022-01-13T21:04:40.150838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Model Training ","metadata":{}},{"cell_type":"code","source":"forecast_variable = 'Target'\n\ntrain_columns = ['Count', 'Open', 'High', 'Low',\n                   'minute', 'hour', 'rsi_15', 'Close_lag_5',\n                   'Close_lag_15', 'Close_lag_30', 'Close_return_1m', 'Close_return_15m',\n                   'Close_return_30m', 'Close_return_60m', 'Close_wma_mean',\n                   'Close_wma_std', 'Typical_Price', 'Typical_Price_sma_15',\n                   'Typical_Price_std_15', 'bblow_15', 'bbhigh_15', 'Volume_wma_mean',\n                   'Volume_wma_std', 'VWAP_wma_mean', 'VWAP_wma_std', 'momentum_1_15',\n                   'momentum_1_30', 'momentum_15_30', 'momentum_1_60', 'momentum_15_60']\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:40.153130Z","iopub.execute_input":"2022-01-13T21:04:40.153501Z","iopub.status.idle":"2022-01-13T21:04:40.165778Z","shell.execute_reply.started":"2022-01-13T21:04:40.153457Z","shell.execute_reply":"2022-01-13T21:04:40.164667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assets = data['Asset_ID'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:40.167300Z","iopub.execute_input":"2022-01-13T21:04:40.167741Z","iopub.status.idle":"2022-01-13T21:04:40.192256Z","shell.execute_reply.started":"2022-01-13T21:04:40.167686Z","shell.execute_reply":"2022-01-13T21:04:40.191583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_date = datetime.datetime(2021, 6,13)\n\n# Historical days for model training\ntraining_days = 5\ntrain_range = [test_date - dateutil.relativedelta.relativedelta(days = historical_days), test_date]\n\n# Historical days for model testing\nhours_test_range = 1\ntest_range = [ test_date, test_date + dateutil.relativedelta.relativedelta(hours = hours_test_range)]\n ","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:40.193155Z","iopub.execute_input":"2022-01-13T21:04:40.193392Z","iopub.status.idle":"2022-01-13T21:04:40.199396Z","shell.execute_reply.started":"2022-01-13T21:04:40.193357Z","shell.execute_reply":"2022-01-13T21:04:40.198573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load models\ntrain_models = False\nmodels = {}\nif not train_models:\n    \n    models = {}\n    input_folder_models = '/kaggle/input/cryptomodels'\n\n    for asset_id in assets:\n\n        model_loc = os.path.join(input_folder_models, f'model_asset_{asset_id}.pkl' )\n        models[asset_id]= pickle.load(open(model_loc, 'rb'))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:04:40.200744Z","iopub.execute_input":"2022-01-13T21:04:40.201002Z","iopub.status.idle":"2022-01-13T21:05:27.581395Z","shell.execute_reply.started":"2022-01-13T21:04:40.200974Z","shell.execute_reply":"2022-01-13T21:05:27.580511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = []\npred = []\nstart = time.time()\n\nfor asset_id in assets:\n\n    df_train = data.copy(deep = True)\n    df_train = df_train[df_train['Asset_ID'] == asset_id].copy(deep = True)\n    df_train.set_index(['timestamp', 'Asset_ID'], inplace = True)\n    df_train.dropna(inplace=True)\n\n    \n    # Make feature Selection\n    X_train, X_test, Y_train, Y_test  = feature_selection(df_train, \n                                        train_range, \n                                        test_range, \n                                        train_columns, \n                                        forecast_variable, \n                                        )\n    \n    model = None\n    \n    if train_models:\n        \n        print('Training model for asset', asset_id)\n        \n        learning_rate = model_params[asset_id]['learning_rate']\n        max_depth = model_params[asset_id]['max_depth']\n        n_estimators = model_params[asset_id]['n_estimators']\n\n        \n        xgb_model = xgb.XGBRegressor(learning_rate = learning_rate, \n                                max_depth = max_depth, \n                                n_estimators = n_estimators)\n\n\n        # executes bayesian optimization\n        model = xgb_model.fit(X_train, Y_train)\n\n        models[asset_id] = model\n    else:\n        model = models[asset_id]\n\n    # Predict training set\n    Y_pred = model.predict(X_test)\n\n    # Set inde for Y_pred\n    Y_pred = pd.DataFrame({'Y_pred': Y_pred,} ,index = Y_test.index)\n\n    # Add Test and Pred Y\n    test.append(Y_test)\n    pred.append(Y_pred)\n    \nruntime = np.round((time.time() - start)/60, 3)\n\nprint(\"Runtime is:\",  runtime)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:07:20.615040Z","iopub.execute_input":"2022-01-13T21:07:20.615352Z","iopub.status.idle":"2022-01-13T21:07:25.660597Z","shell.execute_reply.started":"2022-01-13T21:07:20.615318Z","shell.execute_reply":"2022-01-13T21:07:25.659477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check Feature Importance for Bitcoin Model\nasset_id = 1\nImportance = models[asset_id].best_estimator_.feature_importances_\ndf_importance = pd.DataFrame({'Variable': train_columns, 'Importance':Importance})\ndf_importance.sort_values(by = ['Importance'], ascending = False, inplace = True)\ndf_importance.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:07:25.662208Z","iopub.execute_input":"2022-01-13T21:07:25.662438Z","iopub.status.idle":"2022-01-13T21:07:25.696837Z","shell.execute_reply.started":"2022-01-13T21:07:25.662410Z","shell.execute_reply":"2022-01-13T21:07:25.695735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# join all predictions and all test values\ndf_test = pd.concat(test)\ndf_pred = pd.concat(pred)\n\ndf_master = df_test.join(df_pred).reset_index()\n\ncheck_validation_test_data = True\nif check_validation_test_data:\n    df_master = (df_master[df_master['timestamp'].isin([datetime.datetime(2021,6,13,0,0,0),\n                    datetime.datetime(2021,6,13,0,1,0) ,\n                    datetime.datetime(2021,6,13,0,2,0)])].copy(deep = True))\n\n\n# Calculate pearson r per cryptocurrency\ndf_performance = df_master.groupby(['Asset_ID']).apply(lambda x: np.corrcoef(x['Target'].values, x['Y_pred'].values)[0,1]).rename(\"Pearson\").to_frame()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:08:02.036155Z","iopub.execute_input":"2022-01-13T21:08:02.036488Z","iopub.status.idle":"2022-01-13T21:08:02.079403Z","shell.execute_reply.started":"2022-01-13T21:08:02.036457Z","shell.execute_reply":"2022-01-13T21:08:02.078533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_performance","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:08:02.812250Z","iopub.execute_input":"2022-01-13T21:08:02.812914Z","iopub.status.idle":"2022-01-13T21:08:02.828373Z","shell.execute_reply.started":"2022-01-13T21:08:02.812774Z","shell.execute_reply":"2022-01-13T21:08:02.826983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_performance = df_performance.join(df_weights)\nweighted_performance = (df_performance['Pearson'] * df_performance['Weight'] ).sum() / df_performance['Weight'].sum()\n\nprint(\" Weighted performance is :\", np.round(weighted_performance, 4 ))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:08:04.566241Z","iopub.execute_input":"2022-01-13T21:08:04.566589Z","iopub.status.idle":"2022-01-13T21:08:04.576072Z","shell.execute_reply.started":"2022-01-13T21:08:04.566552Z","shell.execute_reply":"2022-01-13T21:08:04.574728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Model Testing","metadata":{}},{"cell_type":"code","source":"# Append new candle (minute) data to historical\ndef append_new_data(_df_hist, test_df, max_hist_hold, minute_test):\n    \"\"\"\n    Append new minute data to historical data\n    \"\"\"\n    \n    # Make sure to filter out future data\n    _df_hist = _df_hist[_df_hist['timestamp'] < minute_test].copy(deep = True)\n\n    _df_hist = _df_hist.append(test_df).copy(deep = True)\n    \n    _df_hist.sort_values(by = ['timestamp', 'Asset_ID'], inplace = True)\n    \n    _df_hist = (_df_hist[\n                _df_hist['timestamp'] >= max_hist_hold]\n                .copy(deep = True))\n\n    _df_hist.drop(columns = ['row_id'], inplace = True)\n    \n    return _df_hist","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:05:37.470210Z","iopub.execute_input":"2022-01-13T21:05:37.471082Z","iopub.status.idle":"2022-01-13T21:05:37.478946Z","shell.execute_reply.started":"2022-01-13T21:05:37.471040Z","shell.execute_reply":"2022-01-13T21:05:37.477935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:05:37.480071Z","iopub.execute_input":"2022-01-13T21:05:37.480306Z","iopub.status.idle":"2022-01-13T21:05:37.517368Z","shell.execute_reply.started":"2022-01-13T21:05:37.480278Z","shell.execute_reply":"2022-01-13T21:05:37.516694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\n\nenv = gresearch_crypto.make_env()   # initialize the environment\n\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\n\n# df holding historical crypto data\n_df_hist = df_hist.drop(columns = ['Target']).copy(deep = True)\n\n# max. historical hours to hold for _df_hist\nhistorical_minutes = 70\n\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    asset_id_row_map = dict(zip(test_df['Asset_ID'].values, test_df['row_id'].values))\n\n    # Transform new candle (minute) data to expected format\n    test_df.loc[:,'timestamp'] = test_df['timestamp'].apply(todatetime)\n\n    # Get Test Minute \n    minute_test = pd.to_datetime(test_df['timestamp'].values[0])\n\n    # Set max of historical data to hold\n    # to generate rolling features\n    max_hist_hold = minute_test - dateutil.relativedelta.relativedelta(minutes = historical_minutes)\n\n    # Append new test minute data to historical df\n    #_df_hist = append_new_data(_df_hist, test_df, max_hist_hold, minute_test)\n\n    # Generate Features\n    #X_feat = get_features(_df_hist.copy(deep=True), technical_indicators, n_lags, \n    #                    ref_variable_lags, outlier_cutoff, \n     #                   return_lags, binary_lags, variable_returns, \n     #                   rolling_features_params)\n\n    # Filter for features generated only for test minute\n    #X_feat = (X_feat[X_feat['timestamp'] == minute_test]\n    #        .filter(['timestamp', 'Asset_ID'] + train_columns )\n    #        .copy(deep = True))\n    \n    # Test with already genertated features\n    X_feat = (data[data['timestamp'] == minute_test]\n            .filter(['timestamp', 'Asset_ID'] + train_columns )\n            .copy(deep = True))\n    \n    for asset_id in assets:\n\n        asset_feat = X_feat[X_feat['Asset_ID'] == asset_id].copy(deep = True)\n        asset_feat.set_index(['timestamp', 'Asset_ID'], inplace = True)\n        \n        # Use last fitted modell on trainig data < 2021.06.13\n        try:\n            asset_model = models[asset_id]\n            pred = asset_model.predict(asset_feat)[0]\n            #print(\"Predicting for minute\", minute_test, \"predictig for asset\", asset_id, \":\", pred)\n        except:\n            print(\"Model could not predict\")\n            pred = 0\n            \n        sample_prediction_df.loc[:,'Target'] = np.where( sample_prediction_df['row_id'] == asset_id_row_map.get(asset_id), \n                                                  pred, \n                                                  sample_prediction_df['Target'])\n\n    # add prediction\n    env.predict(sample_prediction_df) \n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T21:05:37.518789Z","iopub.execute_input":"2022-01-13T21:05:37.519193Z","iopub.status.idle":"2022-01-13T21:05:38.625216Z","shell.execute_reply.started":"2022-01-13T21:05:37.519161Z","shell.execute_reply":"2022-01-13T21:05:38.624264Z"},"trusted":true},"execution_count":null,"outputs":[]}]}