{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport datatable as dt\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\npd.set_option('display.max_columns', None)\nimport matplotlib.pyplot as plt # for visualization \nimport seaborn as sns\nimport plotly.graph_objects as go # can resize graphs\nimport plotly.express as px # for graphing volumes traded\nimport gc # to collect garbage\nimport traceback\nimport gresearch_crypto # required to use their API\n\nimport pandas_datareader as web # Getting web data\n\nimport datetime\nimport time\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, OrdinalEncoder, RobustScaler\nfrom scipy.stats import pearsonr\nfrom sklearn.metrics import r2_score\n\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom catboost import CatBoostRegressor, Pool\nfrom lightgbm import LGBMRegressor\nfrom numpy import nan\nimport math # to use sqrt function\nimport sys\nif not sys.warnoptions:\n    import warnings\n    warnings.simplefilter(\"ignore\")\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-20T11:24:24.234006Z","iopub.execute_input":"2022-01-20T11:24:24.234636Z","iopub.status.idle":"2022-01-20T11:24:27.906298Z","shell.execute_reply.started":"2022-01-20T11:24:24.234542Z","shell.execute_reply":"2022-01-20T11:24:27.905585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set graph style and font\n# sns.set()\n# Change the axes' title and label size to 18 & 16 by default and default figure size, and make title bold\n# Axes formatter limit will only display scientific notation if it's > 10^7 (or 10 million JPY) or < 10^-5\n# plt.rcParams.update({'axes.titleweight': 'bold','figure.figsize': (16,10),'axes.titlesize': 18,'axes.labelsize': 16, \n#                      'legend.fontsize': 12, 'xtick.labelsize': 12, 'ytick.labelsize': 12, 'font.family': 'serif', \n#                      'axes.formatter.limits':'-5, 7'}) ","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:24:27.907855Z","iopub.execute_input":"2022-01-20T11:24:27.908128Z","iopub.status.idle":"2022-01-20T11:24:27.912487Z","shell.execute_reply.started":"2022-01-20T11:24:27.908092Z","shell.execute_reply":"2022-01-20T11:24:27.911518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading in data","metadata":{}},{"cell_type":"code","source":"# For Kaggle only\npath = '/kaggle/input/g-research-crypto-forecasting/'\norig_df_train = pd.read_csv(path + 'train.csv')\nsupp_df_train = pd.read_csv(path + 'supplemental_train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:24:27.913857Z","iopub.execute_input":"2022-01-20T11:24:27.914637Z","iopub.status.idle":"2022-01-20T11:25:27.946139Z","shell.execute_reply.started":"2022-01-20T11:24:27.914600Z","shell.execute_reply":"2022-01-20T11:25:27.945218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Merging main data with supplemental data together","metadata":{}},{"cell_type":"code","source":"# The right join is outer as I'm stacking supplemental data on top of main data\nmerged_df = orig_df_train.merge(supp_df_train, how = 'outer')\n# merged_df","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:25:27.947995Z","iopub.execute_input":"2022-01-20T11:25:27.948250Z","iopub.status.idle":"2022-01-20T11:28:28.610674Z","shell.execute_reply.started":"2022-01-20T11:25:27.948220Z","shell.execute_reply":"2022-01-20T11:28:28.609939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Delete unused variables to save some memory\n# del orig_df_train, supp_df_train","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:28.611958Z","iopub.execute_input":"2022-01-20T11:28:28.612334Z","iopub.status.idle":"2022-01-20T11:28:28.615981Z","shell.execute_reply.started":"2022-01-20T11:28:28.612295Z","shell.execute_reply":"2022-01-20T11:28:28.615255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reduce size by converting to float\nmerged_df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', \\\n           'Target']] = merged_df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Target']].astype(np.float32)\nprint(merged_df.shape)\n\n# Fill in missing 'target' values with 0\n# merged_df['Target'] = merged_df['Target'].fillna(0) # Fill in missing values\n\n# Drop any rows with missing values\nmerged_df = merged_df.dropna(how = 'any')\nprint(merged_df.shape)\n\n# Convert UNIX timestamp to human time\nmerged_df['date'] = pd.to_datetime(merged_df['timestamp'],unit='s')","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:28.617125Z","iopub.execute_input":"2022-01-20T11:28:28.617744Z","iopub.status.idle":"2022-01-20T11:28:36.366800Z","shell.execute_reply.started":"2022-01-20T11:28:28.617674Z","shell.execute_reply":"2022-01-20T11:28:36.366085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fill in the target column with 0 if nan.","metadata":{}},{"cell_type":"code","source":"# Assign human time as index\nmerged_df.set_index('date', inplace = True)\n# merged_df.describe(include = 'all')","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:36.368253Z","iopub.execute_input":"2022-01-20T11:28:36.368526Z","iopub.status.idle":"2022-01-20T11:28:36.374186Z","shell.execute_reply.started":"2022-01-20T11:28:36.368490Z","shell.execute_reply":"2022-01-20T11:28:36.373343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group data by the hour to reduce the rows\nhourly_df = merged_df.groupby('Asset_ID').resample('H').mean()\nhourly_df['date'] = pd.to_datetime(hourly_df['timestamp'],unit='s')\nhourly_df = hourly_df.sort_index()\n# hourly_df.sort_values('date', axis = 0)\nhourly_df","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:36.375660Z","iopub.execute_input":"2022-01-20T11:28:36.375970Z","iopub.status.idle":"2022-01-20T11:28:41.404395Z","shell.execute_reply.started":"2022-01-20T11:28:36.375933Z","shell.execute_reply":"2022-01-20T11:28:41.403740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hourly_df.index","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:41.405445Z","iopub.execute_input":"2022-01-20T11:28:41.405682Z","iopub.status.idle":"2022-01-20T11:28:41.416407Z","shell.execute_reply.started":"2022-01-20T11:28:41.405648Z","shell.execute_reply":"2022-01-20T11:28:41.415557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details = pd.read_csv(path + 'asset_details.csv')\n# Add the asset symbols\nasset_details['Symbol'] = ['BCH', 'BNB', 'BTC', 'EOS', 'ETC', 'ETH', 'LTC', 'XMR', 'TRX', 'XLM', 'ADA', 'MIOTA', 'MKR', 'DOGE']\n\n# Sort value from smallest to highest\nasset_details.sort_values('Asset_ID', inplace = True)\nasset_weight_dict = {asset_details['Asset_ID'].tolist()[idx]: asset_details['Weight'].tolist()[idx] for idx in range(len(asset_details))}\nasset_name_dict = {asset_details['Asset_ID'].tolist()[idx]: asset_details['Asset_Name'].tolist()[idx] for idx in range(len(asset_details))}\nasset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:41.420144Z","iopub.execute_input":"2022-01-20T11:28:41.420372Z","iopub.status.idle":"2022-01-20T11:28:41.444507Z","shell.execute_reply.started":"2022-01-20T11:28:41.420345Z","shell.execute_reply":"2022-01-20T11:28:41.443729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dividing each asset into its own dataframe","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:09:56.406884Z","iopub.execute_input":"2022-01-17T14:09:56.407584Z","iopub.status.idle":"2022-01-17T14:09:56.411802Z","shell.execute_reply.started":"2022-01-17T14:09:56.407546Z","shell.execute_reply":"2022-01-17T14:09:56.411007Z"}}},{"cell_type":"code","source":"# Create 2 data frame dictionaries to store all the data frames by the minutes and hours\nminutes = {coin : pd.DataFrame for coin in asset_details['Asset_ID']} # by the minute\nhourly = {hourly_coin : pd.DataFrame for hourly_coin in asset_details['Asset_ID']}\n# Extract data for each asset and set time as index\nfor ID, symbol in zip(asset_details['Asset_ID'], asset_details['Symbol']):\n    minutes[symbol] = merged_df[:][merged_df['Asset_ID'] == ID]\n    minutes[symbol]['date'] = pd.to_datetime(minutes[symbol]['timestamp'],unit='s') # Convert from Unix timestamp to human time\n    minutes[symbol].set_index('date', inplace = True)  # Necessary in order to use resampling\n    hourly[symbol] = minutes[symbol].resample('H').mean().replace(np.NaN, 0)\n    hourly[symbol].reset_index(inplace = True) # Reset index in order to be able to sort\n    hourly[symbol]['date'] = pd.to_datetime(hourly[symbol]['timestamp'],unit='s')\n    hourly[symbol] = hourly[symbol].sort_index()\n    minutes[symbol].reset_index(inplace = True) # Reset index in order to be able to sort and work with the training function","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:48:40.787140Z","iopub.execute_input":"2022-01-20T11:48:40.789047Z","iopub.status.idle":"2022-01-20T11:48:46.398029Z","shell.execute_reply.started":"2022-01-20T11:48:40.789002Z","shell.execute_reply":"2022-01-20T11:48:46.397263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check to see if it's sorted correctly\nhourly['BNB']","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:44:45.726606Z","iopub.execute_input":"2022-01-20T11:44:45.727149Z","iopub.status.idle":"2022-01-20T11:44:45.751789Z","shell.execute_reply.started":"2022-01-20T11:44:45.727110Z","shell.execute_reply":"2022-01-20T11:44:45.751047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See the rows with NaN values\nhourly['ADA'].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:47.265427Z","iopub.execute_input":"2022-01-20T11:28:47.266084Z","iopub.status.idle":"2022-01-20T11:28:47.278335Z","shell.execute_reply.started":"2022-01-20T11:28:47.266049Z","shell.execute_reply":"2022-01-20T11:28:47.277524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"minutes['BNB']","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:49:02.165956Z","iopub.execute_input":"2022-01-20T11:49:02.166396Z","iopub.status.idle":"2022-01-20T11:49:02.191690Z","shell.execute_reply.started":"2022-01-20T11:49:02.166360Z","shell.execute_reply":"2022-01-20T11:49:02.190848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:02:40.252976Z","iopub.execute_input":"2022-01-17T14:02:40.253487Z","iopub.status.idle":"2022-01-17T14:02:40.37501Z","shell.execute_reply.started":"2022-01-17T14:02:40.253448Z","shell.execute_reply":"2022-01-17T14:02:40.374172Z"}}},{"cell_type":"markdown","source":"## Candlestick charts","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:23:32.092644Z","iopub.execute_input":"2022-01-17T14:23:32.092905Z","iopub.status.idle":"2022-01-17T14:23:32.096854Z","shell.execute_reply.started":"2022-01-17T14:23:32.092875Z","shell.execute_reply":"2022-01-17T14:23:32.096137Z"}}},{"cell_type":"code","source":"# Function to draw candlestick charts\ndef candlestick_chart(data, title):\n    candlestick = go.Figure(data = [go.Candlestick(x =data.index, \n                                               open = data[('Open')], \n                                               high = data[('High')], \n                                               low = data[('Low')], \n                                               close = data[('Close')])])\n    candlestick.update_xaxes(title_text = 'Time',\n                             rangeslider_visible = True)\n\n    candlestick.update_layout(\n    title = {\n        'text': '{:} Candlestick Chart'.format(title),\n        'y':0.90,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'} , \n    template=\"plotly_white\")\n\n    candlestick.update_yaxes(title_text = 'Price in USD', ticksuffix = '$')\n    return candlestick","metadata":{"execution":{"iopub.status.busy":"2022-01-19T14:20:13.258291Z","iopub.execute_input":"2022-01-19T14:20:13.258563Z","iopub.status.idle":"2022-01-19T14:20:13.264505Z","shell.execute_reply.started":"2022-01-19T14:20:13.258533Z","shell.execute_reply":"2022-01-19T14:20:13.263816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot BTC price for last 1000 points\nbtc_hourly = candlestick_chart(hourly['BTC'][-1000:], 'BTC price over time')\nbtc_hourly.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-19T14:26:39.214554Z","iopub.execute_input":"2022-01-19T14:26:39.214809Z","iopub.status.idle":"2022-01-19T14:26:40.175379Z","shell.execute_reply.started":"2022-01-19T14:26:39.21478Z","shell.execute_reply":"2022-01-19T14:26:40.174569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## OHLC charts","metadata":{}},{"cell_type":"code","source":"def ohlc_chart(data,title):\n    ohlc = go.Figure(data = [go.Ohlc(x =data.index, \n                                               open = data[('Open')], \n                                               high = data[('High')], \n                                               low = data[('Low')], \n                                               close = data[('Close')])])\n    ohlc.update_xaxes(title_text = 'Time',\n                             rangeslider_visible = True)\n\n    ohlc.update_layout(\n    title = {\n        'text': '{:} OHLC Chart'.format(title),\n        'y':0.9,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'},\n        template=\"plotly_white\")\n\n    ohlc.update_yaxes(title_text = 'Price in USD', ticksuffix = '$')\n    return ohlc","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:50:57.281153Z","iopub.execute_input":"2022-01-19T02:50:57.281424Z","iopub.status.idle":"2022-01-19T02:50:57.289459Z","shell.execute_reply.started":"2022-01-19T02:50:57.281387Z","shell.execute_reply":"2022-01-19T02:50:57.2885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Chart for ETH for last 200 rows\n# ohlc_chart(dfDict['ETH'][-200:], title = \"Ethereum(ETH) price over last 200 minutes\")","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:50:57.29133Z","iopub.execute_input":"2022-01-19T02:50:57.292145Z","iopub.status.idle":"2022-01-19T02:50:57.299557Z","shell.execute_reply.started":"2022-01-19T02:50:57.291939Z","shell.execute_reply":"2022-01-19T02:50:57.298712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Area plots (volumes traded)","metadata":{}},{"cell_type":"code","source":"def vol_traded(data ,title,color):\n    area = px.area(data_frame=data,\n               x = data.index ,\n               y = \"Volume\",\n               markers = True)\n    area.update_traces(line_color=color)\n    area.update_xaxes(\n        title_text = 'Time',\n        rangeslider_visible = True)\n    area.update_yaxes(title_text = 'Number of trades every minute')\n    area.update_layout(showlegend = True,\n        title = {\n            'text': '{:} Volume Traded'.format(title),\n            'y':0.94,\n            'x':0.5,\n            'xanchor': 'center',\n            'yanchor': 'top'},\n        template=\"plotly_white\")\n    return area","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:50:57.301138Z","iopub.execute_input":"2022-01-19T02:50:57.30142Z","iopub.status.idle":"2022-01-19T02:50:57.309195Z","shell.execute_reply.started":"2022-01-19T02:50:57.301384Z","shell.execute_reply":"2022-01-19T02:50:57.308413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vol_traded(dfDict['ETH'][-50:], \"Ethereum (ETH) volume over last 50 minutes\", color = \"Red\")","metadata":{"execution":{"iopub.status.busy":"2022-01-19T02:50:57.310561Z","iopub.execute_input":"2022-01-19T02:50:57.311149Z","iopub.status.idle":"2022-01-19T02:50:57.317043Z","shell.execute_reply.started":"2022-01-19T02:50:57.311112Z","shell.execute_reply":"2022-01-19T02:50:57.31636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"markdown","source":"# Variables to change","metadata":{}},{"cell_type":"code","source":"DEVICE = \"GPU\" #or \"CPU\"\n\nSEED = 25\n\n# CV PARAMS\nFOLDS = 5\nGROUP_GAP = 130\nMAX_TEST_GROUP_SIZE = 180\nMAX_TRAIN_GROUP_SIZE = 280\n\n# LOAD STRICT? YES=1 NO=0 | see: https://www.kaggle.com/julian3833/proposal-for-a-meaningful-lb-strict-lgbm\nLOAD_STRICT = True\n\n# WHICH YEARS TO INCLUDE? YES=1 NO=0\nINC2021 = 0\nINC2020 = 0\nINC2019 = 0\nINC2018 = 0\nINC2017 = 0\nINCCOMP = 1\nINCSUPP = 0\n\n# HYPER PARAMETERS\nLEARNING_RATE = [0.09, 0.09, 0.09, 0.09, 0.09]\nN_ESTIMATORS = [1000, 1000, 1000, 1000, 1000]\nMAX_DEPTH = [10, 10, 10, 10, 10]","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:44:39.519397Z","iopub.execute_input":"2022-01-20T11:44:39.519655Z","iopub.status.idle":"2022-01-20T11:44:39.527433Z","shell.execute_reply.started":"2022-01-20T11:44:39.519627Z","shell.execute_reply":"2022-01-20T11:44:39.526043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"code","source":"# Two features from the competition tutorial\ndef upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['lower_Shadow'] = lower_shadow(df_feat)\n    df_feat[\"high_div_low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    df_feat[\"open_sub_close\"] = df_feat[\"Open\"] - df_feat[\"Close\"]\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:47.289195Z","iopub.execute_input":"2022-01-20T11:28:47.289404Z","iopub.status.idle":"2022-01-20T11:28:47.298200Z","shell.execute_reply.started":"2022-01-20T11:28:47.289374Z","shell.execute_reply":"2022-01-20T11:28:47.297430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configure the model","metadata":{}},{"cell_type":"markdown","source":"## Competition metric","metadata":{}},{"cell_type":"code","source":"def corr(a, b, w):\n    cov = lambda x, y: np.sum(w * (x - np.average(x, weights=w)) * (y - np.average(y, weights=w))) / np.sum(w)\n    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))\n\n# Catboost Version\ndef get_catb_metric(w):\n    class WCorrMetric(object):\n        def get_final_error(self, error, weight): return error \n        def is_max_optimal(self): return True\n        def evaluate(self, approxes, target, weight):\n            approx = approxes[0]\n            weight_sum = 1.0\n            error_sum = corr(target, approx, w)\n            return error_sum, weight_sum","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:47.299331Z","iopub.execute_input":"2022-01-20T11:28:47.299645Z","iopub.status.idle":"2022-01-20T11:28:47.310976Z","shell.execute_reply.started":"2022-01-20T11:28:47.299608Z","shell.execute_reply":"2022-01-20T11:28:47.310247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The model","metadata":{}},{"cell_type":"code","source":"def build_model(fold, weight = 1.0):\n\n    model = CatBoostRegressor(iterations = N_ESTIMATORS[fold], depth = MAX_DEPTH[fold], \\\n                              learning_rate = LEARNING_RATE[fold], task_type = \"GPU\" if DEVICE == 'GPU' else None, \\\n                              eval_metric = get_catb_metric(weight))\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:28:47.312205Z","iopub.execute_input":"2022-01-20T11:28:47.312474Z","iopub.status.idle":"2022-01-20T11:28:47.320289Z","shell.execute_reply.started":"2022-01-20T11:28:47.312437Z","shell.execute_reply":"2022-01-20T11:28:47.319567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Time Series CV","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\nfrom sklearn.utils.validation import _deprecate_positional_args\n\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass GroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n    Provides train/test indices to split time series data samples\n    that are observed at fixed time intervals according to a\n    third-party provided group.\n    In each split, test indices must be higher than before, and thus shuffling\n    in cross validator is inappropriate.\n    This cross-validation object is a variation of :class:`KFold`.\n    In the kth split, it returns first k folds as train set and the\n    (k+1)th fold as test set.\n    The same group will not appear in two different folds (the number of\n    distinct groups has to be at least equal to the number of folds).\n    Note that unlike standard cross-validation methods, successive\n    training sets are supersets of those that come before them.\n    Read more in the :ref:`User Guide <cross_validation>`.\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_size : int, default=None\n        Maximum size for a single training set.\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from sklearn.model_selection import GroupTimeSeriesSplit\n    >>> groups = np.array(['a', 'a', 'a', 'a', 'a', 'a',\\\n                           'b', 'b', 'b', 'b', 'b',\\\n                           'c', 'c', 'c', 'c',\\\n                           'd', 'd', 'd'])\n    >>> gtss = GroupTimeSeriesSplit(n_splits=3)\n    >>> for train_idx, test_idx in gtss.split(groups, groups=groups):\n    ...     print(\"TRAIN:\", train_idx, \"TEST:\", test_idx)\n    ...     print(\"TRAIN GROUP:\", groups[train_idx],\\\n                  \"TEST GROUP:\", groups[test_idx])\n    TRAIN: [0, 1, 2, 3, 4, 5] TEST: [6, 7, 8, 9, 10]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a']\\\n    TEST GROUP: ['b' 'b' 'b' 'b' 'b']\n    TRAIN: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] TEST: [11, 12, 13, 14]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a' 'b' 'b' 'b' 'b' 'b']\\\n    TEST GROUP: ['c' 'c' 'c' 'c']\n    TRAIN: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]\\\n    TEST: [15, 16, 17]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a' 'b' 'b' 'b' 'b' 'b' 'c' 'c' 'c' 'c']\\\n    TEST GROUP: ['d' 'd' 'd']\n    \"\"\"\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_size=None\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_size = max_train_size\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n        group_test_size = n_groups // n_folds\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n            for train_group_idx in unique_groups[:group_test_start]:\n                train_array_tmp = group_dict[train_group_idx]\n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n            train_end = train_array.size\n            if self.max_train_size and self.max_train_size < train_end:\n                train_array = train_array[train_end -\n                                          self.max_train_size:train_end]\n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n            yield [int(i) for i in train_array], [int(i) for i in test_array]\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\nfrom sklearn.utils.validation import _deprecate_positional_args\n\n# modified code for group gaps; source\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass PurgedGroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n    Allows for a gap in groups to avoid potentially leaking info from\n    train into test if the model has windowed or lag features.\n    Provides train/test indices to split time series data samples\n    that are observed at fixed time intervals according to a\n    third-party provided group.\n    In each split, test indices must be higher than before, and thus shuffling\n    in cross validator is inappropriate.\n    This cross-validation object is a variation of :class:`KFold`.\n    In the kth split, it returns first k folds as train set and the\n    (k+1)th fold as test set.\n    The same group will not appear in two different folds (the number of\n    distinct groups has to be at least equal to the number of folds).\n    Note that unlike standard cross-validation methods, successive\n    training sets are supersets of those that come before them.\n    Read more in the :ref:`User Guide <cross_validation>`.\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_group_size : int, default=Inf\n        Maximum group size for a single training set.\n    group_gap : int, default=None\n        Gap between train and test\n    max_test_group_size : int, default=Inf\n        We discard this number of groups from the end of each train split\n    \"\"\"\n\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_group_size=np.inf,\n                 max_test_group_size=np.inf,\n                 group_gap=None,\n                 verbose=False\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_group_size = max_train_group_size\n        self.group_gap = group_gap\n        self.max_test_group_size = max_test_group_size\n        self.verbose = verbose\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        group_gap = self.group_gap\n        max_test_group_size = self.max_test_group_size\n        max_train_group_size = self.max_train_group_size\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n\n        group_test_size = min(n_groups // n_folds, max_test_group_size)\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n\n            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n                train_array_tmp = group_dict[train_group_idx]\n\n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n\n            train_end = train_array.size\n\n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n\n            test_array  = test_array[group_gap:]\n\n\n            if self.verbose > 0:\n                    pass\n\n            yield [int(i) for i in train_array], [int(i) for i in test_array]","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:29:03.619670Z","iopub.execute_input":"2022-01-20T11:29:03.620048Z","iopub.status.idle":"2022-01-20T11:29:03.685732Z","shell.execute_reply.started":"2022-01-20T11:29:03.619992Z","shell.execute_reply":"2022-01-20T11:29:03.684871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"## See the CV visually","metadata":{"execution":{"iopub.status.busy":"2022-01-20T06:33:54.942726Z","iopub.execute_input":"2022-01-20T06:33:54.94353Z","iopub.status.idle":"2022-01-20T06:33:54.947828Z","shell.execute_reply.started":"2022-01-20T06:33:54.943487Z","shell.execute_reply":"2022-01-20T06:33:54.946746Z"}}},{"cell_type":"code","source":"import numpy as np\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\n\ndef plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10):\n    cmap_cv = plt.cm.coolwarm\n    jet = plt.cm.get_cmap('jet', 256)\n    seq = np.linspace(0, 1, 256)\n    _ = np.random.shuffle(seq)   # inplace\n    cmap_data = ListedColormap(jet(seq))    \n    for ii, (tr, tt) in enumerate(list(cv.split(X=X, y=y, groups=group))):\n        indices = np.array([np.nan] * len(X))\n        indices[tt] = 1\n        indices[tr] = 0        \n        ax.scatter(range(len(indices)), [ii + .5] * len(indices), c=indices, marker='_', lw=lw, cmap=cmap_cv, vmin=-.2, vmax=1.2)\n    ax.scatter(range(len(X)), [ii + 1.5] * len(X), c=y, marker='_', lw=lw, cmap=plt.cm.Set3)\n    ax.scatter(range(len(X)), [ii + 2.5] * len(X), c=group, marker='_', lw=lw, cmap=cmap_data)\n    yticklabels = list(range(n_splits)) + ['target', 'day']\n    ax.set(yticks=np.arange(n_splits+2) + .5, yticklabels=yticklabels, xlabel='Sample index', ylabel=\"CV iteration\", ylim=[n_splits+2.2, -.2], xlim=[0, len(y)])\n    ax.set_title('{}'.format(type(cv).__name__), fontsize=15)\n    return ax\n\ndef plot_importance(importances, features_names, PLOT_TOP_N = 20, figsize=(12, 20)):\n    try: plt.close()\n    except: pass\n    importance_df = pd.DataFrame(data=importances, columns=features_names)\n    sorted_indices = importance_df.median(axis=0).sort_values(ascending=False).index\n    sorted_importance_df = importance_df.loc[:, sorted_indices]\n    plot_cols = sorted_importance_df.columns[:PLOT_TOP_N]\n    _, ax = plt.subplots(figsize=figsize)\n    ax.grid()\n    ax.set_xscale('log')\n    ax.set_ylabel('Feature')\n    ax.set_xlabel('Importance')\n    plt.title('Feature Importances')\n    sns.boxplot(data=sorted_importance_df[plot_cols], orient='h', ax=ax)\n    plt.show()\n    \n# asset_id = 0\n# df = load_training_data_for_asset(asset_id) # not loading because I use my the original data 'merged_df' (too big)\n# Using hourly_df, if this is still too big, I'll just use each asset df, e.g. hourly['BNB']\ndf = minutes['BTC']  # Change here to see different asset and timeline (minutes, hourly)\ndf_proc = get_features(df)\ndf_proc['date'] = df['date'].copy() \ndf_proc['y'] = df['Target']\ndf_proc = df_proc.dropna(how=\"any\")\nX = df_proc.drop(\"y\", axis=1)\ny = df_proc[\"y\"]\ngroups = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\nX = X.drop(columns = 'date')\n\nfig, ax = plt.subplots(figsize = (12, 6))\ncv = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap = GROUP_GAP, max_train_group_size=MAX_TRAIN_GROUP_SIZE, max_test_group_size=MAX_TEST_GROUP_SIZE)\nplot_cv_indices(cv, X, y, groups, ax, FOLDS, lw=20)","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:49:25.900117Z","iopub.execute_input":"2022-01-20T11:49:25.900558Z","iopub.status.idle":"2022-01-20T11:52:18.830398Z","shell.execute_reply.started":"2022-01-20T11:49:25.900522Z","shell.execute_reply":"2022-01-20T11:52:18.829709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main training function","metadata":{}},{"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 0\n\n# Train per asset, so I need to use hourly df of each asset\ndef get_Xy_and_model_for_asset(asset_id, symbol):\n#     df = load_training_data_for_asset(asset_id) # No need because my data is separate\n    df = minutes[symbol]   # modified to work with the hourly dataframe by asset\n    orig_close = df['Close'].copy()\n    df_proc = get_features(df)\n    df_proc['date'] = df['date'].copy()\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    groups = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\n    X = X.drop(columns = 'date')\n    oof_preds = np.zeros(len(X))\n    importances, scores, models = [], [], []\n    for fold, (train_idx, val_idx) in enumerate(PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap = GROUP_GAP, max_train_group_size = MAX_TRAIN_GROUP_SIZE, max_test_group_size = MAX_TEST_GROUP_SIZE).split(X, y, groups)):\n        # GET TRAINING, VALIDATION SET\n        x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n        # DISPLAY FOLD INFO\n        print('#' * 25); print('#### FOLD', fold + 1)\n        print('#### Training N_ESTIMATORS %s | MAX_DEPTH %s | LEARNING_RATE %s' % (N_ESTIMATORS[fold], MAX_DEPTH[fold], LEARNING_RATE[fold]))\n\n        model = build_model(fold, weight = np.array([asset_weight_dict[asset_id]] * len(y_val)))\n\n        # TRAIN\n        model.fit( x_train, y_train, eval_set = [(x_val, y_val)], early_stopping_rounds = 50, verbose = VERBOSE)\n\n        # PREDICT OOF\n        pred = model.predict(x_val)\n        models.append(model)       \n        \n        # REPORT RESULTS\n        try: mse = mean_squared_error(np.nan_to_num(y_val), np.nan_to_num(pred))\n        except: mse = 0.0\n\n        scores.append(mse)\n        oof_preds[val_idx] = pred\n        importances.append(model.get_feature_importance())\n        \n        w_score = corr(np.nan_to_num(y_val), np.nan_to_num(pred.flatten()), np.array([asset_weight_dict[asset_id]] * len(y_val)))\n        print('#### FOLD %i OOF MSE %.3f | WCORR: %.3f' % (fold + 1, mse, w_score))\n        \n    df = df_proc\n    df['oof_preds'] = np.nan_to_num(oof_preds)\n    df['Close'] = orig_close\n    print('\\n\\n' + ('-' * 80) + '\\n' + 'Finished training %s. Results:' % asset_name_dict[asset_id])\n    print('Model: r2_score: %s | pearsonr: %s | wcorr: %s ' % (r2_score(df['y'], df['oof_preds']), pearsonr(df['y'], df['oof_preds'])[0], corr(df['y'].values, df['oof_preds'].values, np.array([asset_weight_dict[asset_id]] * len(df['y'].values)))))\n    print('Predictions std: %s | Target std: %s' % (df['oof_preds'].std(), df['y'].std()))\n    try: plt.close()\n    except: pass   \n    df2 = df.reset_index(drop = True).set_index('date')\n    fig = plt.figure(figsize = (12, 6))\n    # fig, ax_left = plt.subplots(figsize = (12, 6))\n    ax_left = fig.add_subplot(111)\n    ax_left.set_facecolor('azure')    \n    ax_right = ax_left.twinx()\n    ax_left.plot(df2['y'].rolling(3 * 30 * 24 * 60).corr(df2['oof_preds']).iloc[::24 * 60], color = 'crimson', label = \"Target WCorr\")\n    ax_right.plot(df2['Close'].iloc[::24 * 60], color = 'darkgrey', label = \"%s Close\" % asset_name_dict[asset_id])   \n    plt.legend()\n    plt.grid()\n    plt.xlabel('Time')\n    plt.title('3 month rolling pearsonr for %s' % (asset_name_dict[asset_id]))\n    plt.show()\n    \n    plot_importance(np.array(importances), list(X.columns), PLOT_TOP_N = 20)\n    \n    return scores, oof_preds, models, y\n\nmodels, scores, targets, oof_preds = {}, {}, {}, {}\n\n# Traing all assets at once, modified to work with my hourly df for each model\nfor asset_id, symbol in zip(asset_details['Asset_ID'], asset_details['Symbol']):\n    print(f\"Training model for {symbol:<6} (ID={asset_id:<2})\")\n    cur_scores, cur_oof_preds, cur_models, cur_targets = get_Xy_and_model_for_asset(asset_id, symbol)\n    scores[asset_id], oof_preds[asset_id], models[asset_id], targets[asset_id] = np.mean(cur_scores), cur_oof_preds, cur_models, cur_targets","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:52:18.831988Z","iopub.execute_input":"2022-01-20T11:52:18.832412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate OOF MSE","metadata":{}},{"cell_type":"code","source":"# COMPUTE OVERALL OOF MSE\nprint('Overall MEAN OOF MSE %s' % np.mean(list(scores.values())))\n\n# SAVE OOF TO DISK \ny_pred, y_true, weights = [], [], []\nfor asset in oof_preds:\n    df_oof = pd.DataFrame(dict(asset_id = asset, oof_preds=oof_preds[asset]))\n    df_oof.to_csv(str(asset) + '_oof.csv',index=False)\n    y_pred += oof_preds[asset].tolist()\n    y_true += targets[asset].tolist() \n    weights += ([asset_weight_dict[asset]] * len(oof_preds[asset].tolist()))\n    print('%s score: %s' % (asset_name_dict[asset], corr(np.nan_to_num(np.array(y_true).flatten()), np.nan_to_num(np.array(y_pred).flatten()), np.nan_to_num(np.array(weights).flatten()))))\n    \nprint('Overall score %s' % corr(np.nan_to_num(np.array(y_true).flatten()), np.nan_to_num(np.array(y_pred).flatten()), np.nan_to_num(np.array(weights).flatten())))","metadata":{"execution":{"iopub.status.busy":"2022-01-20T11:36:09.851142Z","iopub.execute_input":"2022-01-20T11:36:09.852040Z","iopub.status.idle":"2022-01-20T11:36:12.503827Z","shell.execute_reply.started":"2022-01-20T11:36:09.851993Z","shell.execute_reply":"2022-01-20T11:36:12.502949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save the results for submission to Kaggle","metadata":{}},{"cell_type":"code","source":"all_df_test = []\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        try:\n            if row['Asset_ID'] in models:\n                cur_models = models[row['Asset_ID']]\n                x_test = get_features(row)\n                y_pred = np.mean(np.concatenate([np.expand_dims(model.predict([x_test]), axis = 0) for model in cur_models], axis = 0), axis = 0)\n            else: y_pred = 0.0\n        except: \n            y_pred = 0.0\n            traceback.print_exc()\n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n    all_df_test.append(df_test)\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:14:54.83231Z","iopub.execute_input":"2022-01-20T08:14:54.833201Z","iopub.status.idle":"2022-01-20T08:14:55.289749Z","shell.execute_reply.started":"2022-01-20T08:14:54.833144Z","shell.execute_reply":"2022-01-20T08:14:55.288971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_df_test","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:18:04.839719Z","iopub.execute_input":"2022-01-20T08:18:04.840389Z","iopub.status.idle":"2022-01-20T08:18:04.867705Z","shell.execute_reply.started":"2022-01-20T08:18:04.840343Z","shell.execute_reply":"2022-01-20T08:18:04.866976Z"},"trusted":true},"execution_count":null,"outputs":[]}]}