{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0\"></a>\n# Competition \"[G-Research Crypto Forecasting](https://www.kaggle.com/c/g-research-crypto-forecasting)\"\n\n## Research","metadata":{"papermill":{"duration":0.015064,"end_time":"2022-01-21T18:18:23.574093","exception":false,"start_time":"2022-01-21T18:18:23.559029","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Acknowledgements:\n* the main code - from notebooks [G-Research Crypto Forecasting - baseline & FE](https://www.kaggle.com/vbmokin/g-research-crypto-forecasting-baseline-fe) and [💲💲G-Research- Starter LGBM Pipeline](https://www.kaggle.com/julian3833/g-research-starter-lgbm-pipeline)\n* the model tuning - from the notebook [G-Research: XGBoost with GPU (Fit in 1min)](https://www.kaggle.com/yamqwe/g-research-xgboost-with-gpu-fit-in-1min)\n* FE - from notebooks [GResearch Simple LGB Starter](https://www.kaggle.com/code1110/gresearch-simple-lgb-starter) and [⚡⚡Bitcoin Crash Prediction⚡⚡](https://www.kaggle.com/kartik2khandelwal/bitcoin-crash-prediction)\n* remove inf, NaN - from [[Crypto] Beginner's Try for simple LGBM (En/Jp)](https://www.kaggle.com/junjitakeshima/crypto-beginner-s-try-for-simple-lgbm-en-jp)","metadata":{"papermill":{"duration":0.013482,"end_time":"2022-01-21T18:18:23.603433","exception":false,"start_time":"2022-01-21T18:18:23.589951","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0.1\"></a>\n\n## Table of Contents\n\n1. [Import libraries](#1)\n1. [Download datasets](#2)\n1. [FE](#3)\n1. [Model training and prediction](#4)\n1. [Save results](#5)","metadata":{"papermill":{"duration":0.013314,"end_time":"2022-01-21T18:18:23.630291","exception":false,"start_time":"2022-01-21T18:18:23.616977","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1. Import libraries <a class=\"anchor\" id=\"1\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.013181,"end_time":"2022-01-21T18:18:23.656991","exception":false,"start_time":"2022-01-21T18:18:23.64381","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport random\nimport gc\n\nfrom scipy import spatial\nfrom datetime import datetime\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n\n# model tuning\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe, space_eval\nfrom sklearn.model_selection import cross_val_score\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n# Import library with the model\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import RandomForestRegressor\n\n#import gresearch_crypto\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"papermill":{"duration":2.138076,"end_time":"2022-01-21T18:18:25.808597","exception":false,"start_time":"2022-01-21T18:18:23.670521","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-31T14:53:16.979203Z","iopub.execute_input":"2022-01-31T14:53:16.979557Z","iopub.status.idle":"2022-01-31T14:53:21.117633Z","shell.execute_reply.started":"2022-01-31T14:53:16.979479Z","shell.execute_reply":"2022-01-31T14:53:21.116905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nseed=4134\nfix_all_seeds(seed)","metadata":{"papermill":{"duration":0.022078,"end_time":"2022-01-21T18:18:25.847899","exception":false,"start_time":"2022-01-21T18:18:25.825821","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-31T14:53:21.12036Z","iopub.execute_input":"2022-01-31T14:53:21.120888Z","iopub.status.idle":"2022-01-31T14:53:21.125755Z","shell.execute_reply.started":"2022-01-31T14:53:21.120825Z","shell.execute_reply":"2022-01-31T14:53:21.125071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#models = ['XGBRegressor', 'LGBMRegressor', 'RandomForestRegressor']\nmodels = ['LGBMRegressor']\n# models = ['CatBoostRegressor']\nAsset_ID = 2   # Bitcoin Cash\ncv_number = 3   # cross-validation\nnum_models = 10   # number of models under tuning","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:53:21.127244Z","iopub.execute_input":"2022-01-31T14:53:21.127792Z","iopub.status.idle":"2022-01-31T14:53:21.142295Z","shell.execute_reply.started":"2022-01-31T14:53:21.127752Z","shell.execute_reply":"2022-01-31T14:53:21.141628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Download datasets <a class=\"anchor\" id=\"2\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.014341,"end_time":"2022-01-21T18:18:25.876777","exception":false,"start_time":"2022-01-21T18:18:25.862436","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\ndf_train = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')\n# df_train = df_train[(df_train['Asset_ID']==Asset_ID)].reset_index(drop=True)\ndf_train.head()","metadata":{"papermill":{"duration":60.663951,"end_time":"2022-01-21T18:19:26.555922","exception":false,"start_time":"2022-01-21T18:18:25.891971","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-31T14:53:21.146185Z","iopub.execute_input":"2022-01-31T14:53:21.146391Z","iopub.status.idle":"2022-01-31T14:54:19.249832Z","shell.execute_reply.started":"2022-01-31T14:53:21.146366Z","shell.execute_reply":"2022-01-31T14:54:19.249146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['ds'] = pd.to_datetime(df_train[\"timestamp\"], unit=\"s\", infer_datetime_format=True)\ndf_train[:3]","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:19.251289Z","iopub.execute_input":"2022-01-31T14:54:19.251762Z","iopub.status.idle":"2022-01-31T14:54:20.562089Z","shell.execute_reply.started":"2022-01-31T14:54:19.251723Z","shell.execute_reply":"2022-01-31T14:54:20.56141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[df_train.ds.dt.year.isin([2020,2021])]\ndf_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:20.563154Z","iopub.execute_input":"2022-01-31T14:54:20.563404Z","iopub.status.idle":"2022-01-31T14:54:23.149472Z","shell.execute_reply.started":"2022-01-31T14:54:20.563368Z","shell.execute_reply":"2022-01-31T14:54:23.148703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv')\ndf_asset_details","metadata":{"papermill":{"duration":0.034984,"end_time":"2022-01-21T18:19:26.606231","exception":false,"start_time":"2022-01-21T18:19:26.571247","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-31T14:54:23.15071Z","iopub.execute_input":"2022-01-31T14:54:23.150965Z","iopub.status.idle":"2022-01-31T14:54:23.167317Z","shell.execute_reply.started":"2022-01-31T14:54:23.150937Z","shell.execute_reply":"2022-01-31T14:54:23.166716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_name = df_asset_details[df_asset_details['Asset_ID']==Asset_ID]['Asset_Name'].tolist()[0]\nasset_name","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:23.168618Z","iopub.execute_input":"2022-01-31T14:54:23.168895Z","iopub.status.idle":"2022-01-31T14:54:23.174953Z","shell.execute_reply.started":"2022-01-31T14:54:23.168858Z","shell.execute_reply":"2022-01-31T14:54:23.174281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_names = df_asset_details.Asset_Name\nasset_names","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:23.176294Z","iopub.execute_input":"2022-01-31T14:54:23.176817Z","iopub.status.idle":"2022-01-31T14:54:23.186358Z","shell.execute_reply.started":"2022-01-31T14:54:23.17677Z","shell.execute_reply":"2022-01-31T14:54:23.185669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clear memory\n# del df_asset_details\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:23.189513Z","iopub.execute_input":"2022-01-31T14:54:23.190122Z","iopub.status.idle":"2022-01-31T14:54:23.195093Z","shell.execute_reply.started":"2022-01-31T14:54:23.190091Z","shell.execute_reply":"2022-01-31T14:54:23.194353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_asset_to_other_assets = {\n    1: [9, 12, 6, 11, 8],\n    2: [9, 11, 8, 12, 5],\n    0: [6, 11, 8, 13, 10],\n    5: [2, 9, 11, 13],\n    7: [6, 3, 10, 4],\n    6: [0, 1, 7, 3],\n    9: [1, 2, 11, 12, 8],\n    11: [2, 0, 9, 11, 13],\n    13: [0, 8, 11, 2],\n    12: [1, 2, 9, 8],\n    3: [6, 10, 0],\n    8: [9, 11, 13, 12, 0],\n    10: [2, 0, 6, 11, 4],\n    4: [7, 10, 0, 6]\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:23.196414Z","iopub.execute_input":"2022-01-31T14:54:23.196936Z","iopub.status.idle":"2022-01-31T14:54:23.205723Z","shell.execute_reply.started":"2022-01-31T14:54:23.196898Z","shell.execute_reply":"2022-01-31T14:54:23.205037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"asset_list = base_asset_to_other_assets[Asset_ID]\nasset_list.append(Asset_ID)\nasset_list","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:23.207081Z","iopub.execute_input":"2022-01-31T14:54:23.207551Z","iopub.status.idle":"2022-01-31T14:54:23.218173Z","shell.execute_reply.started":"2022-01-31T14:54:23.207515Z","shell.execute_reply":"2022-01-31T14:54:23.217404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# close_corr = df_train[df_train.Asset_ID == 1][['timestamp','Close']].rename(columns={'Close': 'Bitcoin'}).dropna(axis=0)\n# for ass_id, ass_name in assets_dict.items():\n#     if ass_id != 1:\n#         close_asset = df_train[df_train.Asset_ID == ass_id][['timestamp','Close']].rename(columns={'Close': ass_name}).dropna(axis=0)\n#         close_corr = pd.merge(close_corr, close_asset, how='left', on='timestamp')\n\n# close_corr.dropna(inplace=True)\n# close_corr.dropna(inplace=True)\n# close_corr\n# corr_matrix = round(close_corr.drop(['timestamp'], axis=1).corr(), 2)\n# plt.figure(figsize=(10, 10))\n# sns.heatmap(corr_matrix)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:23.219492Z","iopub.execute_input":"2022-01-31T14:54:23.220011Z","iopub.status.idle":"2022-01-31T14:54:23.226645Z","shell.execute_reply.started":"2022-01-31T14:54:23.219972Z","shell.execute_reply":"2022-01-31T14:54:23.225753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a  =pd.DataFrame({'a': [1, 2], 'b': [1, 2]})\n","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:23.228007Z","iopub.execute_input":"2022-01-31T14:54:23.228498Z","iopub.status.idle":"2022-01-31T14:54:23.234733Z","shell.execute_reply.started":"2022-01-31T14:54:23.228464Z","shell.execute_reply":"2022-01-31T14:54:23.233989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. FE <a class=\"anchor\" id=\"3\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.015208,"end_time":"2022-01-21T18:19:26.636959","exception":false,"start_time":"2022-01-21T18:19:26.621751","status":"completed"},"tags":[]}},{"cell_type":"code","source":"thr_date = datetime.strptime(\"2020-01-01\", '%Y-%m-%d') # Remove date before\ndf_train = df_train[(df_train['Asset_ID'].isin(asset_list)) & (df_train['ds'] >= thr_date)].reset_index(drop=True)\ndf_train = df_train[df_train['ds'] >= thr_date].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:23.237412Z","iopub.execute_input":"2022-01-31T14:54:23.238115Z","iopub.status.idle":"2022-01-31T14:54:24.325502Z","shell.execute_reply.started":"2022-01-31T14:54:23.238079Z","shell.execute_reply":"2022-01-31T14:54:24.324757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('max_columns',100)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:24.352914Z","iopub.execute_input":"2022-01-31T14:54:24.353162Z","iopub.status.idle":"2022-01-31T14:54:24.35911Z","shell.execute_reply.started":"2022-01-31T14:54:24.353127Z","shell.execute_reply":"2022-01-31T14:54:24.35837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef get_features(data, asset_list, Asset_ID, is_row=False):\n    # FE for data as row of DataFrame\n    \n    asset_ids = asset_list\n    \n    def sliced_hourly_fearture(df, start_hour=3, divide_hour=4):\n        # Get hourly fearture\n        \n        list_hours = list(range(0, 24))\n        list_hours = list_hours[start_hour:] + list_hours[:start_hour]\n        divide_hour_start = 0\n        dic = {}\n        divide_tmp = divide_hour\n        for i in range(1, 7):\n            dic[tuple(list_hours[divide_hour_start: divide_hour])] = i\n            divide_hour_start = divide_hour\n            divide_hour += divide_tmp\n        feature = df.ds.dt.hour.apply(lambda x: [v for k, v in dic.items() if x in k][0])\n        return feature\n\n    def get_other_asset_as_feature(asset_ids: list, data, Asset_ID, df_feat):\n        \n        asset_ids.remove(Asset_ID)\n\n        for asset in asset_ids:\n            df_asset = data[data.Asset_ID == asset][['ds', 'Open', 'High', 'Low', 'Close', 'VWAP']]\n\n            df_asset['hammer_'+str(asset)] = df_asset[['High', 'Low']].mean(axis=1) - df_asset[['Open', 'Close']].mean(axis=1)\n            df_asset['OpnCls_sub_VWAP_'+str(asset)] = df_asset[['Open', 'Close']].mean(axis=1) - df_asset['VWAP']\n            df_asset['OpnVWAP_2_ClsVWAP_'+str(asset)] = ((df_asset['Open'] - df_asset['VWAP']) / (df_asset['Close'] - df_asset['VWAP'])).replace([np.inf, -np.inf, np.nan], 0.)\n\n            print(f'df_asset {asset} {df_asset.shape}')\n            asset_features = ['ds', 'hammer_'+str(asset), 'OpnCls_sub_VWAP_'+str(asset), 'OpnVWAP_2_ClsVWAP_'+str(asset)]    \n            df_feat = pd.merge(df_feat, df_asset[asset_features], how='left', on='ds')\n            print(f'df_feat {asset} {df_asset.shape}')\n        return df_feat            \n    \n    \n    \n    # Time features     \n    df_feat = data[['ds', 'Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Target']].copy()[data.Asset_ID == Asset_ID]\n    df_feat = df_feat.replace([np.inf, -np.inf], np.nan)\n    df_feat = df_feat.dropna()    \n    \n    # Thanks to https://www.kaggle.com/kartik2khandelwal/bitcoin-crash-prediction\n#     df_feat['ds'] = pd.to_datetime(df_feat[\"timestamp\"], unit=\"s\", infer_datetime_format=True)\n    print(df_feat.shape)\n    print('after dropna 1', df_feat['ds'].max())\n    print(df_feat.shape)\n#     # Thanks to https://www.kaggle.com/code1110/gresearch-simple-lgb-starter\n    df_feat['high2low'] = df_feat['High'] / df_feat['Low']\n    df_feat['volume2count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    \n    if is_row:\n        df_feat[\"hour\"] = df_feat['ds'].hour\n        df_feat[\"dayofweek\"] = df_feat['ds'].dayofweek \n        df_feat[\"month\"] = df_feat['ds'].month  # my upgrade\n        df_feat[\"year\"] = df_feat['ds'].year    # my upgrade\n    else:\n        df_feat[\"hour\"] = df_feat['ds'].dt.hour\n        df_feat[\"dayofweek\"] = df_feat['ds'].dt.dayofweek \n        df_feat[\"month\"] = df_feat['ds'].dt.month  # my upgrade\n        df_feat[\"year\"] = df_feat['ds'].dt.year     # my upgrade    \n    \n    # Two new features from the competition tutorial\n    df_feat['upper_shadow'] = df_feat['High'] - np.maximum(df_feat['Close'], df_feat['Open'])\n    df_feat['lower_shadow'] = np.minimum(df_feat['Close'], df_feat['Open']) - df_feat['Low']\n    \n       \n    # LM fearatures\n    df_feat['h_l'] = df_feat['High'] - df_feat['Low']\n    df_feat['return'] = (df_feat['Close'] - df_feat['Open']) / df_feat['Open']\n    df_feat['close2open'] = (df_feat['Close'] - df_feat['Open']) / 2\n    df_feat['hlco_ratio'] = (df_feat['High'] - df_feat['Low'])/(df_feat['Close'] - df_feat['Open'] + 1.001)\n#     df_feat['ohlc'] = np.mean((df_feat['High'].values, df_feat['Open'].values, df_feat['Low'].values, df_feat['Close'].values))\n    df_feat['Close_position']=(data['Close'] - data['Low'])/(data['High'] - data['Low'] + 1.001)\n    df_feat['Open_position']=(data['Open'] - data['Low'])/(data['High'] - data['Low'] + 1.001)\n    \n    #binary features\n    df_feat['close_mt_open'] = (df_feat['Close'] > df_feat['Open']).astype('int')\n    \n#     timestamp = df_train.timestamp.astype('datetime64[s]')\n    df_feat['is_weekend'] = df_feat['dayofweek'].isin([5, 6]).astype('int')\n    df_feat['yearday'] = df_feat['ds'].dt.dayofyear\n    df_feat['monthday'] = df_feat['ds'].dt.day\n    df_feat['minute'] = df_feat['ds'].dt.minute\n    df_feat['is_PM'] = (df_feat['hour'] >= 12).astype('int')\n    df_feat['quarter'] = df_feat['ds'].dt.quarter\n    \n    # quater of hour (as one hot) or one feture {1, 2, 3, 4}\n    df_feat['1q'] = ((df_feat['minute'] < 15) & (df_feat['minute']>=0)).astype('int')\n    df_feat['2q'] = ((df_feat['minute'] < 30) & (df_feat['minute']>=15)).astype('int')\n    df_feat['3q'] = ((df_feat['minute'] < 45) & (df_feat['minute']>=30)).astype('int')\n    df_feat['4q'] = ((df_feat['minute'] < 60) & (df_feat['minute']>=45)).astype('int')\n    \n    #sliced fature\n    df_feat['h4_0'] = sliced_hourly_fearture(df_feat, start_hour=0, divide_hour=4)\n    df_feat['h4_1'] = sliced_hourly_fearture(df_feat, start_hour=1, divide_hour=4)\n    df_feat['h4_2'] = sliced_hourly_fearture(df_feat, start_hour=2, divide_hour=4)\n    df_feat['h4_3'] = sliced_hourly_fearture(df_feat, start_hour=3, divide_hour=4)\n    \n    # Adding some more features - thanks to https://www.kaggle.com/kartik2khandelwal/bitcoin-crash-prediction\n    df_feat[\"close/open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n#     df_feat[\"close-open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"high-low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"high/low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    \n    # Adding features from SO\n    df_feat['hammer'] = df_feat[['High', 'Low']].mean(axis=1) - df_feat[['Open', 'Close']].mean(axis=1)\n    df_feat['body'] = np.abs(df_feat['Close'] - df_feat['Open'])\n    df_feat['dozhi'] = (df_feat['upper_shadow'] + df_feat['lower_shadow']) - df_feat['body']\n    df_feat['dozhi_div'] = ((df_feat['upper_shadow'] + df_feat['lower_shadow']) / df_feat['body']).replace([np.inf, -np.inf, np.nan], 0.)\n    df_feat['up_shdw_sub_lo_shdw'] = df_feat['upper_shadow'] - df_feat['lower_shadow']\n    df_feat['up_shdw_div_lo_shdw'] = (df_feat['upper_shadow'] / df_feat['lower_shadow']).replace([np.inf, -np.inf, np.nan], 0.)\n    df_feat['Cls_sub_VWAP'] = df_feat['Close'] - df_feat['VWAP']\n    df_feat['OpnCls_sub_VWAP'] = df_feat[['Open', 'Close']].mean(axis=1) - df_feat['VWAP']\n    df_feat['OpnVWAP_2_ClsVWAP'] = ((df_feat['Open'] - df_feat['VWAP']) / (df_feat['Close'] - df_feat['VWAP'])).replace([np.inf, -np.inf, np.nan], 0.)\n    df_feat['HighVWAP_2_LowVWAP'] = ((df_feat['High'] - df_feat['VWAP']) / (df_feat['Low'] - df_feat['VWAP'])).replace([np.inf, -np.inf, np.nan], 0.)\n    \n    if is_row:\n        df_feat['mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n\n    df_feat[\"high/mean\"] = df_feat[\"High\"] / df_feat[\"mean\"]\n    df_feat[\"low/mean\"] = df_feat[\"Low\"] / df_feat[\"mean\"]\n    \n    if is_row:\n        df_feat[\"median\"] = df_feat[[\"Open\", \"High\", \"Low\", \"Close\"]].median()\n    else:\n        df_feat[\"median\"] = df_feat[[\"Open\", \"High\", \"Low\", \"Close\"]].median(axis=1)\n    df_feat[\"high/median\"] = df_feat[\"High\"] / df_feat[\"median\"]\n    df_feat[\"low/median\"] = df_feat[\"Low\"] / df_feat[\"median\"]\n\n    for col in ['Open', 'High', 'Low', 'Close', 'VWAP']:\n        df_feat[f\"log_1p_{col}\"] = np.log1p(df_feat[col])\n        \n    # My upgrade\n    if is_row:\n        df_feat['mean_main'] = df_feat[['Open', 'Close']].mean()\n    else:\n        df_feat['mean_main'] = df_feat[['Open', 'Close']].mean(axis=1)\n\n    df_feat[\"high/mean_main\"] = df_feat[\"High\"] / df_feat[\"mean_main\"]\n    df_feat[\"low/mean_main\"] = df_feat[\"Low\"] / df_feat[\"mean_main\"]\n    \n    if is_row:\n        df_feat[\"median_main\"] = df_feat[[\"Open\", \"Close\"]].median()\n    else:\n        df_feat[\"median_main\"] = df_feat[[\"Open\", \"Close\"]].median(axis=1)\n    df_feat[\"high/median_main\"] = df_feat[\"High\"] / df_feat[\"median_main\"]\n    df_feat[\"low/median_main\"] = df_feat[\"Low\"] / df_feat[\"median_main\"]\n\n    #     # ******** Additional features ******\n    df_feat['Rel_Upper'] = ((df_feat['High'] - df_feat['VWAP']) / (df_feat['High'] - df_feat['Low'] + 0.0001))#.replace([np.inf, -np.inf, np.nan], 0.)\n    df_feat['Upper_VWAP'] = ((df_feat['High'] - df_feat['VWAP']) / df_feat['VWAP'] )#.replace([np.inf, -np.inf, np.nan], 0.)\n    df_feat['Upper_Vol'] = ((df_feat['High'] - df_feat['VWAP']) / df_feat['Volume'])#.replace([np.inf, -np.inf, np.nan], 0.)\n    \n    # ******** Additional features ******\n    # https://www.kaggle.com/azzamradman/g-research-more-eda\n    df_feat['is_month_start'] = df_feat['ds'].dt.is_month_start.astype(int).astype('uint8')\n    df_feat['is_month_end'] = df_feat['ds'].dt.is_month_end.astype(int).astype('uint8')\n    df_feat['is_quarter_start'] = df_feat['ds'].dt.is_quarter_start.astype(int).astype('uint8')\n    df_feat['is_quarter_end'] = df_feat['ds'].dt.is_quarter_end.astype(int).astype('uint8')\n    df_feat['is_year_start'] = df_feat['ds'].dt.is_year_start.astype(int).astype('uint8')\n    df_feat['is_year_end'] = df_feat['ds'].dt.is_year_end.astype(int).astype('uint8')\n\n    # ******** Additional features ******\n    df_feat['Dollars'] = df_feat['Volume'] * df_feat['Close']\n    #Volume_per_trade = Volume/Count\n    df_feat['Dollars_per_trade'] = df_feat.Dollars / (df_feat.Count + 1)\n\n    df_feat['log_ret'] = np.log1p(df_feat.Close/df_feat.Open)\n    df_feat['log_ret_H'] = np.log1p(df_feat.High/df_feat.Close)\n    df_feat['log_ret_L'] = np.log1p(df_feat.Close/df_feat.Low)\n    df_feat['log_ret_VWAP'] = np.log1p(df_feat.Close/df_feat.VWAP)\n    \n    df_feat['GK_vol'] = (1 / 2 * np.log1p(df_feat.High/df_feat.Low) ** 2 - (2 * np.log(2) - 1) * np.log(df_feat.Close/df_feat.Open) ** 2)\n    df_feat['RS_vol'] = np.log(df_feat.High/df_feat.Close)*np.log(df_feat.High/df_feat.Open) + np.log(df_feat.Low/df_feat.Close)*np.log(df_feat.Low/df_feat.Open)\n    print(f\"shape before oher feat {df_feat.shape}\")\n    # get features from another asset\n    \n    \n    df_feat = get_other_asset_as_feature(asset_ids, data, Asset_ID, df_feat)\n    \n    print('after FE', df_feat['ds'].max())\n    #df_feat = df_feat[~df_feat.isin([np.nan, np.inf, -np.inf]).any(1)].reset_index(drop=True)\n    #df_feat = df_feat.dropna(how=\"any\")\n    df_feat = df_feat.replace([np.inf, -np.inf], np.nan)\n    df_feat = df_feat.dropna()\n    print('after dropna 2', df_feat['ds'].max())\n    \n    return df_feat\n\ndf_train = get_features(df_train, asset_list, Asset_ID=Asset_ID)\ndisplay(df_train)\ndf_train.shape","metadata":{"papermill":{"duration":0.026093,"end_time":"2022-01-21T18:19:26.678473","exception":false,"start_time":"2022-01-21T18:19:26.65238","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-31T14:54:24.360566Z","iopub.execute_input":"2022-01-31T14:54:24.360903Z","iopub.status.idle":"2022-01-31T14:54:33.1798Z","shell.execute_reply.started":"2022-01-31T14:54:24.360868Z","shell.execute_reply":"2022-01-31T14:54:33.179098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:33.181029Z","iopub.execute_input":"2022-01-31T14:54:33.18158Z","iopub.status.idle":"2022-01-31T14:54:33.195922Z","shell.execute_reply.started":"2022-01-31T14:54:33.181539Z","shell.execute_reply":"2022-01-31T14:54:33.195223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:33.196865Z","iopub.execute_input":"2022-01-31T14:54:33.197467Z","iopub.status.idle":"2022-01-31T14:54:34.773282Z","shell.execute_reply.started":"2022-01-31T14:54:33.19743Z","shell.execute_reply":"2022-01-31T14:54:34.772602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of features (without target)\nnum_features = df_train.shape[1]-1\nnum_features","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:34.774453Z","iopub.execute_input":"2022-01-31T14:54:34.774878Z","iopub.status.idle":"2022-01-31T14:54:34.78353Z","shell.execute_reply.started":"2022-01-31T14:54:34.774825Z","shell.execute_reply":"2022-01-31T14:54:34.782798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_test_data_from_02_to_05_month(df, scaler=False):\n    # Get train, train_target, valid, valid_target in 02, 03, 04 months\n    \n    df_02_05 = df[df.ds.dt.month.isin([2, 3, 4])]\n    \n    # select 10% fraction in each year for test\n    df_02_05_test3 = df_02_05[df.ds.dt.year == 2020].sample(frac=0.1, random_state=seed).sort_values(by='ds')\n    df_02_05_test4 = df_02_05[df.ds.dt.year == 2021].sample(frac=0.1, random_state=seed).sort_values(by='ds')\n    \n    #crate train and test \n    df_02_05_test = pd.concat([df_02_05_test3, df_02_05_test4])\n\n    df_02_05_train = df_02_05[ ~df_02_05.isin(df_02_05_test)].dropna()\n    df_02_05_train = df_02_05_train.reset_index(drop=True)\n     \n    def get_X_y(df):\n        # Get dateframe, it's target from 'Target' and drop 'ds'\n        y = df.pop('Target')\n#         X = df.drop(columns=['ds', 'timestamp']).reset_index(drop=True)        \n        X = df.drop(columns=['ds']).reset_index(drop=True)        \n\n        return X, y\n\n    # Get train/valid datasets\n    train, train_target = get_X_y(df_02_05_train)\n    valid, valid_target = get_X_y(df_02_05_test)\n    print(f'Length of the train dataset - {len(train)}, length of the valid dataset - {len(valid)}')\n        \n    display(train[:3])\n    display(valid[:3])\n    print('Nulls train: ', train.isnull().sum().sum(), 'Nulls test: ', valid.isnull().sum().sum())\n    \n    # Standartization\n    if scaler:\n        scaler = StandardScaler().fit(train)\n        train = pd.DataFrame(scaler.transform(train), columns = train.columns)\n        valid = pd.DataFrame(scaler.transform(valid), columns = valid.columns)\n    \n    return train, train_target, valid, valid_target","metadata":{"papermill":{"duration":0.024354,"end_time":"2022-01-21T18:19:26.718376","exception":false,"start_time":"2022-01-21T18:19:26.694022","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-31T14:54:34.784884Z","iopub.execute_input":"2022-01-31T14:54:34.785559Z","iopub.status.idle":"2022-01-31T14:54:34.796942Z","shell.execute_reply.started":"2022-01-31T14:54:34.785522Z","shell.execute_reply":"2022-01-31T14:54:34.796267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Model training and prediction <a class=\"anchor\" id=\"4\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.015273,"end_time":"2022-01-21T18:19:26.749247","exception":false,"start_time":"2022-01-21T18:19:26.733974","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def model_training(model_type, train, train_target):\n    # Model training\n        \n    def hyperopt_model_score(params):\n        # Get the modeling score\n        \n        if model_type=='XGBRegressor':\n            clf = XGBRegressor(**params)\n        elif model_type=='LGBMRegressor':\n            clf = LGBMRegressor(**params)\n        elif model_type == 'RandomForestRegressor':\n            clf = RandomForestRegressor(**params)\n        elif model_type == 'CatBoostRegressor':\n            clf = CatBoostRegressor(**params)\n            \n        current_score = cross_val_score(clf, train, train_target, cv=cv_number).mean()\n        print(current_score, params)\n        return current_score \n\n    \n    if model_type=='XGBRegressor':\n        space_par = {\n                    'learning_rate': hp.quniform('learning_rate', 0.01, 0.5, 0.0001),\n                    'n_estimators': hp.choice('n_estimators', range(500, 1200)),\n                    'max_depth':  hp.choice('max_depth', np.arange(5, 8, dtype=int)),\n                    'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.005),\n#                     'tree_method': 'gpu_hist'\n                    }\n\n    elif model_type=='LGBMRegressor':\n        space_par = {\n                     'learning_rate': hp.quniform('learning_rate', 0.1, 0.15, 0.01),\n                     'n_estimators': hp.choice('n_estimators', range(1000, 3000)),\n                     'max_depth':  hp.choice('max_depth', np.arange(8, 13,  dtype=int)),\n                     'num_leaves': hp.choice('num_leaves', np.arange(300, 800, dtype=int)),\n#                      'colsample_bytree': hp.quniform('colsample_bytree', 0.1, 1, 0.1),\n                     'objective': 'regression',\n                     'device': 'gpu'\n                    }\n    \n    elif model_type == 'CatBoostRegressor':\n        space_par = {\n                     'learning_rate': hp.quniform('learning_rate', 0.1, 0.3, 0.01),\n                     'iterations': hp.choice('iterations', range(3000, 8000)),\n                     'max_depth':  hp.choice('max_depth', np.arange(9,13, dtype=int)),\n                     #'num_leaves': hp.choice('num_leaves', np.arange(300, 1000, dtype=int)),\n                     'l2_leaf_reg': hp.quniform('l2_leaf_reg', 0.1, 0.5, 0.05),                     \n                     'task_type': 'GPU',\n                     'verbose': 0,                                \n                    }    \n    best = fmin(fn=hyperopt_model_score, space=space_par, algo=tpe.suggest, max_evals=3)\n    print('best:\\n', best)\n    params = space_eval(space_par, best)\n    print('Model parameters:\\n',params)\n        \n    return params","metadata":{"papermill":{"duration":0.022819,"end_time":"2022-01-21T18:19:26.787517","exception":false,"start_time":"2022-01-21T18:19:26.764698","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-31T14:54:34.798461Z","iopub.execute_input":"2022-01-31T14:54:34.799105Z","iopub.status.idle":"2022-01-31T14:54:34.81396Z","shell.execute_reply.started":"2022-01-31T14:54:34.799068Z","shell.execute_reply":"2022-01-31T14:54:34.813186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_fit(model_type, train, train_target, valid, valid_target, res, w):\n    # Prediction by the model with optimal parameters\n\n    params = model_training(model_type, train, train_target)\n    model = LGBMRegressor(**params)        \n#     model = CatBoostRegressor(**params)        \n\n    model.fit(train, train_target)\n    y_pred = model.predict(valid)\n    res.loc[w, 'r2_score'] = round(r2_score(valid_target, y_pred),4)\n    res.loc[w, 'corr'] = round(np.corrcoef(valid_target, y_pred)[0][1],4)\n    res.loc[w, 'mae'] = round(mean_absolute_error(valid_target, y_pred),4)\n    res.loc[w, 'mse'] = round(mean_squared_error(valid_target, y_pred),4)\n    res.loc[w, 'rmse'] = round(mean_squared_error(valid_target, y_pred, squared=False),4)\n    res.loc[w, 'cos_dist'] = round(1 - spatial.distance.cosine(valid_target, y_pred),4)\n    \n    return res, model","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:34.814839Z","iopub.execute_input":"2022-01-31T14:54:34.815056Z","iopub.status.idle":"2022-01-31T14:54:34.82683Z","shell.execute_reply.started":"2022-01-31T14:54:34.815032Z","shell.execute_reply":"2022-01-31T14:54:34.826173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plotImp(model, name_model, X, acc, num = 100, fig_size = (50, 70)):\n    # Draw FI diagram\n    # Thanks to https://stackoverflow.com/questions/53413701/feature-importance-using-lightgbm\n    \n    feature_imp = pd.DataFrame({'Feature':X.columns, 'Value':model.feature_importances_})\n    plt.figure(figsize=fig_size)\n    sns.set(font_scale = 5)\n    sns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", \n                                                        ascending=False)[0:num])\n    plt.title(f'{name_model} Features Diagram')\n    plt.tight_layout()\n    feature_imp.sort_values(by=\"Value\", ascending=False).to_csv(f'res-FE-{Asset_ID}-{name_model}-acc-{acc}-{num_features}-features.csv', index=False)\n    plt.show()    ","metadata":{"execution":{"iopub.status.busy":"2022-01-31T14:54:34.829607Z","iopub.execute_input":"2022-01-31T14:54:34.830242Z","iopub.status.idle":"2022-01-31T14:54:34.838179Z","shell.execute_reply.started":"2022-01-31T14:54:34.830204Z","shell.execute_reply":"2022-01-31T14:54:34.837428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Get train and valid datasets\ntrain, train_target, valid, valid_target = get_train_test_data_from_02_to_05_month(df_train, scaler=False)\n\n# Results of the models training\nres = pd.DataFrame(columns = ['asset_name', 'num_train', 'num_valid', 'r2_score', 'mae', 'mse', 'rmse', 'corr', 'cos_dist'])\n\n# Training model\n#name_model = models[i]\nname_model = 'LGBMRegressor'\n# name_model = 'CatBoostRegressor'\nres, model =  model_fit(name_model, train, train_target, valid, valid_target, res, 0)\n\n# Draw FI diagram\nplotImp(model, name_model, train, float(res.loc[0,'cos_dist']))\n\nfeature_imp = pd.DataFrame({'Feature':train.columns, 'Value':model.feature_importances_}).sort_values(by='Value', ascending=False)\ndisplay(feature_imp)\n","metadata":{"papermill":{"duration":4512.585801,"end_time":"2022-01-21T19:34:39.390411","exception":false,"start_time":"2022-01-21T18:19:26.80461","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-31T14:54:34.839594Z","iopub.execute_input":"2022-01-31T14:54:34.840141Z","iopub.status.idle":"2022-01-31T15:16:55.468391Z","shell.execute_reply.started":"2022-01-31T14:54:34.840104Z","shell.execute_reply":"2022-01-31T15:16:55.467707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = feature_imp[feature_imp.Value > 0].Feature[:100].to_list()\nprint(f'Selected features: {selected_features}')\n    \nres['asset_name'] = asset_name\nres['num_train'] = len(train)\nres['num_valid'] = len(valid)\nres.sort_values(by='r2_score', ascending=False).to_csv(f'res-metric-{Asset_ID}-{name_model}-score-{res.r2_score[0]}-{num_features}-features.csv', index=False)\nres","metadata":{"execution":{"iopub.status.busy":"2022-01-31T15:16:55.469442Z","iopub.execute_input":"2022-01-31T15:16:55.471011Z","iopub.status.idle":"2022-01-31T15:16:55.491217Z","shell.execute_reply.started":"2022-01-31T15:16:55.470969Z","shell.execute_reply":"2022-01-31T15:16:55.489915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T15:16:55.492184Z","iopub.status.idle":"2022-01-31T15:16:55.493004Z","shell.execute_reply.started":"2022-01-31T15:16:55.492749Z","shell.execute_reply":"2022-01-31T15:16:55.492775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res","metadata":{"execution":{"iopub.status.busy":"2022-01-31T15:16:55.494195Z","iopub.status.idle":"2022-01-31T15:16:55.494973Z","shell.execute_reply.started":"2022-01-31T15:16:55.494717Z","shell.execute_reply":"2022-01-31T15:16:55.494743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = feature_imp[feature_imp.Value > 0].Feature[:100].to_list()\nprint(f'Selected features: {selected_features}')","metadata":{"execution":{"iopub.status.busy":"2022-01-31T15:16:55.496121Z","iopub.status.idle":"2022-01-31T15:16:55.496884Z","shell.execute_reply.started":"2022-01-31T15:16:55.496621Z","shell.execute_reply":"2022-01-31T15:16:55.496649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" = [Go to Top](#0)","metadata":{"papermill":{"duration":0.022857,"end_time":"2022-01-21T19:34:40.431357","exception":false,"start_time":"2022-01-21T19:34:40.4085","status":"completed"},"tags":[]}}]}