{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport pandas as pd\nimport numpy as np\nfrom datetime import datetime\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\nimport traceback\nimport time\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import RandomizedSearchCV\n#from sklearn.metrics import make_scorer\n#import seaborn as sns\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n\n\n### Loading datasets\npath = \"/kaggle/input/g-research-crypto-forecasting/\"\ndf_train = pd.read_csv(path + \"train.csv\")\ndf_test = pd.read_csv(path + \"example_test.csv\")\ndf_asset_details = pd.read_csv(path + \"asset_details.csv\")\ndf_supp_train = pd.read_csv(path + \"supplemental_train.csv\")\n\n\ndf_asset_details.sort_values([\"Asset_ID\"])\n\n#supplemental_train.csv is the same as above (train.csv) except contains less rows.\n\ntraindtype={'timestamp': 'int32', 'Asset_ID': 'int8', 'Count': 'int16', 'Open': 'float16', 'High': 'float16', 'Low': 'float16', 'Close': 'float16',\n       'Volume': 'float32', 'VWAP': 'float16','Target': 'float16'}\n\n\n\ndf_trainFull = pd.concat([df_train,df_supp_train])   ## to train the model completely, combine with moving average feature\n#df_train = df_train.astype(traindtype)\ndf_train = df_trainFull.astype(traindtype)\n\n############################################################################################################################\n# Remove the future\nREMOVE_LB_TEST_OVERLAPPING_DATA = False #False\n\nif REMOVE_LB_TEST_OVERLAPPING_DATA:\n    df_train['datetime'] = pd.to_datetime(df_train['timestamp'], unit='s')\n    df_train = df_train[df_train['datetime'] < '2021-06-13 00:00:00']\n    df_train.drop('datetime', inplace=True, axis=1)\n    \ndf_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n\ndf_train['VWAP'] = np.where(df_train.VWAP.isnull(), (df_train[\"Close\"]+df_train[\"Open\"])/2, df_train.VWAP)\n\ndf_train = df_train.dropna(how=\"any\", axis = 0) # remove rows\n############################################################################################################################\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-28T06:24:55.897318Z","iopub.execute_input":"2022-01-28T06:24:55.897651Z","iopub.status.idle":"2022-01-28T06:26:18.443189Z","shell.execute_reply.started":"2022-01-28T06:24:55.897571Z","shell.execute_reply":"2022-01-28T06:26:18.442199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\nfrom collections import deque\n\nclass RunningMean:\n    def __init__(self, WIN_SIZE=15, n_size = 1):\n        self.n = 0\n        self.mean = np.zeros(n_size)\n        self.cum_sum = 0\n        self.past_value = 0\n        self.WIN_SIZE = WIN_SIZE\n        self.windows = collections.deque(maxlen=WIN_SIZE+1)\n        \n    def clear(self):\n        self.n = 0\n        self.windows.clear()\n\n    def push(self, x):\n        \n        #currently fillna with past value, might want to change that\n        x = fillna_npwhere(x, self.past_value)\n        self.past_value = x\n        \n        self.windows.append(x)\n        self.cum_sum += x\n        \n        if self.n < self.WIN_SIZE:\n            self.n += 1\n            self.mean = self.cum_sum / float(self.n)\n            \n        else:\n            self.cum_sum -= self.windows.popleft()\n            self.mean = self.cum_sum / float(self.WIN_SIZE)\n\n    def get_mean(self):\n        return self.mean if self.n else np.zeros(n_size)\n\n    def __str__(self):\n        return \"Current window values: {}\".format(list(self.windows))\n\n######################################################\ndict_MAclass ={}    \nfor assetID in range(0,14):\n    dict_MAclass[assetID] = RunningMean(15)\n######################################################    \n    \n\ndef fillna_npwhere(array, values):\n    if np.isnan(array.sum()):\n        array = np.where(np.isnan(array), values, array)\n    return array","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def COHL(df): \n    rawCOHL = (df['Close']-df['Open'])/(df['High'] - df['Low'])  \n    rawCOHL.fillna(1.0, inplace = True)\n    return rawCOHL\n        \ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef GK_vol(df):\n    return ((1 / 2 * np.log(df['High']/df['Low']) ** 2 - (2 * np.log(2) - 1) * np.log(df['Close']/df['Open']) ** 2)) * 10000  # *10000 because too small value\n\ndef RS_vol(df):\n    return (np.log(df['High']/df['Close'])*np.log(df['High']/df['Open']) + np.log(df['Low']/df['Close'])*np.log(df['Low']/df['Open'])) * 10000\n\n    \ndef get_features(df):\n    ###  for getting features in training data\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['GK_vol'] = GK_vol(df_feat)\n    df_feat['RS_vol'] = RS_vol(df_feat)\n    df_feat['log_ret'] = df_feat['Close']/df_feat['Open']\n    df_feat['COHL_ratio'] = COHL(df_feat)\n    \n    df_feat['base'] = df_feat.Close\n    df_feat['Open'] = df_feat.Open/df_feat.Close\n    df_feat['High'] = df_feat.High/df_feat.Close\n    df_feat['Low'] = df_feat.Low/df_feat.Close\n    df_feat['VWAP'] = df_feat.VWAP/df_feat.Close\n    df_feat['VolumePerTrade'] = df_feat.Volume  / df_feat.Count\n    \n    \n    df_feat['pandas_SMA_15'] = df_feat.Close.rolling(window=15,min_periods=1).mean()   ### first 15 rows are simply aggregate mean, need to redefine a function (must first)\n    df_feat['SMA_15_percentDiff'] = (df_feat.Close - df_feat.pandas_SMA_15)/df_feat.pandas_SMA_15 * 100\n    \n    df_feat['Close'] = 1\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat) \n\n    df_feat.drop(['Close'], axis=1,inplace=True)\n    return df_feat\n\ndef get_features_iter(df,assetID):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['GK_vol'] = GK_vol(df_feat)\n    df_feat['RS_vol'] = RS_vol(df_feat)\n    df_feat['log_ret'] = df_feat['Close']/df_feat['Open']\n    df_feat['COHL_ratio'] = COHL(df_feat)\n    \n    df_feat['base'] = df_feat.Close\n    df_feat['Open'] = df_feat.Open/df_feat.Close\n    df_feat['High'] = df_feat.High/df_feat.Close\n    df_feat['Low'] = df_feat.Low/df_feat.Close\n    df_feat['VWAP'] = df_feat.VWAP/df_feat.Close\n    df_feat['VolumePerTrade'] = df_feat.Volume  / df_feat.Count\n    \n    \n    ################################################################################\n    dict_MAclass[assetID].push(df_feat.iloc[0]['Close'])\n    df_feat['pandas_SMA_15'] = dict_MAclass[assetID].get_mean()   ### first 15 rows are simply aggregate mean, need to redefine a function (must first)\n    ################################################################################\n    \n    df_feat['SMA_15_percentDiff'] = (df_feat.Close - df_feat.pandas_SMA_15)/df_feat.pandas_SMA_15 * 100\n    df_feat['Upper_Shadow'] = df_feat['High'] - np.maximum(1, df_feat['Open'])\n    df_feat['Lower_Shadow'] = np.minimum(1, df['Open']) - df_feat['Low'] \n\n    \n    df_feat.drop(['Close'], axis=1,inplace=True)\n\n    return df_feat","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"CPU\" # CPU\n\nmodels = {\n2: LGBMRegressor(learning_rate=0.02, max_depth=8, n_estimators=100, num_leaves=20, device = DEVICE), #0.06427949836194251\n0: LGBMRegressor(learning_rate=0.02, max_depth=2, n_estimators=100, num_leaves=15, device = DEVICE), #0.02419793625030214  \n1: LGBMRegressor(learning_rate=0.02, max_depth=2, n_estimators=100, num_leaves=15, device = DEVICE),#0.019377907309737395\n5: LGBMRegressor(learning_rate=0.02, max_depth=6, n_estimators=100, num_leaves=45, device = DEVICE), #0.031013536303656584\n7: LGBMRegressor(learning_rate=0.02, max_depth=2, n_estimators=100, num_leaves=15, device = DEVICE), #0.015706439422546833\n6: LGBMRegressor(learning_rate=0.02, max_depth=2, n_estimators=100, num_leaves=15, device = DEVICE), #0.02345612371174138    \n9: LGBMRegressor(learning_rate=0.02, max_depth=8, n_estimators=100, num_leaves=10, device = DEVICE), #0.02586388368632851    \n11: LGBMRegressor(learning_rate=0.02, max_depth=10, n_estimators=100,num_leaves=25, device = DEVICE), #0.016762287930919273     \n13: LGBMRegressor(learning_rate=0.02, max_depth=4, n_estimators=100,num_leaves=10, device = DEVICE), #0.026184514321744297    \n12: LGBMRegressor(learning_rate=0.02, max_depth=14, n_estimators=100,num_leaves=40, device = DEVICE), #0.017003778362314828\n3: LGBMRegressor(learning_rate=0.02, max_depth=6, n_estimators=100, num_leaves=5, device = DEVICE), #0.0019081766003313284\n8: LGBMRegressor(learning_rate=0.02, max_depth=4, n_estimators=100, num_leaves=25, device = DEVICE), #0.015322020443587976  \n10: LGBMRegressor(learning_rate=0.02, max_depth=18, n_estimators=100,num_leaves=45, device = DEVICE), #0.021674358244667458   \n4: LGBMRegressor(learning_rate=0.02, max_depth=8, n_estimators=100, num_leaves=40, device = DEVICE), #0.01948345322325262\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"############################################################################################################################\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    df_proc = get_features(df)\n    df_proc = df_proc.dropna(how=\"any\", axis = 0) # remove rows\n    df_proc['y'] = df['Target']\n\n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    model = models[asset_id] #,LGBMRegressor( boosting_type= 'gbdt', n_estimators = 2000, learning_rate = 0.05 ) #,device = \"GPU\" , metric='rmse dart\n      \n    return X, y, model\n       \n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nXs = {}\nys = {}\ntrainedModel = {}\n\n\n# asset_id = 1\n# asset_name = \"Bitcoin\"\n\n# print(f\"Trained model for {asset_name:<16} (ID={asset_id:<2})\")\n\n# X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)       \n# try:\n#     trainedModel[asset_id] = model.fit(X,y)\n# except: \n#     trainedModel[asset_id] =  None \n\n\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Trained model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)       \n    try:\n        trainedModel[asset_id] = model.fit(X,y)\n    except: \n        trainedModel[asset_id] =  None ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\n\n# class CFG:\n#     INPUT_DIR = '/kaggle/input/g-research-crypto-forecasting/'\n#     OUTPUT_DIR = './'\n\n# save model\n#joblib.dump(trainedModel, os.path.join('./', 'lgb_model_val.pkl'))\n#print('lgb model saved!')\n\n\nfilename = \"Completed_All_LGBMmodel_100trees.joblib\"\njoblib.dump(trainedModel, filename)\n\n# new_models = joblib.load(filename)","metadata":{},"execution_count":null,"outputs":[]}]}