{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"data_train = '/kaggle/input/g-research-crypto-forecasting/train.csv'\ndata_asset_details = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\ndata_supplemental_train = '/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv'","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:48:51.4671Z","iopub.execute_input":"2022-01-05T09:48:51.467456Z","iopub.status.idle":"2022-01-05T09:48:51.471766Z","shell.execute_reply.started":"2022-01-05T09:48:51.467417Z","shell.execute_reply":"2022-01-05T09:48:51.470949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport time\nimport datetime\nimport plotly.graph_objects as go","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:48:51.47327Z","iopub.execute_input":"2022-01-05T09:48:51.473583Z","iopub.status.idle":"2022-01-05T09:48:51.483668Z","shell.execute_reply.started":"2022-01-05T09:48:51.473548Z","shell.execute_reply":"2022-01-05T09:48:51.482897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_train = pd.read_csv(data_train, \n                       dtype={'Asset_ID': 'int8', 'Count': 'int32', 'row_id': 'int32', 'Count': 'int32', \n                              'Open': 'float64', 'High': 'float64', 'Low': 'float64', 'Close': 'float64', \n                              'Volume': 'float64', 'VWAP': 'float64'\n                             }\n                      )\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:48:51.48519Z","iopub.execute_input":"2022-01-05T09:48:51.485478Z","iopub.status.idle":"2022-01-05T09:49:41.565604Z","shell.execute_reply.started":"2022-01-05T09:48:51.485438Z","shell.execute_reply":"2022-01-05T09:49:41.564773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:49:41.566911Z","iopub.execute_input":"2022-01-05T09:49:41.567151Z","iopub.status.idle":"2022-01-05T09:49:41.58865Z","shell.execute_reply.started":"2022-01-05T09:49:41.567118Z","shell.execute_reply":"2022-01-05T09:49:41.588037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_assets = pd.read_csv(data_asset_details).sort_values(by='Asset_ID')\ndf_assets.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:49:41.591007Z","iopub.execute_input":"2022-01-05T09:49:41.59132Z","iopub.status.idle":"2022-01-05T09:49:41.610498Z","shell.execute_reply.started":"2022-01-05T09:49:41.591285Z","shell.execute_reply":"2022-01-05T09:49:41.609681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(14):\n \n    dfcrop=df_train[df_train['Asset_ID']==i]\n    print('Percentage of values not nan',(1-(np.sum((dfcrop['Target'].isnull()).astype(int))/dfcrop.shape[0]))*100)\n  ","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:49:41.611687Z","iopub.execute_input":"2022-01-05T09:49:41.612018Z","iopub.status.idle":"2022-01-05T09:49:44.391243Z","shell.execute_reply.started":"2022-01-05T09:49:41.611979Z","shell.execute_reply":"2022-01-05T09:49:44.390489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\n\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    \n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    \n    df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n\n    df_feat[\"Median\"] = df_feat[[\"Open\", \"High\", \"Low\", \"Close\"]].median(axis=1)\n\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:49:44.392715Z","iopub.execute_input":"2022-01-05T09:49:44.392993Z","iopub.status.idle":"2022-01-05T09:49:44.469573Z","shell.execute_reply.started":"2022-01-05T09:49:44.392958Z","shell.execute_reply":"2022-01-05T09:49:44.468939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:49:44.470827Z","iopub.execute_input":"2022-01-05T09:49:44.471072Z","iopub.status.idle":"2022-01-05T09:49:44.475872Z","shell.execute_reply.started":"2022-01-05T09:49:44.471038Z","shell.execute_reply":"2022-01-05T09:49:44.474021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.drop(labels=np.where(np.isinf(df_proc))[0], axis=0)\n    df_proc = df_proc.dropna(how=\"any\")\n    #df_proc = df_proc.reset_index()\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    #scaler = StandardScaler()\n    #X = scaler.fit_transform(X)\n    \n    if asset_id == 0:\n        model = xgb.XGBRegressor(\n            n_estimators=317,\n            max_depth=8,\n            learning_rate= 0.008967159857886885,\n            subsample=0.8074685834714562,\n            colsample_bytree=0.6156249507619749,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 1:\n        model = xgb.XGBRegressor(\n            n_estimators=637,\n            max_depth=13,\n            learning_rate= 0.09253396014321574,\n            subsample=0.700624738784116,\n            colsample_bytree=0.73896289605807,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 2:\n        model = xgb.XGBRegressor(\n            n_estimators=452,\n            max_depth=8,\n            learning_rate=  0.4167642609563461,\n            subsample=0.582006239504628,\n            colsample_bytree=0.6829989973347863,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 3:\n        model = xgb.XGBRegressor(\n            n_estimators=452,\n            max_depth=8,\n            learning_rate= 0.4167642609563461,\n            subsample=0.5820062395046286,\n            colsample_bytree=0.6829989973347863,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 4:\n        model = xgb.XGBRegressor(\n            n_estimators=471,\n            max_depth=9,\n            learning_rate= 0.05918488024797159,\n            subsample=0.7693819367697938,\n            colsample_bytree=0.7266084230952958,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 5:\n        model = xgb.XGBRegressor(\n            n_estimators=452,\n            max_depth=8,\n            learning_rate= 0.4167642609563461,\n            subsample=0.582006239504628,\n            colsample_bytree=0.6829989973347863,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 6:\n        model = xgb.XGBRegressor(\n            n_estimators=476,\n            max_depth=8,\n            learning_rate= 0.4202769113980745,\n            subsample=0.5563315209270074,\n            colsample_bytree=0.6951993738259458,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 7:\n        model = xgb.XGBRegressor(\n            n_estimators=520,\n            max_depth=15,\n            learning_rate=0.19184853364231427,\n            subsample=0.8869731830313443,\n            colsample_bytree=0.6855158027999262,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 8:\n        model = xgb.XGBRegressor(\n            n_estimators=471,\n            max_depth=9,\n            learning_rate= 0.05918488024797159,\n            subsample=0.7693819367697938,\n            colsample_bytree=0.7266084230952958,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 9:\n        model = xgb.XGBRegressor(\n            n_estimators=229,\n            max_depth=9,\n            learning_rate= 0.23016519709096778,\n            subsample=0.7928998128269837,\n            colsample_bytree=0.5299924747454009,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 10:\n        model = xgb.XGBRegressor(\n            n_estimators=229,\n            max_depth=9,\n            learning_rate=0.23016519709096778,\n            subsample=0.7928998128269837,\n            colsample_bytree=0.5299924747454009,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 11:\n        model = xgb.XGBRegressor(\n            n_estimators=229,\n            max_depth=9,\n            learning_rate=0.23016519709096778,\n            subsample=0.7928998128269837,\n            colsample_bytree=0.5299924747454009,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 12:\n        model = xgb.XGBRegressor(\n            n_estimators=229,\n            max_depth=9,\n            learning_rate=0.23016519709096778,\n            subsample=0.7928998128269837,\n            colsample_bytree=0.5299924747454009,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    elif asset_id == 13:\n        model = xgb.XGBRegressor(\n            n_estimators=121,\n            max_depth=6,\n            learning_rate=0.3062149270836522,\n            subsample=0.7361485971162751,\n            colsample_bytree=0.685244452888315,\n            missing=-999,\n            random_state=2022,\n            tree_method='gpu_hist'\n        )\n    model.fit(X, y)\n    del X\n    del y\n    #return X, y, model\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:49:44.477344Z","iopub.execute_input":"2022-01-05T09:49:44.478163Z","iopub.status.idle":"2022-01-05T09:49:44.486598Z","shell.execute_reply.started":"2022-01-05T09:49:44.47813Z","shell.execute_reply":"2022-01-05T09:49:44.485912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset_with_reset_index(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.reset_index(drop=True)\n    df_proc = df_proc.drop(labels=np.where(np.isinf(df_proc))[0], axis=0)\n    df_proc = df_proc.dropna(how=\"any\")\n    df_proc = df_proc.reset_index(drop=True)\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    #scaler = StandardScaler()\n    #X = scaler.fit_transform(X)\n    \n    model = xgb.XGBRegressor(\n        n_estimators=229,\n        max_depth=9,\n        learning_rate=0.23016519709096778,\n        subsample=0.7928998128269837,\n        colsample_bytree=0.5299924747454009,\n        missing=-999,\n        random_state=2022,\n        tree_method='gpu_hist'\n    )\n    model.fit(X, y)\n    del X\n    del y\n    #return X, y, model\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-05T10:02:12.70999Z","iopub.execute_input":"2022-01-05T10:02:12.710637Z","iopub.status.idle":"2022-01-05T10:02:12.724687Z","shell.execute_reply.started":"2022-01-05T10:02:12.710602Z","shell.execute_reply":"2022-01-05T10:02:12.723859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_assets['Asset_ID'], df_assets['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    try:\n        if asset_id == 10:\n            models[asset_id] = get_Xy_and_model_for_asset_with_reset_index(df_train, asset_id)\n            continue\n        #X, y, model = get_Xy_and_model_for_asset(df_train, asset_id) \n        model = get_Xy_and_model_for_asset(df_train, asset_id)\n        models[asset_id] = model\n        #Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model\n    except:         \n        models[asset_id] = None\n        #Xs[asset_id], ys[asset_id], models[asset_id] = None, None, None    ","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:49:45.820802Z","iopub.execute_input":"2022-01-05T09:49:45.821066Z","iopub.status.idle":"2022-01-05T09:49:46.108272Z","shell.execute_reply.started":"2022-01-05T09:49:45.821034Z","shell.execute_reply":"2022-01-05T09:49:46.107482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import traceback","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:52:08.174673Z","iopub.execute_input":"2022-01-05T09:52:08.17494Z","iopub.status.idle":"2022-01-05T09:52:08.179438Z","shell.execute_reply.started":"2022-01-05T09:52:08.174906Z","shell.execute_reply":"2022-01-05T09:52:08.17859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:52:08.18107Z","iopub.execute_input":"2022-01-05T09:52:08.181457Z","iopub.status.idle":"2022-01-05T09:52:08.195498Z","shell.execute_reply.started":"2022-01-05T09:52:08.181423Z","shell.execute_reply":"2022-01-05T09:52:08.194675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for asset_id, asset_name in zip(df_assets['Asset_ID'], df_assets['Asset_Name']):\n    print(f\"Feature importances for {asset_name:<16} (ID={asset_id:<2}) - {models[asset_id].feature_importances_}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n#scaler = StandardScaler()\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        if models[row['Asset_ID']] is not None:\n            try:\n                model = models[row['Asset_ID']]\n                x_test = get_features(pd.DataFrame([row]))\n                #x_test = scaler.fit_transform(x_test)\n                y_pred = model.predict(x_test)[0]\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n            except:\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n                traceback.print_exc()\n        else: \n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T09:52:08.19661Z","iopub.execute_input":"2022-01-05T09:52:08.196859Z","iopub.status.idle":"2022-01-05T09:52:08.301023Z","shell.execute_reply.started":"2022-01-05T09:52:08.196827Z","shell.execute_reply":"2022-01-05T09:52:08.30031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}