{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Thanks to all kagglers and host team. I learned a lot from this competition and it also sparked my interest in Cryptocurrency.  \nHere's my solution (inference part only).  \n\nSee you in JPX!!","metadata":{}},{"cell_type":"markdown","source":"# Summary","metadata":{}},{"cell_type":"markdown","source":"Predictions are made by ensembles of two LGBM models.\n* Model 1\n  * Learning in each asset.\n  * 20 technical indicators and 2 time-related features (22 features in total).\n* Model 2\n  * Learning in all asset data.\n  * \"Asset_ID\" is added to the features in Model 1.","metadata":{}},{"cell_type":"markdown","source":"Reference: https://www.kaggle.com/julian3833/g-research-starter-lgbm-pipeline/notebook","metadata":{}},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport gresearch_crypto\nimport datetime as dt\nimport gc\nfrom pathlib import Path\nimport pickle\n\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\nSUP_TRAIN = \"../input/g-research-crypto-forecasting/supplemental_train.csv\"\nPROC = Path(\"../input/20220129-process-all-data\")\nMODEL = Path(\"../input/20220129-final-model\")\nMODEL_ALL_ASSET = Path(\"../input/20220131-final-model-one-model\")\n\nSEED = 1\n\nREMOVE_LB_TEST_OVERLAPPING_DATA = True","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-05T07:37:04.321934Z","iopub.execute_input":"2022-05-05T07:37:04.322326Z","iopub.status.idle":"2022-05-05T07:37:06.062442Z","shell.execute_reply.started":"2022-05-05T07:37:04.322212Z","shell.execute_reply":"2022-05-05T07:37:06.06007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    \nfix_all_seeds(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:06.063794Z","iopub.execute_input":"2022-05-05T07:37:06.064447Z","iopub.status.idle":"2022-05-05T07:37:06.070905Z","shell.execute_reply.started":"2022-05-05T07:37:06.064398Z","shell.execute_reply":"2022-05-05T07:37:06.069943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_pickle(PROC / \"train.pkl\")\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:06.072843Z","iopub.execute_input":"2022-05-05T07:37:06.073303Z","iopub.status.idle":"2022-05-05T07:37:07.765725Z","shell.execute_reply.started":"2022-05-05T07:37:06.07326Z","shell.execute_reply":"2022-05-05T07:37:07.764816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sup_train = pd.read_csv(SUP_TRAIN)\nsup_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:07.768523Z","iopub.execute_input":"2022-05-05T07:37:07.769457Z","iopub.status.idle":"2022-05-05T07:37:11.899705Z","shell.execute_reply.started":"2022-05-05T07:37:07.769406Z","shell.execute_reply":"2022-05-05T07:37:11.898592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.concat((df_train, sup_train))\ndf_train['datetime'] = pd.to_datetime(df_train['timestamp'], unit='s')\ndf_train = df_train.set_index(\"datetime\")\n\ndel sup_train\ngc.collect()\n\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:11.90105Z","iopub.execute_input":"2022-05-05T07:37:11.901297Z","iopub.status.idle":"2022-05-05T07:37:19.542986Z","shell.execute_reply.started":"2022-05-05T07:37:11.901266Z","shell.execute_reply":"2022-05-05T07:37:19.542412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:19.544012Z","iopub.execute_input":"2022-05-05T07:37:19.544717Z","iopub.status.idle":"2022-05-05T07:37:19.56072Z","shell.execute_reply.started":"2022-05-05T07:37:19.54468Z","shell.execute_reply":"2022-05-05T07:37:19.559622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details_asset = df_asset_details.set_index(\"Asset_ID\")\nweight = df_asset_details_asset[\"Weight\"].to_dict()\nweight","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:19.562321Z","iopub.execute_input":"2022-05-05T07:37:19.562678Z","iopub.status.idle":"2022-05-05T07:37:19.571028Z","shell.execute_reply.started":"2022-05-05T07:37:19.562617Z","shell.execute_reply":"2022-05-05T07:37:19.570353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Extraction","metadata":{"execution":{"iopub.status.busy":"2022-01-22T00:50:14.398033Z","iopub.execute_input":"2022-01-22T00:50:14.398342Z","iopub.status.idle":"2022-01-22T00:50:14.403267Z","shell.execute_reply.started":"2022-01-22T00:50:14.398305Z","shell.execute_reply":"2022-01-22T00:50:14.402569Z"}}},{"cell_type":"code","source":"def calc_sma_diff_test(close, timeperiod_short, timeperiod_long):\n    res_short = close[-timeperiod_short:].mean()\n    res_long = close[-timeperiod_long:].mean()\n    res = (res_long - res_short) / res_long\n    return res\n\n# 計算早いけどちょっとだけ値がずれる。\ndef calc_bbands_test(close, timeperiod, std=2):\n    close_std = close[-timeperiod:].std(ddof=0)\n    rol = close[-timeperiod:].mean()\n    upper = rol + close_std * std\n    lower = rol - close_std * std\n    res = (upper - close[-1]) / (upper - lower)\n    return res\n\n# # これ使うならcloseをSeriesで渡す必要あり\n# # def calc_bbands_test(close, timeperiod, std=2):\n# #     close_std = close[-timeperiod:].rolling(timeperiod).std(ddof=0)[-1]\n# #     rol = close[-timeperiod:].rolling(timeperiod).mean()[-1]\n# #     upper = rol + close_std * std\n# #     lower = rol - close_std * std\n# #     res = (upper - close[-1]) / (upper - lower)\n# #     return res\n\ndef calc_rsi_test(close, timeperiod):\n    # https://kabu.com/investment/guide/technical/08.html\n    diff = np.diff(close[-(timeperiod+1):])\n    A = diff[diff>0].sum() / timeperiod\n    B = -diff[diff<0].sum() / timeperiod\n    rsi = 100 * A / (A + B)\n#     rsi = 100 - (100 / (1 + A/B))\n    return rsi\n\ndef calc_atr_test(high, low, close, timeperiod):\n    A = high[-timeperiod:] - close[-(timeperiod+1):-1]\n    B = close[-(timeperiod+1):-1] - low[-timeperiod:]\n    C = high[-timeperiod:] - low[-timeperiod:]\n    res = np.vstack((A, B, C)).max(axis=0).mean()\n    return res\n\ndef calc_natr_test(high, low, close, timeperiod):\n    res = calc_atr_test(high, low, close, timeperiod) / close[-1]\n    return res\n\ndef calc_minus_di_test(high, low, close, timeperiod):\n    high_diff = np.diff(high[-(timeperiod+1):])\n    low_diff = np.diff(low[-(timeperiod+1):])\n    high_diff[(high_diff<0)] = 0\n    low_diff[(low_diff<0)] = 0\n#     high_diff[(high_diff<low_diff)] = 0\n    low_diff[(high_diff>low_diff)] = 0\n    tr = calc_atr_test(high, low, close, timeperiod)*timeperiod\n    res = 100 * low_diff.sum() / tr\n    return res\n\ndef calc_plus_di_test(high, low, close, timeperiod):\n    high_diff = np.diff(high[-(timeperiod+1):])\n    low_diff = np.diff(low[-(timeperiod+1):])\n    high_diff[(high_diff<0)] = 0\n    low_diff[(low_diff<0)] = 0\n    high_diff[(high_diff<low_diff)] = 0\n#     low_diff[(high_diff>low_diff)] = 0\n    tr = calc_atr_test(high, low, close, timeperiod)*timeperiod\n    res = 100 * high_diff.sum() / tr\n    return res\n\ndef calc_log_ret_test(close, lag=0):\n    return np.log(close[-(1+15*lag)] / close[-(1+15*(lag+1))])\n\ndef upper_shadow_15_perc_test(high, close, open_, lag=0):\n    if lag:\n        return high[-15*(lag+1):-15*lag].max() * 100 / np.maximum(close[-15*lag-1], open_[-15*(lag+1)-1])\n    else:\n        return high[-15:].max() * 100 / np.maximum(close[-1], open_[-16])\n\ndef lower_shadow_15_perc_test(low, close, open_, lag=0):\n    if lag:\n        return np.minimum(close[-15*lag-1], open_[-15*(lag+1)-1]) * 100 / low[-15*(lag+1):-15*lag].min()\n    else:\n        return np.minimum(close[-1], open_[-16]) * 100 / low[-15:].min()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:19.572472Z","iopub.execute_input":"2022-05-05T07:37:19.573353Z","iopub.status.idle":"2022-05-05T07:37:19.598665Z","shell.execute_reply.started":"2022-05-05T07:37:19.573307Z","shell.execute_reply":"2022-05-05T07:37:19.597668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_test(df, open_, high, low, close, volume):\n    ### original def features\n    res = []\n    res.append(calc_sma_diff_test(close, 12, 26))\n    res.append(calc_sma_diff_test(close, 12*4*4, 24*4*4))\n    res.append(calc_sma_diff_test(close, 12*4*4*4, 24*4*4*4))\n    res.append(calc_sma_diff_test(close, 12*4*4*4*4, 24*4*4*4*4))\n    res.append(calc_sma_diff_test(volume, 12*4*4, 24*4*4))\n    res.append(calc_rsi_test(close, 14*4))\n    res.append(calc_natr_test(high, low, close, 14*4*4*4))\n    res.append(calc_minus_di_test(high, low, close, 14))\n    res.append(calc_minus_di_test(high, low, close, 14*4*4*4))\n    res.append(calc_plus_di_test(high, low, close, 14*4))\n    res.append(calc_plus_di_test(high, low, close, 14*4*4*4))\n    res.append(calc_log_ret_test(close))\n    res.append(calc_log_ret_test(close, lag=1))\n    res.append(calc_log_ret_test(close, lag=2))\n    res.append(upper_shadow_15_perc_test(high, close, open_))\n#     df[\"upper_shadow_15_lag1\"] = upper_shadow_15_perc_test(high, close, open_, lag=1)\n#     df[\"upper_shadow_15_lag2\"] = upper_shadow_15_perc_test(high, close, open_, lag=2)\n    res.append(lower_shadow_15_perc_test(low, close, open_))\n#     df[\"lower_shadow_15_lag1\"] = lower_shadow_15_perc_test(low, close, open_, lag=1)\n#     df[\"lower_shadow_15_lag2\"] = lower_shadow_15_perc_test(low, close, open_, lag=2)\n    res.append(calc_bbands_test(close, 5*4*2))\n    res.append(calc_bbands_test(close, 5*4*4))\n    res.append(calc_bbands_test(close, 5*4*4*2))\n    res.append(calc_bbands_test(close, 5*4*4*4))\n\n    # time features\n    res.append(df[\"datetime\"].hour)\n    res.append(df[\"datetime\"].minute)\n#     df[\"day_of_week\"] = df[\"datetime\"].dayofweek\n    res.append(df[\"Asset_ID\"])\n\n    res = np.array(res).reshape([1, -1])\n    return res","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:19.59992Z","iopub.execute_input":"2022-05-05T07:37:19.600699Z","iopub.status.idle":"2022-05-05T07:37:19.619332Z","shell.execute_reply.started":"2022-05-05T07:37:19.600652Z","shell.execute_reply":"2022-05-05T07:37:19.618466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Models","metadata":{}},{"cell_type":"code","source":"def fillna_in_ohlcv(df_fillna):\n    df_fillna[[\"Open\", \"High\", \"Low\", \"Close\", \"Volume\"]] = df_fillna[[\"Open\", \"High\", \"Low\", \"Close\", \"Volume\"]].fillna(method=\"ffill\")\n    df_cl = df_fillna[[\"Close\"]]\n    all_times = pd.date_range(df_cl.index[0],\n                              df_cl.index[-1],\n                              freq=\"min\")\n    df_cl = df_cl.reindex(all_times, method=\"pad\")\n\n    df_fillna = df_fillna.drop([\"Close\"], axis=1)\n    df_fillna = df_cl.merge(df_fillna,\n                            left_index=True,\n                            right_index=True,\n                            how=\"left\")\n\n    df_fillna[[\"Count\", \"Volume\"]] = df_fillna[[\"Count\", \"Volume\"]].fillna(0)\n    df_fillna[\"Open\"] = df_fillna[\"Open\"].fillna(df_fillna[\"Close\"])\n    df_fillna[\"High\"] = df_fillna[\"High\"].fillna(df_fillna[\"Close\"])\n    df_fillna[\"Low\"] = df_fillna[\"Low\"].fillna(df_fillna[\"Close\"])\n#     df_fillna[\"VWAP\"] = df_fillna[\"VWAP\"].fillna(df_fillna[\"Close\"])    \n\n    return df_fillna","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:19.620829Z","iopub.execute_input":"2022-05-05T07:37:19.621137Z","iopub.status.idle":"2022-05-05T07:37:19.638966Z","shell.execute_reply.started":"2022-05-05T07:37:19.621097Z","shell.execute_reply":"2022-05-05T07:37:19.638179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model_for_asset(df_train, asset_id):\n    df_proc = df_train[df_train[\"Asset_ID\"] == asset_id]\n    df_proc = fillna_in_ohlcv(df_proc)\n    np_open = df_proc.Open.values[-(24*4*4*4*4+50):]\n    np_high = df_proc.High.values[-(24*4*4*4*4+50):]\n    np_low = df_proc.Low.values[-(24*4*4*4*4+50):]\n    np_close = df_proc.Close.values[-(24*4*4*4*4+50):]\n    np_volume = df_proc.Volume.values[-(24*4*4*4*4+50):]\n\n    model = pickle.load(open(MODEL / f'model_{asset_id}.sav', 'rb'))\n    return model, np_open, np_high, np_low, np_close, np_volume","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:19.640847Z","iopub.execute_input":"2022-05-05T07:37:19.641077Z","iopub.status.idle":"2022-05-05T07:37:19.657016Z","shell.execute_reply.started":"2022-05-05T07:37:19.641051Z","shell.execute_reply":"2022-05-05T07:37:19.656093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\nnps_open = {}\nnps_high = {}\nnps_low = {}\nnps_close = {}\nnps_volume = {}\nfor asset_id in weight.keys():\n    print(f\"Loading model for ID={asset_id:<2}\")\n    model, np_open, np_high, np_low, np_close, np_volume = get_model_for_asset(df_train, asset_id)\n    models[asset_id] = model\n    nps_open[asset_id] = np_open\n    nps_high[asset_id] = np_high\n    nps_low[asset_id] = np_low\n    nps_close[asset_id] = np_close\n    nps_volume[asset_id] = np_volume\n    \ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:19.658317Z","iopub.execute_input":"2022-05-05T07:37:19.658583Z","iopub.status.idle":"2022-05-05T07:37:39.784317Z","shell.execute_reply.started":"2022-05-05T07:37:19.658553Z","shell.execute_reply":"2022-05-05T07:37:39.783675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_all_asset = pickle.load(open(MODEL_ALL_ASSET / f'model_all_asset.sav', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:39.786606Z","iopub.execute_input":"2022-05-05T07:37:39.787279Z","iopub.status.idle":"2022-05-05T07:37:39.848297Z","shell.execute_reply.started":"2022-05-05T07:37:39.787244Z","shell.execute_reply":"2022-05-05T07:37:39.847447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"def renew_data(data, new_data):\n    last = data[-1]\n    res = np.append(data, new_data)[1:]\n    res = np.nan_to_num(res, nan=last, posinf=last, neginf=last)\n    return res","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:39.849577Z","iopub.execute_input":"2022-05-05T07:37:39.849824Z","iopub.status.idle":"2022-05-05T07:37:39.855246Z","shell.execute_reply.started":"2022-05-05T07:37:39.849794Z","shell.execute_reply":"2022-05-05T07:37:39.85459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_set = set(range(14))\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    df_test[\"datetime\"] = pd.to_datetime(df_test['timestamp'], unit='s')\n    for _, row in df_test.iterrows():\n        try:\n            asset_id = row[\"Asset_ID\"]\n            model = models[asset_id]\n            nps_open[asset_id] = renew_data(nps_open[asset_id], row[\"Open\"])\n            nps_high[asset_id] = renew_data(nps_high[asset_id], row[\"High\"])\n            nps_low[asset_id] = renew_data(nps_low[asset_id], row[\"Low\"])\n            nps_close[asset_id] = renew_data(nps_close[asset_id], row[\"Close\"])\n            nps_volume[asset_id] = renew_data(nps_volume[asset_id], row[\"Volume\"])\n            X_test = get_features_test(row,\n                                       nps_open[asset_id],\n                                       nps_high[asset_id],\n                                       nps_low[asset_id],\n                                       nps_close[asset_id],\n                                       nps_volume[asset_id])\n\n            y_pred_each = model.predict(X_test[:, :-1])\n            y_pred_all = model_all_asset.predict(X_test)\n            y_pred = (y_pred_each + y_pred_all) / 2\n            df_pred.loc[df_pred[\"row_id\"]==row[\"row_id\"], \"Target\"] = y_pred\n        except:\n            df_pred.loc[df_pred[\"row_id\"]==row[\"row_id\"], \"Target\"] = 0\n            \n    asset_in_df_test = set(tuple(df_test.Asset_ID))\n    check = asset_set - asset_in_df_test\n    if check:\n        for asset_id in check:\n            cl = nps_close[asset_id][-1]\n            nps_open[asset_id] = np.append(nps_open[asset_id], cl)[1:]\n            nps_high[asset_id] = np.append(nps_high[asset_id], cl)[1:]\n            nps_low[asset_id] = np.append(nps_low[asset_id], cl)[1:]\n            nps_close[asset_id] = np.append(nps_close[asset_id], cl)[1:]\n            nps_volume[asset_id] = np.append(nps_volume[asset_id], 0)[1:]\n            \n    df_pred['Target'] = df_pred['Target'].fillna(0)\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:37:39.856582Z","iopub.execute_input":"2022-05-05T07:37:39.857027Z","iopub.status.idle":"2022-05-05T07:37:40.281398Z","shell.execute_reply.started":"2022-05-05T07:37:39.856993Z","shell.execute_reply":"2022-05-05T07:37:40.280283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}