{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --no-index --find-links ../input/talibbinary/talib_binary-0.4.19-cp37-cp37m-manylinux1_x86_64.whl talib-binary","metadata":{"execution":{"iopub.status.busy":"2022-01-24T23:56:21.367622Z","iopub.execute_input":"2022-01-24T23:56:21.368098Z","iopub.status.idle":"2022-01-24T23:56:31.68622Z","shell.execute_reply.started":"2022-01-24T23:56:21.368014Z","shell.execute_reply":"2022-01-24T23:56:31.68521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Very Fast and Accurate feature generator","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pickle\nimport lightgbm as lgb\nimport os\nfrom datetime import datetime\nimport talib\nfrom tqdm.notebook import tqdm\nimport time\nimport json\nEPS = 1e-18","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-24T23:56:31.68861Z","iopub.execute_input":"2022-01-24T23:56:31.688954Z","iopub.status.idle":"2022-01-24T23:56:33.597114Z","shell.execute_reply.started":"2022-01-24T23:56:31.688912Z","shell.execute_reply":"2022-01-24T23:56:33.59625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ASSET_INFO = pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\", index_col=\"Asset_ID\")\nASSET_ID_TO_NAME = ASSET_INFO[\"Asset_Name\"].to_dict()\nASSET_NAME_TO_ID = {v: k for k, v in ASSET_ID_TO_NAME.items()}\none_day = 60 * 24","metadata":{"execution":{"iopub.status.busy":"2022-01-24T23:56:33.598242Z","iopub.execute_input":"2022-01-24T23:56:33.598484Z","iopub.status.idle":"2022-01-24T23:56:33.620995Z","shell.execute_reply.started":"2022-01-24T23:56:33.598455Z","shell.execute_reply":"2022-01-24T23:56:33.620076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_feather(\"../input/filleddataset/train.feather\")\nasset_infos = pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\", index_col=\"Asset_Name\")\ndf = df.set_index([\"timestamp\"])\nasset_infos_dict = asset_infos[\"Asset_ID\"].to_dict()\nassets = list(asset_infos.index)\nasset_feature_df = {}","metadata":{"execution":{"iopub.status.busy":"2022-01-24T23:56:33.622348Z","iopub.execute_input":"2022-01-24T23:56:33.622596Z","iopub.status.idle":"2022-01-24T23:56:47.782638Z","shell.execute_reply.started":"2022-01-24T23:56:33.622567Z","shell.execute_reply":"2022-01-24T23:56:47.781761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_feature_name(func_name, meta):\n    if \"timeperiod\" in meta:\n        if isinstance(meta[\"timeperiod\"], str):\n            name = func_name + \"_\" + \"_\".join([str(i) for i in eval(meta[\"timeperiod\"])])\n        else:\n            name = func_name + \"_\" + str(meta[\"timeperiod\"])\n    else:\n        name = func_name\n    return name\n\ndef read_feature_config(symbol):\n    with open(os.path.join(\"../input/bestparameter\", \"best_period_{}.json\".format(symbol))) as f:\n        config = json.load(f)\n    new_config = {}\n    for func_name, setting in config.items():\n        setting[\"func_name\"] = func_name\n        feature_name = get_feature_name(func_name, setting)\n        new_config[feature_name] = setting\n    return new_config","metadata":{"execution":{"iopub.status.busy":"2022-01-24T23:56:47.785013Z","iopub.execute_input":"2022-01-24T23:56:47.785266Z","iopub.status.idle":"2022-01-24T23:56:47.79338Z","shell.execute_reply.started":"2022-01-24T23:56:47.785235Z","shell.execute_reply":"2022-01-24T23:56:47.792444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def RET(df, n):\n    return df['Close'].pct_change(n)\n\ndef RET_C(df, n):\n    return df['Close'].pct_change(n)\n\ndef RET_H(df, n):\n    return df['High'].pct_change(n)\n    \ndef RET_L(df, n):\n    return df['Low'].pct_change(n)\n    \ndef RET_O(df, n):\n    return df['Open'].pct_change(n)\n    \ndef RET_V(df, n):\n    return df['Volume'].pct_change(n)\n    \ndef RET_VWAP(df, n):\n    return df['VWAP'].pct_change(n)\n    \ndef RET_Cnt(df, n):\n    return df['Count'].pct_change(n)\n\ndef STD(df, n):\n    return df['Close'].pct_change(1).rolling(n).std()\n\ndef RET_STD(df, n):\n    return RET(df, n) * STD(df, n)\n\ndef RSI(df, n):\n    return talib.RSI(df['Close'], n)\n\ndef ATR(df, n):\n    return talib.ATR(df[\"High\"], df.Low, df.Close, n)\n\ndef MFI(df, n):\n    return talib.MFI(df['High'], df['Low'], df['Close'], df['Volume'], n)\n\ndef VOL(df, n):\n    ret = df['Close'].pct_change(1)\n    return np.sqrt((ret ** 2).rolling(n).mean())\n\ndef TRIX(df, n):\n    return talib.TRIX(df['Close'], n)\n\ndef MACD(df, fast, slow):\n    return talib.MACD(df.Close, fast, slow)[0]\n\ndef MACD_HIST(df, fast, slow):\n    return talib.MACD(df.Close, fast, slow)[2]\n\ndef DEMA(df, n1, n2):\n    return np.log(df['Close'].rolling(n1).mean() / (df['Close'].rolling(n2).mean() + EPS))\n\ndef EFFICIENCY(df, n):\n    speed = (df.Close - df.Close.shift(n))\n    volatility = (df.Close - df.Close.shift(1)).abs().rolling(n).sum()\n    return speed / (volatility+ EPS)\n\ndef EI(df, n):\n    neg = df.Close - df.Low.rolling(n).min()\n    high = df.High.rolling(n).max() - df.Close\n    ei = high / (neg + EPS)\n    ei = np.clip(ei, -100, 100)\n    return ei\n\ndef EVEMT_BIGGER_VOLUME(df):\n    return (df.Volume > df.Volume.shift(1)).astype(float)\n\ndef EVENT_MACROSS(df, fast, slow):\n    fast_s = df.Close.rolling(fast).mean()\n    slow_s = df.Close.rolling(slow).mean()\n    return (fast_s > slow_s).astype(float)\n\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']","metadata":{"execution":{"iopub.status.busy":"2022-01-24T23:56:47.794614Z","iopub.execute_input":"2022-01-24T23:56:47.794837Z","iopub.status.idle":"2022-01-24T23:56:47.818057Z","shell.execute_reply.started":"2022-01-24T23:56:47.79481Z","shell.execute_reply":"2022-01-24T23:56:47.817333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, row=False, config=None):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    d = {\"Count\": \"RET_Cnt\", \n         \"Open\": \"RET_O\", \n         \"Close\": \"RET_C\", \n         \"Volume\": \"RET_V\", \n         \"VWAP\": \"RET_VWAP\",\n         \"High\": \"RET_H\",\n         \"Cnt\": \"RET_Cnt\",\n         \"Low\": \"RET_L\",\n        }\n    if config is None:\n        config = {}\n    for col in ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']:\n        config[\"RET1_{}\".format(col)] = {\"func_name\": d[col], \"timeperiod\": 1}\n        config[\"RET15_{}\".format(col)] = {\"func_name\": d[col], \"timeperiod\": 15}\n    \n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    df_feat[\"MoM1\"] = df_feat[\"Close\"] - df_feat[\"Close\"].shift(1)\n    df_feat[\"MoM2\"] = df_feat[\"High\"] - df_feat[\"Close\"].shift(1)\n    df_feat[\"MoM3\"] = df_feat[\"High\"] - df_feat[\"Low\"].shift(1)\n    \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    if row:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    \n    df_feat['High_div_Mean'] = df_feat['High'] / df_feat['Mean']\n    df_feat['Low_div_Mean'] = df_feat['Low'] / df_feat['Mean']\n    df_feat['Volume_div_Count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n#         df_feat[\"dayofweek\"] = times.dayofweek \n#         df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n#         df_feat[\"dayofweek\"] = times.dt.dayofweek \n#         df_feat[\"day\"] = times.dt.day \n\n    # self-define feature engineering\n    if config is not None:\n        for feature_name, setting in config.items():\n            func_name = setting[\"func_name\"]\n            func = eval(func_name)\n            if \"timeperiod\" not in setting:\n                # no argument\n                feature: float = func(df_feat)\n            elif isinstance(setting.get(\"timeperiod\"), str):\n                args = eval(setting.get(\"timeperiod\"))\n                feature: float = func(df_feat, *args)\n            else:\n                feature: float = func(df_feat, setting.get(\"timeperiod\"))\n            df_feat[feature_name] = feature\n            \n    df_feat.pop(\"Close\")\n    df_feat.pop(\"High\")\n    df_feat.pop(\"Low\")\n    df_feat.pop(\"Open\")\n    df_feat.pop(\"VWAP\")\n    df_feat.pop(\"Volume\")\n    df_feat.pop(\"Mean\")\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-24T23:56:47.819385Z","iopub.execute_input":"2022-01-24T23:56:47.819903Z","iopub.status.idle":"2022-01-24T23:56:47.84152Z","shell.execute_reply.started":"2022-01-24T23:56:47.81986Z","shell.execute_reply":"2022-01-24T23:56:47.840637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def RET_stream(df, n):\n    return (df[\"C\"][-1] - df[\"C\"][-(1+n)]) / (df[\"C\"][-(1+n)] + EPS)\n    \ndef RET_H_stream(df, n):\n    return (df[\"H\"][-1] - df[\"H\"][-(1+n)]) / (df[\"H\"][-(1+n)] + EPS)\n    \ndef RET_C_stream(df, n):\n    return (df[\"C\"][-1] - df[\"C\"][-(1+n)]) / (df[\"C\"][-(1+n)] + EPS)\n    \ndef RET_L_stream(df, n):\n    return (df[\"L\"][-1] - df[\"L\"][-(1+n)]) / (df[\"L\"][-(1+n)] + EPS)\n    \ndef RET_O_stream(df, n):\n    return (df[\"O\"][-1] - df[\"O\"][-(1+n)]) / (df[\"O\"][-(1+n)] + EPS)\n    \ndef RET_V_stream(df, n):\n    return (df[\"V\"][-1] - df[\"V\"][-(1+n)]) / (df[\"V\"][-(1+n)] + EPS)\n    \ndef RET_VWAP_stream(df, n):\n    return (df[\"VWAP\"][-1] - df[\"VWAP\"][-(1+n)]) / (df[\"VWAP\"][-(1+n)] + EPS)\n    \ndef RET_Cnt_stream(df, n):\n    return (df[\"Cnt\"][-1] - df[\"Cnt\"][-(1+n)]) / (df[\"Cnt\"][-(1+n)] + EPS)\n    \ndef STD_stream(df, n):\n    ret1 = (df[\"C\"][1:] - df[\"C\"][:-1]) / df[\"C\"][:-1]\n    return np.nanstd(ret1[-n:], ddof=1)\n#     return pd.Series(df[\"C\"]).pct_change(1).rolling(n).std().iloc[-1]\n\ndef RET_STD_stream(df, n):\n    return RET_stream(df, n) * STD_stream(df, n)\n\ndef RSI_stream(df, n):\n    return talib.RSI(df['C'], n)[-1]\n\ndef ATR_stream(df, n):\n    return talib.ATR(df[\"H\"], df[\"L\"], df[\"C\"], n)[-1]\n\ndef MFI_stream(df, n):\n    return talib.MFI(df['H'], df['L'], df['C'], df['V'], n)[-1]\n\ndef VOL_stream(df, n):\n    ret1 = (df[\"C\"][1:] - df[\"C\"][:-1]) / (df[\"C\"][:-1] + EPS)\n    ret1 = ret1[-n:]\n    return np.sqrt(np.nanmean(ret1 ** 2))\n\ndef TRIX_stream(df, n):\n    return talib.TRIX(df['C'], n)[-1]\n\ndef MACD_stream(df, fast, slow):\n    return talib.MACD(df[\"C\"], fast, slow)[0][-1]\n\ndef MACD_HIST_stream(df, fast, slow):\n    return talib.MACD(df[\"C\"], fast, slow)[2][-1]\n\ndef DEMA_stream(df, n1, n2):\n    C1 = df['C'][-n1:]\n    C2 = df['C'][-n2:]\n    return np.log(np.nanmean(C1) / (np.nanmean(C2)) + EPS)\n\ndef EFFICIENCY_stream(df, n):\n    speed = df[\"C\"][-1] - df[\"C\"][-(1+n)]\n    volatility = np.nansum(np.abs(df[\"C\"][1:] - df[\"C\"][:-1])[-n:])\n    return speed / volatility\n\ndef EI_stream(df, n):\n    neg = df[\"C\"][-1] - df[\"L\"][-n:].min()\n    high = df[\"H\"][-n:].max() - df[\"C\"][-1]\n    ei = high / (neg + EPS)\n    ei = np.clip(ei, -100, 100)\n    return ei\n\ndef EVEMT_BIGGER_VOLUME_stream(df):\n    return float(df[\"V\"][-1] > df[\"V\"][-2])\n\ndef EVENT_MACROSS_stream(df, fast, slow):\n    fast_s = np.nanmean(df[\"C\"][-fast:])\n    slow_s = np.nanmean(df[\"C\"][-slow:])\n    return float(fast_s > slow_s)\n\ndef upper_shadow_stream(df) -> float:\n    return float(df['H'][-1] - np.maximum(df['C'][-1], df['O'][-1]))\n\ndef lower_shadow_stream(df) -> float:\n    return float(np.minimum(df['C'][-1], df['O'][-1]) - df['L'][-1])","metadata":{"execution":{"iopub.status.busy":"2022-01-24T23:56:47.843103Z","iopub.execute_input":"2022-01-24T23:56:47.84338Z","iopub.status.idle":"2022-01-24T23:56:47.877275Z","shell.execute_reply.started":"2022-01-24T23:56:47.843342Z","shell.execute_reply":"2022-01-24T23:56:47.876538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vecterized version\n# 所有的函数接受的feature都是一个的矩阵(T, K), T是历史数据，K是len(assets)\ndef RET_stream_vec(df, n):\n    # df: (T, K)\n    # return: (K,)\n    return (df[\"C\"][-1] - df[\"C\"][-(1+n)]) / (df[\"C\"][-(1+n)] + EPS)\n    \ndef RET_H_stream_vec(df, n):\n    return (df[\"H\"][-1] - df[\"H\"][-(1+n)]) / (df[\"H\"][-(1+n)] + EPS)\n    \ndef RET_C_stream_vec(df, n):\n    return (df[\"C\"][-1] - df[\"C\"][-(1+n)]) / (df[\"C\"][-(1+n)] + EPS)\n    \ndef RET_L_stream_vec(df, n):\n    return (df[\"L\"][-1] - df[\"L\"][-(1+n)]) / (df[\"L\"][-(1+n)] + EPS)\n    \ndef RET_O_stream_vec(df, n):\n    return (df[\"O\"][-1] - df[\"O\"][-(1+n)]) / (df[\"O\"][-(1+n)] + EPS)\n    \ndef RET_V_stream_vec(df, n):\n    return (df[\"V\"][-1] - df[\"V\"][-(1+n)]) / (df[\"V\"][-(1+n)] + EPS)\n    \ndef RET_VWAP_stream_vec(df, n):\n    return (df[\"VWAP\"][-1] - df[\"VWAP\"][-(1+n)]) / (df[\"VWAP\"][-(1+n)] + EPS)\n    \ndef RET_Cnt_stream_vec(df, n):\n    return (df[\"Cnt\"][-1] - df[\"Cnt\"][-(1+n)]) / (df[\"Cnt\"][-(1+n)] + EPS)\n    \ndef STD_stream_vec(df, n):\n    ret1 = (df[\"C\"][1:] - df[\"C\"][:-1]) / df[\"C\"][:-1]  # (T, K)\n    return np.nanstd(ret1[-n:], ddof=1, axis=0)\n\ndef RET_STD_stream_vec(df, n):\n    return RET_stream_vec(df, n) * STD_stream_vec(df, n)\n\ndef VOL_stream_vec(df, n):\n    ret1 = (df[\"C\"][1:] - df[\"C\"][:-1]) / (df[\"C\"][:-1] + EPS)\n    ret1 = ret1[-n:]\n    return np.sqrt(np.nanmean(ret1 ** 2, axis=0))\n\ndef DEMA_stream_vec(df, n1, n2):\n    C1 = df['C'][-n1:]\n    C2 = df['C'][-n2:]\n    return np.log(np.nanmean(C1, axis=0) / (np.nanmean(C2, axis=0)) + EPS)\n\ndef EFFICIENCY_stream_vec(df, n):\n    speed = df[\"C\"][-1] - df[\"C\"][-(1+n)]\n    volatility = np.nansum(np.abs(df[\"C\"][1:] - df[\"C\"][:-1])[-n:], axis=0)\n    return speed / volatility\n\ndef EI_stream_vec(df, n):\n    neg = df[\"C\"][-1] - df[\"L\"][-n:].min(axis=0)\n    high = df[\"H\"][-n:].max(axis=0) - df[\"C\"][-1]\n    ei = high / (neg + EPS)\n    ei = np.clip(ei, -100, 100)\n    return ei\n\ndef EVEMT_BIGGER_VOLUME_stream_vec(df):\n    return (df[\"V\"][-1] > df[\"V\"][-2]).astype(float)\n\ndef EVENT_MACROSS_stream_vec(df, fast, slow):\n    fast_s = np.nanmean(df[\"C\"][-fast:], axis=0)\n    slow_s = np.nanmean(df[\"C\"][-slow:], axis=0)\n    return (fast_s > slow_s).astype(float)\n\ndef upper_shadow_stream_vec(df) -> float:\n    return (df['H'][-1] - np.maximum(df['C'][-1], df['O'][-1])).astype(float)\n\ndef lower_shadow_stream_vec(df) -> float:\n    return (np.minimum(df['C'][-1], df['O'][-1]) - df['L'][-1]).astype(float)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T23:56:47.879099Z","iopub.execute_input":"2022-01-24T23:56:47.879444Z","iopub.status.idle":"2022-01-24T23:56:47.909977Z","shell.execute_reply.started":"2022-01-24T23:56:47.879399Z","shell.execute_reply":"2022-01-24T23:56:47.908987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_stream(array_dict, config=None) -> np.ndarray:\n    \"\"\"\n    array_dict: {Key: Value} mapping. Each Value with shape = (T, K), where K = len(assets)\n    \n    return:\n        features: {Key: Value} mapping. Each Value with shape = (K,)\n    \"\"\"\n    removed_features = [\"Mean\", 'day', 'High_div_Low', 'dayofweek', 'Close_div_Open', \"timestamp\"]\n    d = {\"Count\": \"RET_Cnt\", \n         \"Open\": \"RET_O\", \n         \"Close\": \"RET_C\", \n         \"Volume\": \"RET_V\", \n         \"VWAP\": \"RET_VWAP\",\n         \"High\": \"RET_H\",\n         \"Low\": \"RET_L\",\n        }\n    features = {}\n    if config is None:\n        config = {}\n    t = time.time()\n    for col in ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']:\n        config[\"RET1_{}\".format(col)] = {\"func_name\": d[col], \"timeperiod\": 1}\n        config[\"RET15_{}\".format(col)] = {\"func_name\": d[col], \"timeperiod\": 15}\n    features['Count'] = array_dict[\"Cnt\"][-1]\n    features['Upper_Shadow'] = upper_shadow_stream_vec(array_dict)\n    features['Lower_Shadow'] = lower_shadow_stream_vec(array_dict)\n    features[\"MoM1\"] = array_dict[\"C\"][-1] - array_dict[\"C\"][-2]\n    features[\"MoM2\"] = array_dict[\"H\"][-1] - array_dict[\"C\"][-2]\n    features[\"MoM3\"] = array_dict[\"H\"][-1] - array_dict[\"L\"][-2]\n    \n    features[\"Close_div_Open\"] = array_dict[\"C\"][-1] / (array_dict[\"O\"][-1] + EPS)\n    features[\"Close-Open\"] = array_dict[\"C\"][-1] - array_dict[\"O\"][-1]\n    features[\"High-Low\"] = array_dict[\"H\"][-1] - array_dict[\"L\"][-1]\n    features[\"High_div_Low\"] = array_dict[\"H\"][-1] / (array_dict[\"L\"][-1] + EPS)\n    features[\"Mean\"] = (array_dict[\"H\"][-1] + array_dict[\"L\"][-1] + array_dict[\"C\"][-1] + array_dict[\"O\"][-1]) / 4\n    features[\"High_div_Mean\"] = array_dict[\"H\"][-1] / (features[\"Mean\"] + EPS)\n    features[\"Low_div_Mean\"] = array_dict[\"L\"][-1] / (features[\"Mean\"] + EPS)\n    features[\"Volume_div_Count\"] = array_dict[\"V\"][-1] / (array_dict[\"Cnt\"][-1] + 1)\n    times = pd.to_datetime(array_dict[\"T\"][:, 0],unit=\"s\",infer_datetime_format=True)\n    \n    features[\"hour\"] = float(times.hour[-1])\n    features[\"dayofweek\"] = float(times.dayofweek[-1])\n    features[\"day\"] = float(times.day[-1])\n    t = time.time()\n    n_assets = array_dict[\"C\"].shape[1]\n    if config is not None:\n        for feature_name, setting in config.items():\n            t = time.time()\n            args = ()\n            if \"timeperiod\" not in setting:\n                # no argument\n                pass\n            elif isinstance(setting.get(\"timeperiod\"), str):\n                args = eval(setting.get(\"timeperiod\"))\n            else:\n                args = (setting.get(\"timeperiod\"), )\n            \n            func_name = setting[\"func_name\"] + \"_stream\"\n            vec_func_name = setting[\"func_name\"] + \"_stream_vec\"\n            if vec_func_name in globals():\n                func = eval(vec_func_name)\n                features[feature_name] = func(array_dict, *args)\n            else:\n                func = eval(func_name)\n                \n                features[feature_name] = np.zeros(n_assets)\n                # 对于没办法向量化的就还是用粗糙的办法\n                for i in range(n_assets):\n                    sub_array_dict = {k: v[:, i] for k, v in array_dict.items()}\n                    features[feature_name][i] = func(sub_array_dict, *args)\n            \n    for feature in removed_features:\n        if feature in features:\n            features.pop(feature)\n    for k, v in features.items():\n        if np.isscalar(features[k]):\n            features[k] = np.broadcast_to(features[k], n_assets)\n        else:\n            features[k] = np.ravel(features[k])\n#         assert len(features[k]) == n_assets, \"{}_{}\".format(len(features[k]), k)\n#         assert not np.isnan(features[k])\n    return features","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:12:00.810002Z","iopub.execute_input":"2022-01-25T00:12:00.81031Z","iopub.status.idle":"2022-01-25T00:12:01.104806Z","shell.execute_reply.started":"2022-01-25T00:12:00.81028Z","shell.execute_reply":"2022-01-25T00:12:01.103301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BarData:\n    def __init__(self, symbol_names, \n                 O: np.array, \n                 H: np.array, \n                 L: np.array, \n                 C: np.array, \n                 V: np.array, \n                 VWAP: np.array, \n                 Cnt: np.array,\n                 T: np.array):\n        symbol_names = sorted(symbol_names)\n#         assert len(O) == len(symbol_names)\n#         assert len(H) == len(symbol_names)\n#         assert len(L) == len(symbol_names)\n#         assert len(C) == len(symbol_names)\n#         assert len(VWAP) == len(symbol_names)\n#         assert len(V) == len(symbol_names)\n#         assert len(Cnt) == len(symbol_names)\n        self.O = O\n        self.H = H\n        self.L = L\n        self.C = C\n        self.V = V\n        self.VWAP = VWAP\n        self.Cnt = Cnt\n        self.T = T\n\nclass ArrayManager(object):\n    def __init__(self, size, asset_names, feature_config=None):\n        \"\"\"Constructor\"\"\"\n        self.count: int = 0\n        self.ndim = len(asset_names)\n        self.asset_names = asset_names\n        self.size: int = size\n        self.inited: bool = False\n        self.O: np.ndarray = np.zeros((size, self.ndim))\n        self.H: np.ndarray = np.zeros((size, self.ndim))\n        self.L: np.ndarray = np.zeros((size, self.ndim))\n        self.C: np.ndarray = np.zeros((size, self.ndim))\n        self.V: np.ndarray = np.zeros((size, self.ndim))\n        self.Cnt: np.ndarray = np.zeros((size, self.ndim))\n        self.VWAP: np.ndarray = np.zeros((size, self.ndim))\n        self.T: np.ndarray = np.zeros((size, self.ndim), dtype=np.int32)\n            \n        self.feature_config = feature_config\n    \n    def update_bar(self, bar: BarData, calculate_feature=True) -> None:\n        \"\"\"\n        Update new bar data into array manager.\n        \"\"\"\n        self.count += 1\n        if not self.inited and self.count >= self.size:\n            self.inited = True\n        self.O[:-1] = self.O[1:]\n        self.H[:-1] = self.H[1:]\n        self.L[:-1] = self.L[1:]\n        self.C[:-1] = self.C[1:]\n        self.V[:-1] = self.V[1:]\n        self.T[:-1] = self.T[1:]\n        self.VWAP[:-1] = self.VWAP[1:]\n        self.Cnt[:-1] = self.Cnt[1:]\n        \n        # forward fill nan automatically\n        self.O[-1] = np.where(np.isnan(bar.O), self.O[-1], bar.O)\n        self.H[-1] = np.where(np.isnan(bar.H), self.H[-1], bar.H)\n        self.L[-1] = np.where(np.isnan(bar.L), self.L[-1], bar.L)\n        self.C[-1] = np.where(np.isnan(bar.C), self.C[-1], bar.C)\n        self.V[-1] = np.where(np.isnan(bar.V), self.V[-1], bar.V)\n        self.VWAP[-1] = np.where(np.isnan(bar.VWAP), self.VWAP[-1], bar.VWAP)\n        self.Cnt[-1] = np.where(np.isnan(bar.Cnt), self.Cnt[-1], bar.Cnt)\n        self.T[-1] = np.where(np.isnan(bar.T), self.T[-1], bar.T)\n        \n        self.feature_dict = {}  # Dict[str, vector of size K]\n        if calculate_feature:\n            bar_data = {\"O\": self.O, \n                        \"C\": self.C,\n                        \"L\": self.L,\n                        \"H\": self.H,\n                        \"V\": self.V,\n                        \"T\": self.T,\n                        \"Cnt\": self.Cnt,\n                        \"VWAP\": self.VWAP,\n                       }\n            self.feature_dict = get_features_stream(bar_data, self.feature_config)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:06:13.770381Z","iopub.execute_input":"2022-01-25T00:06:13.770698Z","iopub.status.idle":"2022-01-25T00:06:13.797435Z","shell.execute_reply.started":"2022-01-25T00:06:13.770663Z","shell.execute_reply":"2022-01-25T00:06:13.796454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test base features","metadata":{}},{"cell_type":"code","source":"# test base features\nasset_names = [0] * len(ASSET_ID_TO_NAME)\nfor k, v in ASSET_ID_TO_NAME.items():\n    asset_names[k] = v\nstart_time = int(datetime(2021, 6, 1).timestamp())\nam = ArrayManager(one_day, asset_names)\nfor _ in tqdm(range(one_day * 4)):\n    data = df.loc[start_time].set_index(\"Asset_ID\").loc[asset_names]\n    bar = BarData(asset_names, \n        O=data.Open.values, \n        C=data.Close.values, \n        L=data.Low.values, \n        H=data.High.values, \n        VWAP=data.VWAP.values, \n        V=data.Volume.values,\n        T=np.array([start_time] * len(asset_names)).astype(int),\n        Cnt=data.Count.values)\n    am.update_bar(bar, False)\n    start_time += 60\n    \n\nprevision = 7\nprev_data = None\nfor i in range(am.T.shape[0]):\n    t = am.T[i][0]\n    data = df.loc[t].set_index(\"Asset_ID\").loc[asset_names]\n    if prev_data is not None:\n        data = data.fillna(prev_data)\n        if not np.any(np.isnan(prev_data)):\n            np.testing.assert_almost_equal(am.O[i], data.Open.values, decimal=prevision)\n            np.testing.assert_almost_equal(am.C[i], data.Close.values, decimal=prevision)\n            np.testing.assert_almost_equal(am.H[i], data.High.values, decimal=prevision)\n            np.testing.assert_almost_equal(am.V[i], data.Volume.values, decimal=prevision)\n            np.testing.assert_almost_equal(am.VWAP[i], data.VWAP.values, decimal=prevision)\n            np.testing.assert_almost_equal(am.L[i], data.Low.values, decimal=prevision)\n            np.testing.assert_almost_equal(am.Cnt[i], data.Count.values, decimal=prevision)\n    prev_data = data.copy()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:01:33.003663Z","iopub.execute_input":"2022-01-25T00:01:33.004543Z","iopub.status.idle":"2022-01-25T00:01:45.748743Z","shell.execute_reply.started":"2022-01-25T00:01:33.004494Z","shell.execute_reply":"2022-01-25T00:01:45.747891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test whether we can replicate the features","metadata":{}},{"cell_type":"code","source":"# we use only one asset\nASSET_NAME = \"Binance Coin\"\nconfig = read_feature_config(ASSET_NAME)\n\n# get offline features\nsub_df = df[df.Asset_ID == ASSET_NAME]\nsub_df = sub_df.sort_index()\nsub_df[\"timestamp\"] = sub_df.index\nsub_df = sub_df.fillna(method=\"ffill\").dropna()\nfeat_df = get_features(sub_df, config=config)\n\n# get online features\nfeat_df2 = []\nasset_names = [ASSET_NAME]\nstart_time = int(datetime(2020, 8, 1).timestamp())\none_day = 60 * 24\nam = ArrayManager(one_day, asset_names, config)\ncount = 0\navg_time = 0\nfor _ in tqdm(range(one_day * 4)):\n    data = df.loc[start_time].set_index(\"Asset_ID\").loc[asset_names]\n    start = time.time()\n    bar = BarData(asset_names, \n        O=data.Open.values, \n        C=data.Close.values, \n        L=data.Low.values, \n        H=data.High.values, \n        VWAP=data.VWAP.values, \n        V=data.Volume.values,\n        T=np.array([start_time] * len(asset_names)).astype(int),\n        Cnt=data.Count.values)\n    am.update_bar(bar)\n    end = time.time()\n    feature = am.feature_dict\n    feature[\"timestamp\"] = np.broadcast_to(start_time, len(asset_names))\n    feature = {k: v[0] for k, v in feature.items()}\n    feat_df2.append(feature)\n    start_time += 60\n    count += 1\n    avg_time += (end - start)\n    \n# test\nfeat_df2 = pd.DataFrame(feat_df2).set_index(\"timestamp\")\nfeat_df2 = feat_df2.sort_index()\nfeat_df2 = feat_df2.iloc[-one_day:]  # we only test the latest one day\nfeat_df = feat_df.reindex(index=feat_df2.index)\n\n# 1) check column diff\nprint(\"more feature than needed\")\nprint(set(feat_df2.columns) - set(feat_df.columns))\nprint(\"insufficient features\")\nprint(set(feat_df.columns) - set(feat_df2.columns))\n\n# 2) check value diff\nfor name in feature.keys():\n    if name in feat_df.columns:\n        try:\n            np.testing.assert_almost_equal(feat_df[name].values, feat_df2[name].values)\n        except AssertionError as e:\n            print(\"****** error for name {}\".format(name))\n            print(str(e))\n    else:\n        print(\"skip \", name)\n        \nprint(\"time cost = \", str(avg_time / count))","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:17:59.630214Z","iopub.execute_input":"2022-01-25T00:17:59.631033Z","iopub.status.idle":"2022-01-25T00:18:36.036586Z","shell.execute_reply.started":"2022-01-25T00:17:59.630993Z","shell.execute_reply":"2022-01-25T00:18:36.035649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## StartFromHere: Submitting","metadata":{}},{"cell_type":"code","source":"all_configs = {}\nfeatures_by_asset = {}\nfor asset_name in assets:\n    config = read_feature_config(asset_name)\n    path = os.path.join(\"../input/febaselinev2result/ckpt/\", asset_name, \"4\", \"used_features.pickle\")\n    feat_names = pickle.load(open(path, \"rb\"))\n    assert feat_names == sorted(feat_names)\n    features_by_asset[asset_name] = feat_names\n    for feature_name, value in config.items():\n        all_configs[feature_name] = value","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:19:11.172221Z","iopub.execute_input":"2022-01-25T00:19:11.172498Z","iopub.status.idle":"2022-01-25T00:19:11.408106Z","shell.execute_reply.started":"2022-01-25T00:19:11.172471Z","shell.execute_reply":"2022-01-25T00:19:11.40721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"supplement_train = pd.read_csv(\"../input/g-research-crypto-forecasting/supplemental_train.csv\")\nsupplement_train = supplement_train.set_index(\"timestamp\")","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:19:11.655518Z","iopub.execute_input":"2022-01-25T00:19:11.656142Z","iopub.status.idle":"2022-01-25T00:19:17.630675Z","shell.execute_reply.started":"2022-01-25T00:19:11.656086Z","shell.execute_reply":"2022-01-25T00:19:17.629966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_names = list(range(14))","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:19:17.632498Z","iopub.execute_input":"2022-01-25T00:19:17.632949Z","iopub.status.idle":"2022-01-25T00:19:17.636934Z","shell.execute_reply.started":"2022-01-25T00:19:17.632908Z","shell.execute_reply":"2022-01-25T00:19:17.636175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# initialize the ArrayManager using supplemental data\nend_time = supplement_train.index.unique()[-1]\nam = ArrayManager(one_day, asset_names, all_configs)\nprev_data = None\ninit_days = 7\nstart_time = end_time - init_days * 24 * 60 * 60\nfor t in tqdm(range(start_time, end_time+60, 60)):\n    if t in supplement_train.index:\n        # 强制按照asset_names来排序\n        data = supplement_train.loc[t].set_index(\"Asset_ID\").reindex(index=asset_names).loc[asset_names]\n    elif prev_data is not None:\n        data = prev_data\n    else:\n        print(\"no data at {}\".format(t))\n        continue\n    bar = BarData(asset_names, \n            O=data.Open.values, \n            C=data.Close.values, \n            L=data.Low.values, \n            H=data.High.values, \n            VWAP=data.VWAP.values, \n            V=data.Volume.values,\n            T=np.array([t] * len(asset_names)).astype(int),\n            Cnt=data.Count.values)\n    am.update_bar(bar, False)\n    prev_data = data.copy()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:19:17.638187Z","iopub.execute_input":"2022-01-25T00:19:17.638762Z","iopub.status.idle":"2022-01-25T00:19:36.301048Z","shell.execute_reply.started":"2022-01-25T00:19:17.638717Z","shell.execute_reply":"2022-01-25T00:19:36.300171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert am.inited","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:19:36.30269Z","iopub.execute_input":"2022-01-25T00:19:36.302993Z","iopub.status.idle":"2022-01-25T00:19:36.306925Z","shell.execute_reply.started":"2022-01-25T00:19:36.30296Z","shell.execute_reply":"2022-01-25T00:19:36.306046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load pretrained model\nmodel_by_asset_id = {}\nweight_by_asset_id = {}\nmodel_ckpt = \"../input/febaselinev2result/ckpt\"\nfor asset_id, asset_name in ASSET_ID_TO_NAME.items():\n    model_by_asset_id.setdefault(asset_id, [])\n    weight_by_asset_id.setdefault(asset_id, [])\n    for fold in range(5):\n        path = os.path.join(model_ckpt, asset_name, str(fold), \"lgb.ckpt\")\n        if os.path.exists(path):\n            model_by_asset_id[asset_id].append(lgb.Booster(model_file=path))\n            weight_by_asset_id[asset_id].append(1)\n    weight_by_asset_id[asset_id] = [i / sum(weight_by_asset_id[asset_id]) for i in weight_by_asset_id[asset_id]]","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:19:36.308273Z","iopub.execute_input":"2022-01-25T00:19:36.30858Z","iopub.status.idle":"2022-01-25T00:19:37.389732Z","shell.execute_reply.started":"2022-01-25T00:19:36.30854Z","shell.execute_reply":"2022-01-25T00:19:37.388862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# start submitting the data\nimport gresearch_crypto\nenv = gresearch_crypto.make_env()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:19:37.391399Z","iopub.execute_input":"2022-01-25T00:19:37.391723Z","iopub.status.idle":"2022-01-25T00:19:37.417186Z","shell.execute_reply.started":"2022-01-25T00:19:37.391678Z","shell.execute_reply":"2022-01-25T00:19:37.416388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_process_time = 0\navg_inference_time = 0\ntime1 = 0\ntime2 = 0\ntime3 = 0\ncount = 1\nfor df_test, df_pred in env.iter_test():\n    start_time = time.time()\n    df_test = df_test.set_index(\"Asset_ID\")\n    row_ids = dict(zip(df_test.row_id, df_test.index))\n    t = df_test.timestamp.iloc[0]\n    time1 += (time.time() - start_time)\n    start_time = time.time()\n    try:\n        # 强制按照asset_names来排序， 也就是按照0-14的顺序排好，asset_id就是对应的index\n        data = df_test.loc[asset_names]\n    except KeyError:\n        data = df_test.reindex(index=asset_names).loc[asset_names]\n    \n    time2 += (time.time() - start_time)\n    start_time = time.time()\n    bar = BarData(asset_names, \n            O=data.Open.values, \n            C=data.Close.values, \n            L=data.Low.values, \n            H=data.High.values, \n            VWAP=data.VWAP.values, \n            V=data.Volume.values,\n            T=np.array([t] * len(asset_names)).astype(int),\n            Cnt=data.Count.values)\n    am.update_bar(bar)\n    time3 += (time.time() - start_time)\n    start_time = time.time()\n    start_time = time.time()\n    df_pred = df_pred.set_index(\"row_id\")\n    # asset-specific model\n    for row_id, asset_id in row_ids.items():\n        feature_names = features_by_asset[ASSET_ID_TO_NAME[asset_id]]\n        val = np.array([am.feature_dict[feature_name][asset_id] for feature_name in feature_names]).astype(float).reshape(1, -1)\n        for model, weight in zip(model_by_asset_id[asset_id], weight_by_asset_id[asset_id]):\n            pred = model.predict(val)[0]\n#             pred = model.predict(val)[0]  \n            pred = model.predict(val)[0]  # predict double times to test the maximum number of models we could use\n            df_pred.loc[row_id, \"Target\"] += weight * pred\n\n        df_pred.loc[row_id, \"Target\"] += pred\n\n    df_pred = df_pred.reset_index()\n    avg_inference_time += (time.time() - start_time)\n    env.predict(df_pred)\n    count += 1\nprint(\"avg_infer_time=\", avg_inference_time/count)\nprint(\"t1=\", time1/count)\nprint(\"t2=\", time2/count)\nprint(\"t3=\", time3/count)\nprint(\"count=\", str(count))","metadata":{"execution":{"iopub.status.busy":"2022-01-25T00:26:22.136987Z","iopub.execute_input":"2022-01-25T00:26:22.137671Z","iopub.status.idle":"2022-01-25T00:26:22.530078Z","shell.execute_reply.started":"2022-01-25T00:26:22.137631Z","shell.execute_reply":"2022-01-25T00:26:22.52938Z"},"trusted":true},"execution_count":null,"outputs":[]}]}