{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb \nimport sklearn\nimport os\nimport json\nfrom scipy.stats import spearmanr\nimport matplotlib.pyplot as plt\nimport logging\nimport json\nfrom tqdm.notebook import tqdm\nimport seaborn as sns\nSKIPS = []","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-04T07:18:40.799372Z","iopub.execute_input":"2022-01-04T07:18:40.799688Z","iopub.status.idle":"2022-01-04T07:18:40.805665Z","shell.execute_reply.started":"2022-01-04T07:18:40.799652Z","shell.execute_reply":"2022-01-04T07:18:40.8044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install talib-binary","metadata":{"execution":{"iopub.status.busy":"2022-01-04T06:56:49.813679Z","iopub.execute_input":"2022-01-04T06:56:49.814076Z","iopub.status.idle":"2022-01-04T06:56:59.346998Z","shell.execute_reply.started":"2022-01-04T06:56:49.814008Z","shell.execute_reply":"2022-01-04T06:56:59.346054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import talib","metadata":{"execution":{"iopub.status.busy":"2022-01-04T06:58:36.205527Z","iopub.execute_input":"2022-01-04T06:58:36.205858Z","iopub.status.idle":"2022-01-04T06:58:36.220134Z","shell.execute_reply.started":"2022-01-04T06:58:36.205823Z","shell.execute_reply":"2022-01-04T06:58:36.219212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs_1 = {\n    \"RET\": (1, 100),\n    \"STD\": (2, 100),\n    \"RET_STD\": (2, 100),\n    \"RSI\": (2, 100),\n    \"ATR\": (2, 100),\n    \"MFI\": (2, 100),\n    \"VOL\": (2, 100),\n    \"TRIX\": (2, 100),\n    \"EFFICIENCY\": (10, 100),\n    \"EI\": (10, 100)\n}\n\nconfigs_2 = {\n    \"MACD_HIST\": [(2, 50, 10), (3, 200, 10)],\n    \"DEMA\": [(2, 50, 2), (3, 200, 5)],\n    \"MACD\": [(2, 50, 2), (3, 200, 5)],\n    \"EVENT_MACROSS\": [(1, 50, 2), (3, 200, 5)],\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-04T07:14:17.499116Z","iopub.execute_input":"2022-01-04T07:14:17.499425Z","iopub.status.idle":"2022-01-04T07:14:17.50441Z","shell.execute_reply.started":"2022-01-04T07:14:17.499395Z","shell.execute_reply":"2022-01-04T07:14:17.503665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_df = pd.read_feather(\"../input/pipeline-building-dataprepare/train.feather\")\ntotal_df['datetime'] = pd.to_datetime(total_df['timestamp'], unit='s')","metadata":{"execution":{"iopub.status.busy":"2022-01-04T07:11:35.073137Z","iopub.execute_input":"2022-01-04T07:11:35.073757Z","iopub.status.idle":"2022-01-04T07:11:46.749335Z","shell.execute_reply.started":"2022-01-04T07:11:35.073699Z","shell.execute_reply":"2022-01-04T07:11:46.748357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# All features\ndef RET(df, n):\n    return df['Close'].pct_change(n)\n\ndef STD(df, n):\n    return df['Close'].pct_change(1).rolling(n).std()\n\ndef RET_STD(df, n):\n    return RET(df, n) * STD(df, n)\n\ndef RSI(df, n):\n    return talib.RSI(df['Close'], n)\n\ndef ATR(df, n):\n    return talib.ATR(df[\"High\"], df.Low, df.Close, n)\n\ndef MFI(df, n):\n    return talib.MFI(df['High'], df['Low'], df['Close'], df['Volume'], n)\n\ndef VOL(df, n):\n    ret = df['Close'].pct_change(1)\n    return np.sqrt((ret ** 2).rolling(n).mean())\n\ndef TRIX(df, n):\n    return talib.TRIX(df['Close'], n)\n\ndef MACD(df, fast, slow):\n    return talib.MACD(df.Close, fast, slow)[0]\n\ndef MACD_HIST(df, fast, slow):\n    return talib.MACD(df.Close, fast, slow)[2]\n\ndef DEMA(df, n1, n2):\n    return np.log(df['Close'].rolling(n1).mean() / df['Close'].rolling(n2).mean())\n\ndef EFFICIENCY(df, n):\n    speed = (df.Close - df.Close.shift(n))\n    volatility = (df.Close - df.Close.shift(1)).abs().rolling(n).sum()\n    return speed / volatility\n\ndef EI(df, n):\n    neg = df.Close - df.Low.rolling(n).min()\n    high = df.High.rolling(n).max() - df.Close\n    ei = high / neg\n    ei = np.clip(ei, -100, 100)\n    return ei\n\ndef EVEMT_BIGGER_VOLUME(df):\n    return (df.Volume > df.Volume.shift(1)).astype(float)\n\ndef EVENT_MACROSS(df, fast, slow):\n    fast_s = df.Close.rolling(fast).mean()\n    slow_s = df.Close.rolling(slow).mean()\n    return (fast_s > slow_s).astype(float)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T07:14:20.044558Z","iopub.execute_input":"2022-01-04T07:14:20.045223Z","iopub.status.idle":"2022-01-04T07:14:20.060298Z","shell.execute_reply.started":"2022-01-04T07:14:20.045171Z","shell.execute_reply":"2022-01-04T07:14:20.058786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_best_timeperiod_for_one_asset(ASSET):\n    df = total_df[total_df.Asset_ID == ASSET].fillna(method=\"ffill\").dropna()\n    df = df.set_index(\"datetime\")\n    if ASSET == \"Maker\":\n        df = df.loc[\"2020-08-04\":]\n    elif ASSET == \"Monero\":\n        df = df.loc[\"2018-11-05\":]\n    elif ASSET == \"Stellar\":\n        df = df.loc[\"2018-07-14\":]\n    df.drop(\"timestamp\", inplace=True, axis=1)\n    df[\"raw_target\"] = df['Close'].pct_change(15).shift(-15)\n    best_period = {}\n\n    for func_name, range_ in configs_1.items():\n        best_period[func_name] = {}\n        func = eval(func_name)\n        corr_s = {}\n        for i in tqdm(range(range_[0], range_[1])):\n            lag_ret = func(df, i)\n            corr_s[i] = lag_ret.corr(df['Target'])\n        corr_s = pd.Series(corr_s)\n        avg = corr_s.mean()\n        # In order to prevent overfitting. Try not use the period with the highest correlation\n        if avg > 0:\n            t = corr_s.sort_values(ascending=False).index[3]\n            print(\"Momentum Feature. Best Period: {}\".format(t))\n        else:\n            t = corr_s.sort_values(ascending=True).index[3]\n            print(\"Mean-Revertion Feature. Best Period: {}\".format(t))\n        best_period[func_name][\"timeperiod\"] = int(t)\n        best_period[func_name][\"correlation\"] = int(corr_s[t])\n        corr_s.name = ASSET + \"-\" + func_name\n        corr_s.plot(figsize=(15, 5), title=ASSET + \"-\" + func_name)\n        plt.show()\n        print(corr_s.sort_values().head(5))\n        print(corr_s.sort_values().tail(5))\n\n    for func_name, range_ in configs_2.items():\n        best_period[func_name] = {}\n        func = eval(func_name)\n        fs_params = [(fast, slow) for fast in range(range_[0][0], range_[0][1], range_[0][2]) for slow in range(range_[1][0], range_[1][1], range_[1][2]) if fast<slow]\n        dis_res = []\n        for f, s in tqdm(fs_params):\n            dis = func(df, f, s)\n            dis_res.append(dis.corr(df['Target']))\n        fs_corr = pd.Series(dis_res, index=pd.MultiIndex.from_tuples(fs_params))\n        fs_corr.name = ASSET + \"-\" + func_name\n        avg = fs_corr.mean()\n        if avg > 0:\n            t = fs_corr.sort_values(ascending=False).index[3]\n            print(\"Momentum Feature. Best Period: {}\".format(t))\n        else:\n            t = fs_corr.sort_values(ascending=True).index[3]\n            print(\"Mean-Revertion Feature. Best Period: {}\".format(t))\n        best_period[func_name][\"timeperiod\"] = str(t)\n        best_period[func_name][\"correlation\"] = int(fs_corr[t])\n        plt.figure(figsize=(15, 15))\n        print(fs_corr.sort_values().head(5))\n        print(fs_corr.sort_values().tail(5))\n        sns.heatmap(fs_corr.unstack())\n        plt.show()\n\n\n    with open(\"./best_period_{}.json\".format(ASSET), \"w\") as f:\n        json.dump(best_period, f)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T07:17:13.254795Z","iopub.execute_input":"2022-01-04T07:17:13.255122Z","iopub.status.idle":"2022-01-04T07:17:13.273212Z","shell.execute_reply.started":"2022-01-04T07:17:13.255089Z","shell.execute_reply":"2022-01-04T07:17:13.272269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for asset in total_df.Asset_ID.unique():\n    if asset in SKIPS:\n        continue\n    get_best_timeperiod_for_one_asset(asset)","metadata":{},"execution_count":null,"outputs":[]}]}