{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport pandas as pd\nimport numpy as np\nimport time\nimport gc\n\n# from sklearn.linear_model import RidgeCV, Ridge\n# from sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import StandardScaler\n# from sklearn.model_selection import train_test_split\n\nfrom lightgbm import LGBMRegressor\n# from xgboost import XGBRegressor\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom scipy.stats.stats import pearsonr\nfrom tqdm import tqdm\n\nimport gresearch_crypto\n\n# Warningの無効化\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\n# データフレームcolumの全表示\npd.set_option(\"display.max_columns\", None)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-29T02:20:34.448103Z","iopub.execute_input":"2022-04-29T02:20:34.448563Z","iopub.status.idle":"2022-04-29T02:20:36.501239Z","shell.execute_reply.started":"2022-04-29T02:20:34.448465Z","shell.execute_reply":"2022-04-29T02:20:36.500341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        # else:\n            # df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-04-29T02:20:36.502549Z","iopub.execute_input":"2022-04-29T02:20:36.502755Z","iopub.status.idle":"2022-04-29T02:20:36.514889Z","shell.execute_reply.started":"2022-04-29T02:20:36.502730Z","shell.execute_reply":"2022-04-29T02:20:36.514060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(r\"/kaggle/input/g-research-crypto-forecasting/asset_details.csv\").sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-04-29T02:20:36.516478Z","iopub.execute_input":"2022-04-29T02:20:36.516709Z","iopub.status.idle":"2022-04-29T02:20:36.557758Z","shell.execute_reply.started":"2022-04-29T02:20:36.516684Z","shell.execute_reply":"2022-04-29T02:20:36.557173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# val = df_asset_details.loc[ df_asset_details[\"Asset_ID\"] == 3, \"Asset_Name\"]\n# val.values[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/train.csv\"):\n    df = pd.read_csv(file_name).pipe(reduce_mem_usage)\n    df[\"datetime\"] = pd.to_datetime(df[\"timestamp\"], unit=\"s\")\n#     df = df[df[\"datetime\"] > \"2019-06-13 00:00:00\"]\n    df = df[df[\"datetime\"] >= \"2020-05-13 00:00:00\"]\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = read_csv_strict()\n# df_train.head()\ndf_supp_train = read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv\")\n# df_supp_train.head()\ndf_test = read_csv_strict(file_name=r\"/kaggle/input/g-research-crypto-forecasting/example_test.csv\")\n# df_test.head()\ndf_train = pd.concat([df_train, df_supp_train, df_test])\ndf_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_supp_train.shape, df_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_supp_train, df_test\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.memory_usage().sum() / 1024**3","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Outliers","metadata":{}},{"cell_type":"code","source":"## Negative volume implies lack of liquidity in the market\n# df_train[df_train.Volume < 0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Negative volume indicates that the market lacked liquidity for these instances; in other words, the bid and ask price  failed to overlap.","metadata":{}},{"cell_type":"markdown","source":"## Check for NaNs","metadata":{}},{"cell_type":"code","source":"## https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778\n## https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition?scriptVersionId=84976518&cellId=51\n\n# df_train[(df_train.Target.isna()) & (df_train.Asset_ID==8)]\n# df_train[df_train.Target.isna()].groupby(\"Asset_ID\")[\"timestamp\"].count()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train[(df_train.Target.isna()) & (df_train.Asset_ID==8)].describe(percentiles=[0.25, 0.50, 0.75, 0.90, 0.95, 0.99])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## data processing","metadata":{}},{"cell_type":"code","source":"df_train = df_train.sort_values(by=[\"Asset_ID\", \"timestamp\"]\n                               ,ascending=True)\n\ndf_train.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_00=df_train[df_train.Asset_ID==5].copy()\nprint(len(df_train_00))\ndf_train_00.drop_duplicates('timestamp',inplace=True)\nprint(len(df_train_00))\nnew_index = pd.Index(np.arange(df_train_00.iloc[0]['timestamp'], df_train_00.iloc[-1]['timestamp'] , 60), name=\"timestamp\")\ndf_train_00=df_train_00.set_index(\"timestamp\").reindex(new_index).reset_index()\ndf_train_00[\"trade_count\"]=0\ncol_name=[\"Open\",\"High\",\"Low\",\"Close\",\"Volume\",\"VWAP\",\"trade_count\",\"Count\"]\nfor name in col_name:\n    df_train_00[name].fillna(0, inplace=True)\ndf_train_00=df_train_00[[\"Target\",\"Open\",\"High\",\"Low\",\"Close\",\"Volume\",\"VWAP\",\"trade_count\",\"Count\",\"timestamp\"]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rename dataframe\ndf_train_00.columns=[\"y\",\"AlphaCandleWriter_00_0\",\"AlphaCandleWriter_00_1\",\"AlphaCandleWriter_00_2\",\"AlphaCandleWriter_00_3\",\"AlphaCandleWriter_00_4\"\n                     ,\"AlphaCandleWriter_00_5\",\"AlphaCandleWriter_00_6\",\"AlphaCandleWriter_00_7\",\"AlphaCandleWriter_00_8\"]\ndf_train.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature","metadata":{}},{"cell_type":"code","source":" conda install -c conda-forge ta-lib","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import threading\nimport numpy as np\nimport talib\nclass get_indicator():\n    def __init__(self, lib) -> None:\n        self.technical_index = []\n        self.technical_header = []\n        self.total_cnt_indicator = 0\n        self.lib = lib #希望call出的技術指標\n        self.open = []\n        self.high = [] \n        self.low = []\n        self.close = []\n        self.volume = []\n\n    #週期指標\n    def HT_DCPERIOD(self):\n        indicator = talib.HT_DCPERIOD(self.close)\n        indicator = np.reshape(indicator, (-1,1))##要先reshape，不然無法append\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"HT_DCPERIOD\")\n        self.total_cnt_indicator += 1\n\n    def HT_DCPHASE(self):\n        indicator = talib.HT_DCPHASE(self.close)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"HT_DCPHASE\")\n        self.total_cnt_indicator += 1\n\n    def HT_PHASOR_inphase(self):\n        HT_PHASOR_inphase,HT_PHASOR_quadrature = talib.HT_PHASOR(self.close)\n        indicator = np.reshape(HT_PHASOR_inphase, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"HT_PHASOR_inphase\")\n        self.total_cnt_indicator += 1\n\n    def HT_PHASOR_quadrature(self):\n        HT_PHASOR_inphase,HT_PHASOR_quadrature = talib.HT_PHASOR(self.close)\n        indicator = np.reshape(HT_PHASOR_quadrature, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"HT_PHASOR_quadrature\")\n        self.total_cnt_indicator += 1 \n\n    def HT_SINE_sine(self):\n        HT_SINE_sine,HT_SINE_leadsine  = talib.HT_SINE(self.close)\n        indicator = np.reshape(HT_SINE_sine, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"HT_SINE_sine\")\n        self.total_cnt_indicator += 1 \n\n    def HT_SINE_leadsine(self):\n        HT_SINE_sine,HT_SINE_leadsine  = talib.HT_SINE(self.close)\n        indicator = np.reshape(HT_SINE_leadsine, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"HT_SINE_leadsine\")\n        self.total_cnt_indicator += 1 \n\n    def HT_TRENDMODE(self):\n        indicator  = talib.HT_TRENDMODE(self.close)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"HT_TRENDMODE\")\n        self.total_cnt_indicator += 1 \n\n    #動量指標\n    # 1. ADX：平均趋向指数\n    def ADX(self):\n        indicator = talib.ADX(self.high, self.low, self.close, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"ADX\")\n        self.total_cnt_indicator += 1\n    # 2. ADXR：平均趋向指数的趋向指数\n    def ADXR(self):\n        indicator = talib.ADXR(self.high, self.low, self.close, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"ADXR\")\n        self.total_cnt_indicator += 1\n    # 3. APO ：价格震荡指数\n    def APO(self):\n        indicator = talib.APO(self.close, fastperiod=12, slowperiod=26, matype=0)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"APO\")\n        self.total_cnt_indicator += 1\n    # 4. AROON ：阿隆指标\n    def AROON_aroondown(self):\n        AROON_aroondown, AROON_aroonup = talib.AROON(self.high, self.low, timeperiod=14)\n        indicator = np.reshape(AROON_aroondown, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"AROON_aroondown\")\n        self.total_cnt_indicator += 1\n\n    def AROON_aroonup(self):\n        AROON_aroondown, AROON_aroonup = talib.AROON(self.high, self.low, timeperiod=14)\n        indicator = np.reshape(AROON_aroonup, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"AROON_aroonup\")\n        self.total_cnt_indicator += 1  \n   # 5.AROONOSC ：阿隆振荡\n    def AROONOSC(self):\n        indicator = talib.AROONOSC(self.high, self.low, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"AROONOSC\")\n        self.total_cnt_indicator += 1  \n    # 6. BOP ：均势指标\n    def BOP(self):\n        indicator = talib.BOP(self.open, self.high, self.low, self.close)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"BOP\")\n        self.total_cnt_indicator += 1\n    # 7. CCI ：顺势指标\n    def CCI(self):\n        indicator = talib.CCI(self.high, self.low, self.close, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CCI\")\n        self.total_cnt_indicator += 1\n    # 8. CMO ：钱德动量摆动指标\n    def CMO(self):\n        indicator = talib.CMO(self.close, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CMO\")\n        self.total_cnt_indicator += 1\n    # 9. DX ：动向指标或趋向指标\n    def DX(self):\n        indicator = talib.DX(self.high, self.low, self.close, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"DX\")\n        self.total_cnt_indicator += 1\n    # 10. MACD:平滑异同移动平均线\n    def MACD_macd(self):\n        macd, macdsignal, macdhist = talib.MACD(self.close, fastperiod=12, slowperiod=26, signalperiod=9)\n        indicator = np.reshape(macd, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MACD_macd\")\n        self.total_cnt_indicator += 1\n    \n    def MACD_macdsignal(self):\n        macd, macdsignal, macdhist = talib.MACD(self.close, fastperiod=12, slowperiod=26, signalperiod=9)\n        indicator = np.reshape(macdsignal, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MACD_macdsignal\")\n        self.total_cnt_indicator += 1\n    \n    def MACD_macdhist(self):\n        macd, macdsignal, macdhist = talib.MACD(self.close, fastperiod=12, slowperiod=26, signalperiod=9)\n        indicator = np.reshape(macdhist, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MACD_macdhist\")\n        self.total_cnt_indicator += 1\n    # 11. MACDEXT :MACD延伸\n    def MACDEXT(self):\n        ta_tuple = talib.MACDEXT(self.close, fastperiod=12, fastmatype=0, slowperiod=26, slowmatype=0, signalperiod=9, signalmatype=0)\n        for n, indicator in threading.enumerate(ta_tuple):\n            indicator = np.reshape(indicator, (-1,1))\n            self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n            self.technical_header.append(f\"MACDEXT_{n}\")\n            self.total_cnt_indicator += 1\n    # 12. MFI ：资金流量指标\n    def MFI(self):\n        indicator = talib.MFI(self.high, self.low, self.close, self.volume, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MFI\")\n        self.total_cnt_indicator += 1\n    # 13. MINUS_DI：DMI 中的DI指标 负方向指标\n    def MINUS_DI(self):\n        indicator = talib.MINUS_DI(self.high, self.low, self.close, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MINUS_DI\")\n        self.total_cnt_indicator += 1\n    # 14. MINUS_DM：上升动向值\n    def MINUS_DM(self):\n        indicator = talib.MINUS_DM(self.high, self.low, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MINUS_DI\")\n        self.total_cnt_indicator += 1\n\n    #重疊指標\n    # 1.简单移动平均指标SMA\n    def SMA(self):\n        indicator = talib.SMA(self.close,5)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"SMA\")\n        self.total_cnt_indicator += 1 \n    # 2.布林线BBANDS\n    def upper(self):\n        upper, middle, lower = talib.BBANDS(self.close,5,matype = talib.MA_Type.EMA)\n        indicator = np.reshape(upper, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"upper\")\n        self.total_cnt_indicator += 1\n    def middle(self):\n        upper, middle, lower = talib.BBANDS(self.close,5,matype = talib.MA_Type.EMA)\n        indicator = np.reshape(middle, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"middle\")\n        self.total_cnt_indicator += 1 \n    def lower(self):\n        upper, middle, lower = talib.BBANDS(self.close,5,matype = talib.MA_Type.EMA)\n        indicator = np.reshape(lower, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"lower\")\n        self.total_cnt_indicator += 1 \n    # 3. DEMA 双移动平均线:DEMA = 2*EMA-EMA(EMA)\n    def DEMA(self):\n        indicator = talib.DEMA(self.close, timeperiod = 30)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"DEMA\")\n        self.total_cnt_indicator += 1\n    # 4. MA\n    def MA(self):\n        indicator = talib.MA(self.close, timeperiod = 10, matype = 0)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MA\")\n        self.total_cnt_indicator += 1\n    # 5. EMA\n    def EMA(self):\n        indicator = talib.EMA(self.close, timeperiod = 6)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"EMA\")\n        self.total_cnt_indicator += 1\n    # 6.KAMA：考夫曼的自适应移动平均线\n    def KAMA(self):\n        indicator = talib.KAMA(self.close, timeperiod = 30)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"KAMA\")\n        self.total_cnt_indicator += 1\n    # 7. MIDPRICE：阶段中点价格\n    def MIDPOINT(self):\n        indicator = talib.MIDPOINT(self.close, timeperiod = 14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MIDPOINT\")\n        self.total_cnt_indicator += 1\n    # 8.SAR：抛物线指标\n    def SAR(self):\n        indicator = talib.SAR(self.high, self.low, acceleration=0, maximum=0)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"SAR\")\n        self.total_cnt_indicator += 1\n    # 9.MIDPRICE：阶段中点价格（Midpoint Price over period)\n    def MIDPRICE(self):\n        indicator = talib.MIDPRICE(self.high, self.low, timeperiod=14)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"MIDPRICE\")\n        self.total_cnt_indicator += 1\n    # 10. T3:三重移动平均线\n    def T3(self):\n        indicator = talib.T3(self.close, timeperiod = 5, vfactor = 0)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"T3\")\n        self.total_cnt_indicator += 1\n    # # 11.TEMA：三重指数移动平均线\n    def TEMA(self):\n        indicator = talib.TEMA(self.close, timeperiod=30)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"TEMA\")\n        self.total_cnt_indicator += 1\n    # 12.SAREXT:SAR的抛物面扩展\n    def SAREXT(self):\n        indicator = talib.SAREXT(self.high, self.low, startvalue=0, offsetonreverse=0, accelerationinitlong=0, accelerationlong=0, accelerationmaxlong=0, accelerationinitshort=0, accelerationshort=0, accelerationmaxshort=0)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"SAREXT\")\n        self.total_cnt_indicator += 1\n    # 13.WMA：移动加权平均法\n    def WMA(self):\n        indicator = talib.WMA(self.close, timeperiod = 30)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"WMA\")\n        self.total_cnt_indicator += 1\n    #\n    def CDL2CROWS(self):\n        indicator = talib.CDL2CROWS(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDL2CROWS\")\n        self.total_cnt_indicator += 1\n\n    def CDL3BLACKCROWS(self):\n        indicator = talib.CDL3BLACKCROWS(self.open, self.high, self.low, self.close)\n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDL3BLACKCROWS\")\n        self.total_cnt_indicator += 1\n\n    def CDL3INSIDE(self):\n        indicator = talib.CDL3INSIDE(self.open, self.high, self.low, self.close)  \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDL3INSIDE\")\n        self.total_cnt_indicator += 1\n\n    def CDL3LINESTRIKE(self):\n        indicator = talib.CDL3LINESTRIKE(self.open, self.high, self.low, self.close)  \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDL3LINESTRIKE\")\n        self.total_cnt_indicator += 1\n\n    def CDL3OUTSIDE(self):\n        indicator = talib.CDL3OUTSIDE(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDL3OUTSIDE\")\n        self.total_cnt_indicator += 1\n\n    def CDL3STARSINSOUTH(self):\n        indicator = talib.CDL3STARSINSOUTH(self.open, self.high, self.low, self.close)  \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDL3STARSINSOUTH\")\n        self.total_cnt_indicator += 1\n\n    def CDL3WHITESOLDIERS(self):\n        indicator = talib.CDL3WHITESOLDIERS(self.open, self.high, self.low, self.close)  \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDL3WHITESOLDIERS\")\n        self.total_cnt_indicator += 1\n\n    def CDLABANDONEDBABY(self):\n        indicator = talib.CDLABANDONEDBABY(self.open, self.high, self.low, self.close, penetration=0) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLABANDONEDBABY\")\n        self.total_cnt_indicator += 1\n\n    def CDLADVANCEBLOCK(self):\n        indicator = talib.CDLADVANCEBLOCK(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLADVANCEBLOCK\")\n        self.total_cnt_indicator += 1\n\n    def CDLBELTHOLD(self):\n        indicator = talib.CDLBELTHOLD(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLBELTHOLD\")\n        self.total_cnt_indicator += 1\n\n    def CDLBREAKAWAY(self):\n        indicator = talib.CDLBREAKAWAY(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLBREAKAWAY\")\n        self.total_cnt_indicator += 1\n\n    def CDLCLOSINGMARUBOZU(self):\n        indicator = talib.CDLCLOSINGMARUBOZU(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLCLOSINGMARUBOZU\")\n        self.total_cnt_indicator += 1\n\n    def CDLCONCEALBABYSWALL(self):\n        indicator = talib.CDLCONCEALBABYSWALL(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLCONCEALBABYSWALL\")\n        self.total_cnt_indicator += 1\n\n    def CDLCOUNTERATTACK(self):\n        indicator = talib.CDLCOUNTERATTACK(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLCOUNTERATTACK\")\n        self.total_cnt_indicator += 1\n \n    def CDLDARKCLOUDCOVER(self):\n        indicator = talib.CDLDARKCLOUDCOVER(self.open, self.high, self.low, self.close, penetration=0) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLDARKCLOUDCOVER\")\n        self.total_cnt_indicator += 1\n\n    def CDLDOJI(self):\n        indicator = talib.CDLDOJI(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLDOJI\")\n        self.total_cnt_indicator += 1\n\n    def CDLDOJISTAR(self):\n        indicator = talib.CDLDOJISTAR(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLDOJISTAR\")\n        self.total_cnt_indicator += 1\n\n    def CDLDRAGONFLYDOJI(self):\n        indicator = talib.CDLDRAGONFLYDOJI(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLDRAGONFLYDOJI\")\n        self.total_cnt_indicator += 1\n\n    def CDLENGULFING(self):\n        indicator = talib.CDLENGULFING(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLENGULFING\")\n        self.total_cnt_indicator += 1\n \n    def CDLEVENINGDOJISTAR(self):\n        indicator = talib.CDLEVENINGDOJISTAR(self.open, self.high, self.low, self.close, penetration=0) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLEVENINGDOJISTAR\")\n        self.total_cnt_indicator += 1\n\n    def CDLEVENINGSTAR(self):\n        indicator = talib.CDLEVENINGSTAR(self.open, self.high, self.low, self.close, penetration=0) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLEVENINGSTAR\")\n        self.total_cnt_indicator += 1\n\n    def CDLGAPSIDESIDEWHITE(self):\n        indicator = talib.CDLGAPSIDESIDEWHITE(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLGAPSIDESIDEWHITE\")\n        self.total_cnt_indicator += 1\n\n    def CDLGRAVESTONEDOJI(self):\n        indicator = talib.CDLGRAVESTONEDOJI(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLGRAVESTONEDOJI\")\n        self.total_cnt_indicator += 1\n\n    def CDLHAMMER(self):\n        indicator = talib.CDLHAMMER(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLHAMMER\")\n        self.total_cnt_indicator += 1\n\n    def CDLHANGINGMAN(self):\n        indicator = talib.CDLHANGINGMAN(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLHANGINGMAN\")\n        self.total_cnt_indicator += 1\n\n    def CDLHARAMI(self):\n        indicator = talib.CDLHARAMI(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLHARAMI\")\n        self.total_cnt_indicator += 1\n\n    def CDLHARAMICROSS(self):\n        indicator = talib.CDLHARAMICROSS(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLHARAMICROSS\")\n        self.total_cnt_indicator += 1\n\n    def CDLHIGHWAVE(self):\n        indicator = talib.CDLHIGHWAVE(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLHIGHWAVE\")\n        self.total_cnt_indicator += 1\n\n    def CDLHIKKAKE(self):\n        indicator = talib.CDLHIKKAKE(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLHIKKAKE\")\n        self.total_cnt_indicator += 1\n\n    def CDLHIKKAKEMOD(self):\n        indicator = talib.CDLHIKKAKEMOD(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLHIKKAKEMOD\")\n        self.total_cnt_indicator += 1\n\n    def CDLHOMINGPIGEON(self):\n        indicator = talib.CDLHOMINGPIGEON(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLHOMINGPIGEON\")\n        self.total_cnt_indicator += 1\n \n    def CDLIDENTICAL3CROWS(self):\n        indicator = talib.CDLIDENTICAL3CROWS(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLIDENTICAL3CROWS\")\n        self.total_cnt_indicator += 1\n\n    def CDLINNECK(self):\n        indicator = talib.CDLINNECK(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLINNECK\")\n        self.total_cnt_indicator += 1\n\n    def CDLINVERTEDHAMMER(self):\n        indicator = talib.CDLINVERTEDHAMMER(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLINVERTEDHAMMER\")\n        self.total_cnt_indicator += 1\n\n    def CDLKICKING(self):\n        indicator = talib.CDLKICKING(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLKICKING\")\n        self.total_cnt_indicator += 1\n\n    def CDLKICKINGBYLENGTH(self):\n        indicator = talib.CDLKICKINGBYLENGTH(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLKICKINGBYLENGTH\")\n        self.total_cnt_indicator += 1\n\n    def CDLLADDERBOTTOM(self):\n        indicator = talib.CDLLADDERBOTTOM(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLLADDERBOTTOM\")\n        self.total_cnt_indicator += 1\n\n    def CDLLONGLEGGEDDOJI(self):\n        indicator = talib.CDLLONGLEGGEDDOJI(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLLONGLEGGEDDOJI\")\n        self.total_cnt_indicator += 1\n\n    def CDLLONGLINE(self):\n        indicator = talib.CDLLONGLINE(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLLONGLINE\")\n        self.total_cnt_indicator += 1\n\n    def CDLMARUBOZU(self):\n        indicator = talib.CDLMARUBOZU(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLMARUBOZU\")\n        self.total_cnt_indicator += 1\n\n    def CDLMATCHINGLOW(self):\n        indicator = talib.CDLMATCHINGLOW(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLMATCHINGLOW\")\n        self.total_cnt_indicator += 1\n\n    def CDLMATHOLD(self):\n        indicator = talib.CDLMATHOLD(self.open, self.high, self.low, self.close, penetration=0) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLMATHOLD\")\n        self.total_cnt_indicator += 1\n\n    def CDLMORNINGDOJISTAR(self):\n        indicator = talib.CDLMORNINGDOJISTAR(self.open, self.high, self.low, self.close, penetration=0) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLMORNINGDOJISTAR\")\n        self.total_cnt_indicator += 1\n\n    def CDLMORNINGSTAR(self):\n        indicator = talib.CDLMORNINGSTAR(self.open, self.high, self.low, self.close, penetration=0) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLMORNINGSTAR\")\n        self.total_cnt_indicator += 1\n\n    def CDLONNECK(self):\n        indicator = talib.CDLONNECK(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLONNECK\")\n        self.total_cnt_indicator += 1\n\n    def CDLPIERCING(self):\n        indicator = talib.CDLPIERCING(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLPIERCING\")\n        self.total_cnt_indicator += 1\n\n    def CDLRICKSHAWMAN(self):\n        indicator = talib.CDLRICKSHAWMAN(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLRICKSHAWMAN\")\n        self.total_cnt_indicator += 1\n\n    def CDLRISEFALL3METHODS(self):\n        indicator = talib.CDLRISEFALL3METHODS(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLRISEFALL3METHODS\")\n        self.total_cnt_indicator += 1\n\n    def CDLSEPARATINGLINES(self):\n        indicator = talib.CDLSEPARATINGLINES(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLSEPARATINGLINES\")\n        self.total_cnt_indicator += 1\n\n    def CDLSHOOTINGSTAR(self):\n        indicator = talib.CDLSHOOTINGSTAR(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLSHOOTINGSTAR\")\n        self.total_cnt_indicator += 1\n\n    def CDLSHORTLINE(self):\n        indicator = talib.CDLSHORTLINE(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLSHORTLINE\")\n        self.total_cnt_indicator += 1\n\n    def CDLSPINNINGTOP(self):\n        indicator = talib.CDLSPINNINGTOP(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLSPINNINGTOP\")\n        self.total_cnt_indicator += 1\n\n    def CDLSTALLEDPATTERN(self):\n        indicator = talib.CDLSTALLEDPATTERN(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLSTALLEDPATTERN\")\n        self.total_cnt_indicator += 1\n\n    def CDLSTICKSANDWICH(self):\n        indicator = talib.CDLSTICKSANDWICH(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLSTICKSANDWICH\")\n        self.total_cnt_indicator += 1\n\n    def CDLTAKURI(self):\n        indicator = talib.CDLTAKURI(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLTAKURI\")\n        self.total_cnt_indicator += 1\n\n    def CDLTASUKIGAP(self):\n        indicator = talib.CDLTASUKIGAP(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLTASUKIGAP\")\n        self.total_cnt_indicator += 1\n\n    def CDLTHRUSTING(self):\n        indicator = talib.CDLTHRUSTING(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLTHRUSTING\")\n        self.total_cnt_indicator += 1\n \n    def CDLTRISTAR(self):\n        indicator = talib.CDLTRISTAR(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLTRISTAR\")\n        self.total_cnt_indicator += 1\n\n    def CDLUNIQUE3RIVER(self):\n        indicator = talib.CDLUNIQUE3RIVER(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLUNIQUE3RIVER\")\n        self.total_cnt_indicator += 1\n\n    def CDLUPSIDEGAP2CROWS(self):\n        indicator = talib.CDLUPSIDEGAP2CROWS(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLUPSIDEGAP2CROWS\")\n        self.total_cnt_indicator += 1\n\n    def CDLXSIDEGAP3METHODS(self):\n        indicator = talib.CDLXSIDEGAP3METHODS(self.open, self.high, self.low, self.close) \n        indicator = np.reshape(indicator, (-1,1))\n        self.technical_index = np.append(self.technical_index, indicator, axis = 1)\n        self.technical_header.append(\"CDLXSIDEGAP3METHODS\")\n        self.total_cnt_indicator += 1\n\n    def config_reading(self):\n        for calling_indicator in  self.lib:\n            if(calling_indicator == \"HT_DCPERIOD\"): self.HT_DCPERIOD()\n            if(calling_indicator == \"HT_DCPHASE\"): self.HT_DCPHASE()\n            if(calling_indicator == \"HT_PHASOR_inphase\"): self.HT_PHASOR_inphase()\n            if(calling_indicator == \"HT_PHASOR_quadrature\"): self.HT_PHASOR_quadrature()\n            if(calling_indicator == \"HT_SINE_sine\"): self.HT_SINE_sine()\n            if(calling_indicator == \"HT_SINE_leadsine\"): self.HT_SINE_leadsine()\n            if(calling_indicator == \"HT_TRENDMODE\"): self.HT_TRENDMODE()\n            if(calling_indicator == \"ADX\"): self.ADX()\n            if(calling_indicator == \"ADXR\"): self.ADXR()\n            if(calling_indicator == \"APO\"): self.APO()\n            if(calling_indicator == \"AROON_aroondown\"): self.AROON_aroondown()\n            if(calling_indicator == \"AROON_aroonup\"): self.AROON_aroonup()\n            if(calling_indicator == \"AROONOSC\"): self.AROONOSC()\n            if(calling_indicator == \"BOP\"): self.BOP()\n            if(calling_indicator == \"CCI\"): self.CCI()\n            if(calling_indicator == \"CMO\"): self.CMO()\n            if(calling_indicator == \"DX\"): self.DX()\n            if(calling_indicator == \"MACD_macd\"): self.MACD_macd()\n            if(calling_indicator == \"MACD_macdsignal\"): self.MACD_macdsignal()\n            if(calling_indicator == \"MACD_macdhist\"): self.MACD_macdhist()\n            if(calling_indicator == \"MFI\"): self.MFI()\n            if(calling_indicator == \"MINUS_DI\"): self.MINUS_DI()\n            if(calling_indicator == \"MINUS_DM\"): self.MINUS_DM()\n            if(calling_indicator == \"SMA\"): self.SMA()\n            if(calling_indicator == \"upper\"): self.upper()\n            if(calling_indicator == \"middle\"): self.middle()\n            if(calling_indicator == \"lower\"): self.lower()\n            if(calling_indicator == \"DEMA\"): self.DEMA()\n            if(calling_indicator == \"EMA\"): self.EMA()\n            if(calling_indicator == \"KAMA\"): self.KAMA()\n            if(calling_indicator == \"MA\"): self.MA()\n            if(calling_indicator == \"SAR\"): self.SAR()\n            if(calling_indicator == \"SAREXT\"): self.SAREXT()\n            if(calling_indicator == \"T3\"): self.T3()\n            if(calling_indicator == \"TEMA\"): self.TEMA()\n            if(calling_indicator == \"WMA\"): self.WMA()\n            if(calling_indicator == \"CDL2CROWS\"): self.CDL2CROWS()\n            if(calling_indicator == \"CDL3BLACKCROWS\"): self.CDL3BLACKCROWS()\n            if(calling_indicator == \"CDL3INSIDE\"): self.CDL3INSIDE()\n            if(calling_indicator == \"CDL3LINESTRIKE\"): self.CDL3LINESTRIKE()\n            if(calling_indicator == \"CDL3OUTSIDE\"): self.CDL3OUTSIDE()\n            if(calling_indicator == \"CDL3STARSINSOUTH\"): self.CDL3STARSINSOUTH()\n            if(calling_indicator == \"CDL3WHITESOLDIERS\"): self.CDL3WHITESOLDIERS()\n            if(calling_indicator == \"CDLABANDONEDBABY\"): self.CDLABANDONEDBABY()\n            if(calling_indicator == \"CDLADVANCEBLOCK\"): self.CDLADVANCEBLOCK()\n            if(calling_indicator == \"CDLBELTHOLD\"): self.CDLBELTHOLD()\n            if(calling_indicator == \"CDLBREAKAWAY\"): self.CDLBREAKAWAY()\n            if(calling_indicator == \"CDLCLOSINGMARUBOZU\"): self.CDLCLOSINGMARUBOZU()\n            if(calling_indicator == \"CDLCONCEALBABYSWALL\"): self.CDLCONCEALBABYSWALL()\n            if(calling_indicator == \"CDLCOUNTERATTACK\"): self.CDLCOUNTERATTACK()\n            if(calling_indicator == \"CDLDARKCLOUDCOVER\"): self.CDLDARKCLOUDCOVER()\n            if(calling_indicator == \"CDLDOJI\"): self.CDLDOJI()\n            if(calling_indicator == \"CDLDOJISTAR\"): self.CDLDOJISTAR()\n            if(calling_indicator == \"CDLDRAGONFLYDOJI\"): self.CDLDRAGONFLYDOJI()\n            if(calling_indicator == \"CDLENGULFING\"): self.CDLENGULFING()\n            if(calling_indicator == \"CDLEVENINGDOJISTAR\"): self.CDLEVENINGDOJISTAR()\n            if(calling_indicator == \"CDLEVENINGSTAR\"): self.CDLEVENINGSTAR()\n            if(calling_indicator == \"CDLGAPSIDESIDEWHITE\"): self.CDLGAPSIDESIDEWHITE()\n            if(calling_indicator == \"CDLGRAVESTONEDOJI\"): self.CDLGRAVESTONEDOJI()\n            if(calling_indicator == \"CDLHAMMER\"): self.CDLHAMMER()\n            if(calling_indicator == \"CDLHANGINGMAN\"): self.CDLHANGINGMAN()\n            if(calling_indicator == \"CDLHARAMI\"): self.CDLHARAMI()\n            if(calling_indicator == \"CDLHARAMICROSS\"): self.CDLHARAMICROSS()\n            if(calling_indicator == \"CDLHIGHWAVE\"): self.CDLHIGHWAVE()\n            if(calling_indicator == \"CDLHIKKAKE\"): self.CDLHIKKAKE()\n            if(calling_indicator == \"CDLHIKKAKEMOD\"): self.CDLHIKKAKEMOD()\n            if(calling_indicator == \"CDLHOMINGPIGEON\"): self.CDLHOMINGPIGEON()\n            if(calling_indicator == \"CDLIDENTICAL3CROWS\"): self.CDLIDENTICAL3CROWS()\n            if(calling_indicator == \"CDLINNECK\"): self.CDLINNECK()\n            if(calling_indicator == \"CDLINVERTEDHAMMER\"): self.CDLINVERTEDHAMMER()\n            if(calling_indicator == \"CDLKICKING\"): self.CDLKICKING()\n            if(calling_indicator == \"CDLKICKINGBYLENGTH\"): self.CDLKICKINGBYLENGTH()\n            if(calling_indicator == \"CDLLADDERBOTTOM\"): self.CDLLADDERBOTTOM()\n            if(calling_indicator == \"CDLLONGLEGGEDDOJI\"): self.CDLLONGLEGGEDDOJI()\n            if(calling_indicator == \"CDLLONGLINE\"): self.CDLLONGLINE()\n            if(calling_indicator == \"CDLMARUBOZU\"): self.CDLMARUBOZU()\n            if(calling_indicator == \"CDLMATCHINGLOW\"): self.CDLMATCHINGLOW()\n            if(calling_indicator == \"CDLMATHOLD\"): self.CDLMATHOLD()\n            if(calling_indicator == \"CDLMORNINGDOJISTAR\"): self.CDLMORNINGDOJISTAR()\n            if(calling_indicator == \"CDLMORNINGSTAR\"): self.CDLMORNINGSTAR()\n            if(calling_indicator == \"CDLONNECK\"): self.CDLONNECK()\n            if(calling_indicator == \"CDLPIERCING\"): self.CDLPIERCING()\n            if(calling_indicator == \"CDLRICKSHAWMAN\"): self.CDLRICKSHAWMAN()\n            if(calling_indicator == \"CDLRISEFALL3METHODS\"): self.CDLRISEFALL3METHODS()\n            if(calling_indicator == \"CDLSEPARATINGLINES\"): self.CDLSEPARATINGLINES()\n            if(calling_indicator == \"CDLSHOOTINGSTAR\"): self.CDLSHOOTINGSTAR()\n            if(calling_indicator == \"CDLSHORTLINE\"): self.CDLSHORTLINE()\n            if(calling_indicator == \"CDLSPINNINGTOP\"): self.CDLSPINNINGTOP()\n            if(calling_indicator == \"CDLSTALLEDPATTERN\"): self.CDLSTALLEDPATTERN()\n            if(calling_indicator == \"CDLSTICKSANDWICH\"): self.CDLSTICKSANDWICH()\n            if(calling_indicator == \"CDLTAKURI\"): self.CDLTAKURI()\n            if(calling_indicator == \"CDLTASUKIGAP\"): self.CDLTASUKIGAP()\n            if(calling_indicator == \"CDLTHRUSTING\"): self.CDLTHRUSTING()\n            if(calling_indicator == \"CDLTRISTAR\"): self.CDLTRISTAR()\n            if(calling_indicator == \"CDLUNIQUE3RIVER\"): self.CDLUNIQUE3RIVER()\n            if(calling_indicator == \"CDLUPSIDEGAP2CROWS\"): self.CDLUPSIDEGAP2CROWS()\n            if(calling_indicator == \"CDLXSIDEGAP3METHODS\"): self.CDLXSIDEGAP3METHODS()\n\n    def feature_process(self,df,headers) :\n        all_data_tmp=[]\n        cnt_row = 0\n        cnt_col = 0\n        all_head=[]\n\n        try :\n            self.technical_index=[]\n            self.technical_header=[]\n            self.total_cnt_indicator=0\n            ##補0\n            divider = 0.0\n            for i in range(df.shape[0]):\n                if (i==0):\n                    if (df[i, 1]!=0):\n                        divider=df[i, 1]\n                    else:\n                        raise ZeroDivisionError\n                    continue\n                if (df[i, 1] == 0): ##如果開盤價是0\n                    df[i, 1] = df[i-1, 4]\n                    df[i, 2] = df[i-1, 4]\n                    df[i, 3] = df[i-1, 4]\n                    df[i, 4] = df[i-1, 4]\n                elif (divider==0):\n                    divider=df[i, 1]\n                    \n            self.open  = df[:,1]/divider\n            self.high  = df[:,2]/divider\n            self.low   = df[:,3]/divider\n            self.close = df[:,4]/divider\n            self.volume = df[:,5]\n            self.technical_index = df\n            self.config_reading()\n            df = self.technical_index\n            all_data_tmp = np.append(all_data_tmp, df)\n            cnt_row += df.shape[0]\n            cnt_col = df.shape[1]\n        except IndexError :\n            print('資料有缺失在{}\\n'.format(file))\n        all_head = headers + self.technical_header\n\n        all_data= np.reshape(all_data_tmp, (cnt_row, cnt_col)) ##重新reshape\n        return all_data, all_head, self.total_cnt_indicator, cnt_col","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom scipy.stats import kurtosis\nfrom scipy.stats import skew\nfrom matplotlib import pyplot as plt\nfrom matplotlib.pyplot import MultipleLocator\nimport json\nfrom reg_model import regression_model\nimport os\nclass output_reg(regression_model):\n    def create_folder(self):\n        directory = \"output\"\n        parent_dir = self.directory\n        path_graph = os.path.join(parent_dir, directory)\n        os.mkdir(path_graph)\n        return path_graph\n\n    def statistics(self, data):\n        mean = np.mean(data)\n        std = np.std(data)\n        skewness = skew (data)\n        kurt = kurtosis(data)\n        return mean, std, skewness, kurt\n\n    def feature_details(self, path):\n        #create feature_details folder\n        directory = \"feature_details\"\n        parent_dir = path\n        path = os.path.join(parent_dir, directory)\n        os.mkdir(path)\n\n        X = self.x\n        Y = self.y\n        ##label detail\n        label = np.reshape(Y, (-1, 1))\n        mean = np.mean(label)\n        std = np.std(label)\n        skewness = skew(label)\n        kurt = kurtosis(label)\n        skewness = np.float64(skewness)\n        kurt = np.float64(kurt)\n        plt.hist(x=label, bins=20)\n        plt.title(\"Mean : {:.3f}, Std: {:.3f}, Skewness : {:.3f}, Kurtosis: {:.3f}\".format(mean, std, skewness, kurt))\n        plt.savefig(\"{}/label.png\".format(path))\n        self.plot_clear()\n        \n        ##feature detail\n        feature_result =[]\n        for feature in range(X.shape[1]):\n            mean = np.mean(X[:, feature])\n            std = np.std(X[:, feature])\n            skewness = skew (X[:, feature])\n            kurt = kurtosis(X[:, feature])\n            feature_result_tmp =[mean, std, skewness, kurt]\n            feature_result = np.append(feature_result, feature_result_tmp)\n        feature_result= np.reshape(feature_result, (X.shape[1], 4)) ##重新reshape\n\n        for feature in range(X.shape[1]):\n            mean = feature_result[feature,0]\n            std = feature_result[feature,1]\n            skewness = feature_result[feature,2]\n            kurt = feature_result[feature,3]\n            \n            x_distribution = (X[:,feature]- np.mean(X[:,feature]))/np.std(X[:,feature])\n            plt.hist(x_distribution,range=[-1, 1], bins=20) ##設定xmin 和 xmax才會畫得出來\n            plt.title(\"Mean : {:.3f}, Std: {:.3f}, Skewness : {:.3f}, Kurtosis: {:.3f}\".format(mean, std, skewness, kurt))\n            plt.savefig(\"{}/\".format(path)+self.headers[feature]+\".png\")\n            self.plot_clear()\n\n    def feature_label_time_series(self, path):\n        #create feature_label_time_series folder\n        directory = \"time_series\"\n        parent_dir = path\n        path_graph = os.path.join(parent_dir, directory)\n        os.mkdir(path_graph)\n\n        X = self.x\n        Y = self.y\n        ##label time series\n        self.plot_clear()\n        plt.plot(np.arange(0, X.shape[0], 1), Y)\n        plt.title(\"label\")\n        plt.savefig(\"{}/label.png\".format(path_graph))\n        self.plot_clear()\n        for feature in range(X.shape[1]):\n            self.plot_clear()\n            plt.plot(np.arange(0, X.shape[0], 1), X[:, feature])\n            plt.title(\"{}\".format(self.headers[feature+1]))\n            plt.savefig(\"{}/\".format(path_graph)+self.headers[feature+1]+\".png\")\n            self.plot_clear()\n\n    def output_graph(self, path):\n        #create graph and result folder\n        directory_graph = \"graph\"\n        directory_result = \"result\"\n        parent_dir = path\n        path_graph = os.path.join(parent_dir, directory_graph)\n        path_result = os.path.join(parent_dir, directory_result)\n        os.mkdir(path_graph)\n        os.mkdir(path_result)\n        with open(\"{}/output_reg.json\".format(path_result), \"w\") as outfile:\n            json.dump(self.dict, outfile, indent = 4)\n        ##index\n        train_index = range(0, self.train_y.shape[0])\n        test_index = range(0, self.test_y.shape[0])\n\n        ##linear regression \n        plt.scatter(self.train_y, self.y_pred_train[:,0], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/linear_regression_insample.png\".format(path_graph))\n        self.plot_clear()\n        plt.scatter(self.test_y, self.y_pred[:, 0], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/linear_regression_outsample.png\".format(path_graph))\n        self.plot_clear()\n\n        ## detail\n        mean, std, skewness, kurt = self.statistics(self.train_y)\n        self.insample_y_detail[\"Name\"].append(\"linear_regression\")\n        self.insample_y_detail[\"Mean\"].append(mean)\n        self.insample_y_detail[\"Std\"].append(std)\n        self.insample_y_detail[\"Skewness\"].append(skewness)\n        self.insample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred_train[:,0])\n        self.insample_predict_detail[\"Name\"].append(\"linear_regression\")\n        self.insample_predict_detail[\"Mean\"].append(mean)\n        self.insample_predict_detail[\"Std\"].append(std)\n        self.insample_predict_detail[\"Skewness\"].append(skewness)\n        self.insample_predict_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.test_y)\n        self.outsample_y_detail[\"Name\"].append(\"linear_regression\")\n        self.outsample_y_detail[\"Mean\"].append(mean)\n        self.outsample_y_detail[\"Std\"].append(std)\n        self.outsample_y_detail[\"Skewness\"].append(skewness)\n        self.outsample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred[:,0])\n        self.outsample_predict_detail[\"Name\"].append(\"linear_regression\")\n        self.outsample_predict_detail[\"Mean\"].append(mean)\n        self.outsample_predict_detail[\"Std\"].append(std)\n        self.outsample_predict_detail[\"Skewness\"].append(skewness)\n        self.outsample_predict_detail[\"Kurtosis\"].append(kurt)\n\n        ##linear regression predict distribution\n        insample_y_distribution = (self.train_y- np.mean(self.train_y))/np.std(self.train_y)\n        insample_predict_distribution = (self.y_pred_train[:,0]- np.mean(self.y_pred_train[:,0]))/np.std(self.y_pred_train[:,0])\n        plt.hist(insample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(insample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/linear_regression_insample_distribution.png\".format(path_graph))\n        self.plot_clear()\n        outsample_y_distribution = (self.test_y- np.mean(self.test_y))/np.std(self.test_y)\n        outsample_predict_distribution = (self.y_pred[:,0]- np.mean(self.y_pred[:,0]))/np.std(self.y_pred[:,0])\n        plt.hist(outsample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(outsample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/linear_regression_outsample_distribution.png\".format(path_graph))\n        self.plot_clear()\n\n        ##lasso\n        plt.scatter(self.train_y, self.y_pred_train[:, 1], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/lasso_insample.png\".format(path_graph))\n        self.plot_clear()\n        plt.scatter(self.test_y, self.y_pred[:, 1], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/lasso_outsample.png\".format(path_graph))\n        self.plot_clear()\n\n        ## detail\n        mean, std, skewness, kurt = self.statistics(self.train_y)\n        self.insample_y_detail[\"Name\"].append(\"lasso\")\n        self.insample_y_detail[\"Mean\"].append(mean)\n        self.insample_y_detail[\"Std\"].append(std)\n        self.insample_y_detail[\"Skewness\"].append(skewness)\n        self.insample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred_train[:, 1])\n        self.insample_predict_detail[\"Name\"].append(\"lasso\")\n        self.insample_predict_detail[\"Mean\"].append(mean)\n        self.insample_predict_detail[\"Std\"].append(std)\n        self.insample_predict_detail[\"Skewness\"].append(skewness)\n        self.insample_predict_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.test_y)\n        self.outsample_y_detail[\"Name\"].append(\"lasso\")\n        self.outsample_y_detail[\"Mean\"].append(mean)\n        self.outsample_y_detail[\"Std\"].append(std)\n        self.outsample_y_detail[\"Skewness\"].append(skewness)\n        self.outsample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred[:, 1])\n        self.outsample_predict_detail[\"Name\"].append(\"lasso\")\n        self.outsample_predict_detail[\"Mean\"].append(mean)\n        self.outsample_predict_detail[\"Std\"].append(std)\n        self.outsample_predict_detail[\"Skewness\"].append(skewness)\n        self.outsample_predict_detail[\"Kurtosis\"].append(kurt)\n\n        ##lasso predict distribution\n        insample_y_distribution = (self.train_y- np.mean(self.train_y))/np.std(self.train_y)\n        insample_predict_distribution = (self.y_pred_train[:,1]- np.mean(self.y_pred_train[:,1]))/np.std(self.y_pred_train[:,1])\n        plt.hist(insample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(insample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/lasso_insample_distribution.png\".format(path_graph))\n        self.plot_clear()\n        outsample_y_distribution = (self.test_y- np.mean(self.test_y))/np.std(self.test_y)\n        outsample_predict_distribution = (self.y_pred[:,1]- np.mean(self.y_pred[:,1]))/np.std(self.y_pred[:,1])\n        plt.hist(outsample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(outsample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/lasso_outsample_distribution.png\".format(path_graph))\n        self.plot_clear()\n\n        ##ridge\n        plt.scatter(self.train_y, self.y_pred_train[:, 2], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/ridge_insample.png\".format(path_graph))\n        self.plot_clear()\n        plt.scatter(self.test_y, self.y_pred[:, 2], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/ridge_outsample.png\".format(path_graph))\n        self.plot_clear()\n\n        ## detail\n        mean, std, skewness, kurt = self.statistics(self.train_y)\n        self.insample_y_detail[\"Name\"].append(\"ridge\")\n        self.insample_y_detail[\"Mean\"].append(mean)\n        self.insample_y_detail[\"Std\"].append(std)\n        self.insample_y_detail[\"Skewness\"].append(skewness)\n        self.insample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred_train[:, 2])\n        self.insample_predict_detail[\"Name\"].append(\"ridge\")\n        self.insample_predict_detail[\"Mean\"].append(mean)\n        self.insample_predict_detail[\"Std\"].append(std)\n        self.insample_predict_detail[\"Skewness\"].append(skewness)\n        self.insample_predict_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.test_y)\n        self.outsample_y_detail[\"Name\"].append(\"ridge\")\n        self.outsample_y_detail[\"Mean\"].append(mean)\n        self.outsample_y_detail[\"Std\"].append(std)\n        self.outsample_y_detail[\"Skewness\"].append(skewness)\n        self.outsample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred[:, 2])\n        self.outsample_predict_detail[\"Name\"].append(\"ridge\")\n        self.outsample_predict_detail[\"Mean\"].append(mean)\n        self.outsample_predict_detail[\"Std\"].append(std)\n        self.outsample_predict_detail[\"Skewness\"].append(skewness)\n        self.outsample_predict_detail[\"Kurtosis\"].append(kurt)\n\n        ##ridge predict distribution\n        insample_y_distribution = (self.train_y- np.mean(self.train_y))/np.std(self.train_y)\n        insample_predict_distribution = (self.y_pred_train[:,2]- np.mean(self.y_pred_train[:,2]))/np.std(self.y_pred_train[:,2])\n        plt.hist(insample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(insample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/ridge_insample_distribution.png\".format(path_graph))\n        self.plot_clear()\n        outsample_y_distribution = (self.test_y- np.mean(self.test_y))/np.std(self.test_y)\n        outsample_predict_distribution = (self.y_pred[:,2]- np.mean(self.y_pred[:,2]))/np.std(self.y_pred[:,2])\n        plt.hist(outsample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(outsample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/ridge_outsample_distribution.png\".format(path_graph))\n        self.plot_clear()\n\n        ##random_forest\n        plt.scatter(self.train_y, self.y_pred_train[:, 3], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/random_forest_insample.png\".format(path_graph))\n        self.plot_clear()\n        plt.scatter(self.test_y, self.y_pred[:, 3], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/random_forest_outsample.png\".format(path_graph))\n        self.plot_clear()\n\n        ## detail\n        mean, std, skewness, kurt = self.statistics(self.train_y)\n        self.insample_y_detail[\"Name\"].append(\"random_forest\")\n        self.insample_y_detail[\"Mean\"].append(mean)\n        self.insample_y_detail[\"Std\"].append(std)\n        self.insample_y_detail[\"Skewness\"].append(skewness)\n        self.insample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred_train[:, 3])\n        self.insample_predict_detail[\"Name\"].append(\"random_forest\")\n        self.insample_predict_detail[\"Mean\"].append(mean)\n        self.insample_predict_detail[\"Std\"].append(std)\n        self.insample_predict_detail[\"Skewness\"].append(skewness)\n        self.insample_predict_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.test_y)\n        self.outsample_y_detail[\"Name\"].append(\"random_forest\")\n        self.outsample_y_detail[\"Mean\"].append(mean)\n        self.outsample_y_detail[\"Std\"].append(std)\n        self.outsample_y_detail[\"Skewness\"].append(skewness)\n        self.outsample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred[:, 3])\n        self.outsample_predict_detail[\"Name\"].append(\"random_forest\")\n        self.outsample_predict_detail[\"Mean\"].append(mean)\n        self.outsample_predict_detail[\"Std\"].append(std)\n        self.outsample_predict_detail[\"Skewness\"].append(skewness)\n        self.outsample_predict_detail[\"Kurtosis\"].append(kurt)\n\n        ##ridge predict distribution\n        insample_y_distribution = (self.train_y- np.mean(self.train_y))/np.std(self.train_y)\n        insample_predict_distribution = (self.y_pred_train[:,3]- np.mean(self.y_pred_train[:,3]))/np.std(self.y_pred_train[:,3])\n        plt.hist(insample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(insample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/random_forest_insample_distribution.png\".format(path_graph))\n        self.plot_clear()\n        outsample_y_distribution = (self.test_y- np.mean(self.test_y))/np.std(self.test_y)\n        outsample_predict_distribution = (self.y_pred[:,3]- np.mean(self.y_pred[:,3]))/np.std(self.y_pred[:,3])\n        plt.hist(outsample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(outsample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/random_forest_outsample_distribution.png\".format(path_graph))\n        self.plot_clear()\n\n        ##lgbm\n        plt.scatter(self.train_y, self.y_pred_train[:, 4], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/lgbm_insample.png\".format(path_graph))\n        self.plot_clear()\n        plt.scatter(self.test_y, self.y_pred[:, 4], c=\"red\", s=10)\n        plt.plot([-0.02, 0.02], [-0.02, 0.02], '--', color = 'black', linewidth = 2)\n        plt.savefig(\"{}/lgbm_outsample.png\".format(path_graph))\n        self.plot_clear()\n\n        ## detail\n        mean, std, skewness, kurt = self.statistics(self.train_y)\n        self.insample_y_detail[\"Name\"].append(\"lgbm\")\n        self.insample_y_detail[\"Mean\"].append(mean)\n        self.insample_y_detail[\"Std\"].append(std)\n        self.insample_y_detail[\"Skewness\"].append(skewness)\n        self.insample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred_train[:, 4])\n        self.insample_predict_detail[\"Name\"].append(\"lgbm\")\n        self.insample_predict_detail[\"Mean\"].append(mean)\n        self.insample_predict_detail[\"Std\"].append(std)\n        self.insample_predict_detail[\"Skewness\"].append(skewness)\n        self.insample_predict_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.test_y)\n        self.outsample_y_detail[\"Name\"].append(\"lgbm\")\n        self.outsample_y_detail[\"Mean\"].append(mean)\n        self.outsample_y_detail[\"Std\"].append(std)\n        self.outsample_y_detail[\"Skewness\"].append(skewness)\n        self.outsample_y_detail[\"Kurtosis\"].append(kurt)\n        mean, std, skewness, kurt = self.statistics(self.y_pred[:, 4])\n        self.outsample_predict_detail[\"Name\"].append(\"lgbm\")\n        self.outsample_predict_detail[\"Mean\"].append(mean)\n        self.outsample_predict_detail[\"Std\"].append(std)\n        self.outsample_predict_detail[\"Skewness\"].append(skewness)\n        self.outsample_predict_detail[\"Kurtosis\"].append(kurt)\n\n        ##lgbm predict distribution\n        insample_y_distribution = (self.train_y- np.mean(self.train_y))/np.std(self.train_y)\n        insample_predict_distribution = (self.y_pred_train[:,4]- np.mean(self.y_pred_train[:,4]))/np.std(self.y_pred_train[:,4])\n\n        plt.hist(insample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(insample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/lgbm_insample_distribution.png\".format(path_graph))\n        self.plot_clear()\n        outsample_y_distribution = (self.test_y- np.mean(self.test_y))/np.std(self.test_y)\n        outsample_predict_distribution = (self.y_pred[:,4]- np.mean(self.y_pred[:,4]))/np.std(self.y_pred[:,4])\n        plt.hist(outsample_y_distribution, alpha=0.5, label='Pred', color = \"skyblue\", bins=20)\n        plt.hist(outsample_predict_distribution, alpha=0.5, label='True', color = \"palegreen\", bins=20)\n        plt.legend(loc='upper right')\n        plt.savefig(\"{}/lgbm_outsample_distribution.png\".format(path_graph))\n        self.plot_clear()\n\n        with open(\"{}/insample_y_statistics.json\".format(parent_dir), \"w\") as outfile:\n            json.dump(self.insample_y_detail, outfile, indent = 4)\n        with open(\"{}/insample_predict_statistics.json\".format(parent_dir), \"w\") as outfile:\n            json.dump(self.insample_predict_detail, outfile, indent = 4)\n        with open(\"{}/outsample_y_statistics.json\".format(parent_dir), \"w\") as outfile:\n            json.dump(self.outsample_y_detail, outfile, indent = 4)\n        with open(\"{}/outsample_predict_statistics.json\".format(parent_dir), \"w\") as outfile:\n            json.dump(self.outsample_predict_detail, outfile, indent = 4)\n\n    def plot_clear(self):\n        plt.figure().clear()\n        plt.close()\n        plt.cla()\n        plt.clf()\n\n    def show_result(self):\n        super().train()\n        path = self.create_folder()\n        self.feature_details(path)\n        self.feature_label_time_series(path)\n        self.output_graph(path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom datetime import datetime\nimport time\nimport os","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" lib = [\"HT_DCPERIOD\",\t\"HT_DCPHASE\",\t\"HT_PHASOR_inphase\",\t\"HT_PHASOR_quadrature\",\t\"HT_SINE_sine\",\t\n    \"HT_SINE_leadsine\",\t\"HT_TRENDMODE\",\t\"ADX\",\t\"ADXR\",\t\"APO\",\t\"AROON_aroondown\",\t\"AROON_aroonup\",\t\"AROONOSC\",\t\n    \"BOP\",\t\"CCI\",\t\"CMO\",\t\"DX\",\t\"MACD_macd\",\t\"MACD_macdsignal\",\t\"MACD_macdhist\", \n    \"MFI\",\t\"MINUS_DI\",\t\"MINUS_DM\",\t\" SMA\",\t\"upper\",\t\"middle\",\t\" lower\",\t\"DEMA\",\t\"EMA\",\t\"KAMA\",\t\"MA\",\t\"MIDPOINT\",\t\n    \"SAR\",\t\"SAREXT\",\t\"T3\",\t\"TEMA\",\t\"WMA\",\t\"CDL2CROWS\",\t\"CDL3BLACKCROWS\",\t\"CDL3INSIDE\",\t\"CDL3LINESTRIKE\",\t\" CDL3OUTSIDE\",\t\n    \"CDL3STARSINSOUTH\",\t\"CDL3WHITESOLDIERS\",\t\"CDLABANDONEDBABY\",\t\"CDLADVANCEBLOCK\",\t\"CDLBELTHOLD\",\t\"CDLBREAKAWAY\",\t\"CDLCLOSINGMARUBOZU\",\n    \"CDLCONCEALBABYSWALL\",\t\"CDLCOUNTERATTACK\",\t\"CDLDARKCLOUDCOVER\", \"CDLDOJI\",\t\"CDLDOJISTAR\",\t\"CDLDRAGONFLYDOJI\",\t\"CDLENGULFING\",\t\"CDLEVENINGDOJISTAR\",\t\n    \"CDLEVENINGSTAR\",\t\"CDLGAPSIDESIDEWHITE\",\t\"CDLGRAVESTONEDOJI\",\t\"CDLHAMMER\",\t\"CDLHANGINGMAN\",\n        \"CDLHARAMI\",\t\"CDLHARAMICROSS\",\t\"CDLHIGHWAVE\",\t\"CDLHIKKAKE\",\t\n    \"CDLHIKKAKEMOD\",\t\"CDLHOMINGPIGEON\",\t\"CDLIDENTICAL3CROWS\",\t\"CDLINNECK\",\t\"CDLINVERTEDHAMMER\",\t\"CDLKICKING\",\t\"CDLKICKINGBYLENGTH\",\t\"CDLLADDERBOTTOM\",\t\"CDLLONGLEGGEDDOJI\",\t\n    \"CDLLONGLINE\",\t\"CDLMARUBOZU\",\t\"CDLMATCHINGLOW\",\t\"CDLMATHOLD\",\t\"CDLMORNINGDOJISTAR\",\t\"CDLMORNINGSTAR\",\t\"CDLONNECK\",\t\"CDLPIERCING\",\t\"CDLRICKSHAWMAN\",\t\"CDLRISEFALL3METHODS\",\t\n    \"CDLSEPARATINGLINES\",\t\"CDLSHOOTINGSTAR\",\t\"CDLSHORTLINE\",\t\"CDLSPINNINGTOP\",\t\" CDLSTALLEDPATTERN\",\t\"CDLSTICKSANDWICH\",\t\"CDLTAKURI\",\t\"CDLTASUKIGAP\",\t\"CDLTHRUSTING\",\t\"CDLTRISTAR\",\n    \t\"CDLUNIQUE3RIVER\",\t\"CDLUPSIDEGAP2CROWS\",\t\"CDLXSIDEGAP3METHODS\"]\n\nfeature_gen = get_indicator(lib)\ndata, headers, total_indicator, total_row = feature_gen.feature_process(df_train_00.to_numpy(),df_train_00.columns.to_list())\nprint(data.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(len(headers)):\n#     print(headers[i],\":\",np.isnan(data[:,i]).sum())\n# drop nan\ndata = data[~np.isnan(data).any(axis=1), :]\nprint(data.shape)\ndata = np.reshape(data, (-1, total_row))\nprint(data.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nnow = datetime.now()\ncurrent_time = str(now.month)+\"_\"+str(now.day)+\"_\"+now.strftime(\"%H_%M_%S\")\nparent_dir = \"/kaggle/working/\"\ndirectory = current_time\npath = os.path.join(parent_dir, directory)\nos.mkdir(path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"candle_tick = output_reg(data, headers, path)\ncandle_tick.show_result()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}