{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-03T18:27:27.750606Z","iopub.execute_input":"2021-12-03T18:27:27.750948Z","iopub.status.idle":"2021-12-03T18:27:27.763575Z","shell.execute_reply.started":"2021-12-03T18:27:27.750914Z","shell.execute_reply":"2021-12-03T18:27:27.762878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install talib-binary","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:26:54.873422Z","iopub.execute_input":"2021-12-03T18:26:54.874112Z","iopub.status.idle":"2021-12-03T18:27:05.920150Z","shell.execute_reply.started":"2021-12-03T18:26:54.873992Z","shell.execute_reply":"2021-12-03T18:27:05.919221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import talib","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:27:05.921993Z","iopub.execute_input":"2021-12-03T18:27:05.922240Z","iopub.status.idle":"2021-12-03T18:27:05.938316Z","shell.execute_reply.started":"2021-12-03T18:27:05.922213Z","shell.execute_reply":"2021-12-03T18:27:05.937341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nimport time\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:27:10.758293Z","iopub.execute_input":"2021-12-03T18:27:10.758577Z","iopub.status.idle":"2021-12-03T18:27:10.763653Z","shell.execute_reply.started":"2021-12-03T18:27:10.758548Z","shell.execute_reply":"2021-12-03T18:27:10.762986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"../input/g-research-crypto-forecasting/\"\ncrypto_df = pd.read_csv(data_folder + 'train.csv')\nasset_details_df = pd.read_csv(data_folder + 'asset_details.csv')\nsupp_train_df = pd.read_csv(data_folder + 'supplemental_train.csv')\nexpl_test = pd.read_csv(data_folder + 'example_test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:28:24.745918Z","iopub.execute_input":"2021-12-03T18:28:24.746379Z","iopub.status.idle":"2021-12-03T18:29:17.859956Z","shell.execute_reply.started":"2021-12-03T18:28:24.746340Z","shell.execute_reply":"2021-12-03T18:29:17.859039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details_df[\"Asset_Name\"] = asset_details_df[\"Asset_Name\"].str.replace(' ','_')\nasset_details_df[\"Asset_Name\"] = asset_details_df[\"Asset_Name\"].str.replace('.','_')","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:29:17.861626Z","iopub.execute_input":"2021-12-03T18:29:17.861903Z","iopub.status.idle":"2021-12-03T18:29:17.874728Z","shell.execute_reply.started":"2021-12-03T18:29:17.861873Z","shell.execute_reply":"2021-12-03T18:29:17.873909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details = asset_details_df.copy()\nasset_details_df = asset_details_df.set_index(\"Asset_ID\")","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:29:56.794358Z","iopub.execute_input":"2021-12-03T18:29:56.795082Z","iopub.status.idle":"2021-12-03T18:29:56.803456Z","shell.execute_reply.started":"2021-12-03T18:29:56.795038Z","shell.execute_reply":"2021-12-03T18:29:56.802519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:30:17.744969Z","iopub.execute_input":"2021-12-03T18:30:17.745581Z","iopub.status.idle":"2021-12-03T18:30:17.774637Z","shell.execute_reply.started":"2021-12-03T18:30:17.745548Z","shell.execute_reply":"2021-12-03T18:30:17.773966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create a dictionary of data frames assigned to each coin \ndataframes = {}\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):    \n    vars()[asset_name] = crypto_df[crypto_df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")#.merge(asset_details, how='left', on='Asset_ID')\n    dataframes[asset_id] = vars()[asset_name]\n\n\ncleaned_dataframes = {}\nfor i in list(dataframes):\n    #cols = ['Asset_ID','Asset_Name','Count','Volume','Open','High','Low', 'Close','VWAP','Target']\n    cleaned_dataframes[i] = dataframes[i].reindex(range(dataframes[i].index[0],dataframes[i].index[-1]+60,60),method='pad')\n    #cleaned_dataframes[i] = cleaned_dataframes[i][cols]\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:30:22.037782Z","iopub.execute_input":"2021-12-03T18:30:22.038064Z","iopub.status.idle":"2021-12-03T18:30:34.391667Z","shell.execute_reply.started":"2021-12-03T18:30:22.038037Z","shell.execute_reply":"2021-12-03T18:30:34.390693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Reduce the datastes to ease the correlation calculation\nreduced_dataframes = {}\nfor i in list(dataframes):\n    #reduced_dataframes[i] = cleaned_dataframes[i].loc[totimestamp('01/01/2021'):totimestamp('01/05/2021')]\n    reduced_dataframes[i] = cleaned_dataframes[i].iloc[-10000:]","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:30:34.393214Z","iopub.execute_input":"2021-12-03T18:30:34.393446Z","iopub.status.idle":"2021-12-03T18:30:34.399088Z","shell.execute_reply.started":"2021-12-03T18:30:34.393412Z","shell.execute_reply":"2021-12-03T18:30:34.398197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduced_dataframes_base = reduced_dataframes\nreduced_dataframes_base[1]","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:30:37.196618Z","iopub.execute_input":"2021-12-03T18:30:37.196920Z","iopub.status.idle":"2021-12-03T18:30:37.218078Z","shell.execute_reply.started":"2021-12-03T18:30:37.196890Z","shell.execute_reply":"2021-12-03T18:30:37.217134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_columns = cleaned_dataframes[2].columns.tolist()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:30:43.756671Z","iopub.execute_input":"2021-12-03T18:30:43.756984Z","iopub.status.idle":"2021-12-03T18:30:43.761521Z","shell.execute_reply.started":"2021-12-03T18:30:43.756953Z","shell.execute_reply":"2021-12-03T18:30:43.760596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here let's generate each Technical indicators grouped by the type of the indicator","metadata":{}},{"cell_type":"code","source":"momentum_indicators_columns = ['adx_mm','adxr_mm','aroondown_mm','aroonup_mm','bop_mm','mfi_mm','cci_mm','cmo_mm','rsi_mm','mdi_mm','pdi_mm','mdm_mm','pdm_mm','dx_mm','roc_mm','rocp_mm','will_mm']\ndef get_momentum_indicators(df) :\n    \n    df[\"adx_mm\"] = talib.ADX(df['High'], df['Low'], df['Close'], timeperiod=14) #Average Directional Movement Index\n    df[\"adxr_mm\"] = talib.ADXR(df['High'], df['Low'], df['Close'], timeperiod=14) #Average Directional Movement Index Rating\n    df[\"aroondown_mm\"], df[\"aroonup_mm\"] = talib.AROON(df['High'], df['Low'],timeperiod=14) \n    df[\"bop_mm\"] = talib.BOP(df['Open'], df['High'], df['Low'], df['Close']) #Balance Of Power\n    df[\"mfi_mm\"] = talib.MFI(df['High'], df['Low'], df['Close'], df['Volume'], timeperiod=14) #Money Flow Index\n    df[\"cci_mm\"] = talib.CCI(df['High'], df['Low'], df['Close'], timeperiod=14) #Commodity Channel Index\n    df[\"cmo_mm\"] = talib.CMO(df['Close'], timeperiod=14) #Chande Momentum Oscillator\n    df[\"rsi_mm\"] = talib.RSI(df['Close'], timeperiod=14) #Relative Strenght Index\n    df[\"mdi_mm\"] = talib.MINUS_DI(df['High'], df['Low'], df['Close'], timeperiod=14) #Minus Directional Indicator\n    df[\"pdi_mm\"] = talib.PLUS_DI(df['High'], df['Low'], df['Close'], timeperiod=14)  #Plus Directional Indicator\n    df[\"mdm_mm\"] = talib.MINUS_DM(df['High'], df['Low'],timeperiod=14)  #Minus Directional Movement\n    df[\"pdm_mm\"] = talib.PLUS_DM(df['High'], df['Low'],timeperiod=14)  #Plus Directional Movement\n    df[\"dx_mm\"] = talib.DX(df['High'], df['Low'], df['Close'], timeperiod=14) #Directional Movement Index\n    df[\"roc_mm\"] = talib.ROC(df['Close'], timeperiod=10) #Rate of change\n    df[\"rocp_mm\"] = talib.ROCP(df['Close'], timeperiod=10) #Rate of change Percentage\n    df[\"will_mm\"] = talib.WILLR(df['High'], df['Low'], df['Close'], timeperiod=14)  #Williams' %R\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:30:49.209407Z","iopub.execute_input":"2021-12-03T18:30:49.209730Z","iopub.status.idle":"2021-12-03T18:30:49.224668Z","shell.execute_reply.started":"2021-12-03T18:30:49.209695Z","shell.execute_reply":"2021-12-03T18:30:49.223660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"volatility_indicators_columns = ['atr_vlt','natr_vlt','trange_vlt']\ndef get_volatility_indicators(df) :\n    \n    df['atr_vlt'] = talib.ATR(df['High'], df['Low'], df['Close'], timeperiod=14)\n    df['natr_vlt'] = talib.NATR(df['High'], df['Low'], df['Close'], timeperiod=14)\n    df['trange_vlt'] = talib.TRANGE(df['High'], df['Low'], df['Close'])","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:30:54.057662Z","iopub.execute_input":"2021-12-03T18:30:54.057989Z","iopub.status.idle":"2021-12-03T18:30:54.063655Z","shell.execute_reply.started":"2021-12-03T18:30:54.057954Z","shell.execute_reply":"2021-12-03T18:30:54.062989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"volume_indicators_columns = ['ad_vol','adosc_vol','obv_vol']\ndef get_volume_indicators(df) :\n    \n    df['ad_vol'] = talib.AD(df['High'], df['Low'], df['Close'], df['Volume'])\n    df['adosc_vol'] = talib.ADOSC(df['High'], df['Low'], df['Close'], df['Volume'], fastperiod=5, slowperiod=20)\n    df['obv_vol'] = talib.OBV(df['Close'], df['Volume'])","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:30:57.387633Z","iopub.execute_input":"2021-12-03T18:30:57.388081Z","iopub.status.idle":"2021-12-03T18:30:57.394898Z","shell.execute_reply.started":"2021-12-03T18:30:57.388049Z","shell.execute_reply":"2021-12-03T18:30:57.393963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cycle_indicators_columns = ['htdcpe_cyc','htdcpa_cyc','htsn_cyc','htinph_cyc','httrnd_cyc']\ndef get_cycle_indicators(df) :\n    \n    df['htdcpe_cyc'] = talib.HT_DCPERIOD(df['Close'])\n    df['htdcpa_cyc'] = talib.HT_DCPHASE(df['Close'])\n    df['htsn_cyc'], df['htldsn_cyc'] = talib.HT_SINE(df['Close'])\n    df['htinph_cyc'], df['htquad_cyc'] = talib.HT_PHASOR(df['Close'])\n    df['httrnd_cyc'] = talib.HT_TRENDLINE(df['Close'])","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:31:04.246475Z","iopub.execute_input":"2021-12-03T18:31:04.246905Z","iopub.status.idle":"2021-12-03T18:31:04.253624Z","shell.execute_reply.started":"2021-12-03T18:31:04.246873Z","shell.execute_reply":"2021-12-03T18:31:04.253005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"statitics_functions_columns = ['beta_stat','correl_stat','stddev_stat','tsf_stat','var_stat']\ndef get_statitics_functions(df) :\n    \n    df['beta_stat'] = talib.BETA(df['High'], df['Low'], timeperiod=5)\n    df['correl_stat'] = talib.CORREL(df['High'], df['Low'], timeperiod=30)\n    df['stddev_stat'] = talib.STDDEV(df['Close'], timeperiod=5, nbdev=1)\n    df['tsf_stat'] = talib.TSF(df['Close'], timeperiod=14)\n    df['var_stat'] = talib.VAR(df['Close'], timeperiod=5, nbdev=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:31:08.495100Z","iopub.execute_input":"2021-12-03T18:31:08.495624Z","iopub.status.idle":"2021-12-03T18:31:08.502006Z","shell.execute_reply.started":"2021-12-03T18:31:08.495575Z","shell.execute_reply":"2021-12-03T18:31:08.501216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"overlap_functions_columns = ['bbup_ovlp','kama_ovlp','sar_ovlp']\ndef get_overlap_functions(df) :\n    \n    df['bbup_ovlp'], df['bbmidl_ovlp'], df['bblow_ovlp'] = talib.BBANDS(df['Close'], timeperiod=10, nbdevup=2, nbdevdn=2, matype=0)\n    df['kama_ovlp']= talib.KAMA(df['Close'], timeperiod=30)\n    #df['mama_ovlp'], df['fama'] = talib.MAMA(df['Close'], fastlimit=0, slowlimit=0)\n    df['sar_ovlp']= talib.SAR(df['High'], df['Low'], acceleration=0, maximum=0)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:31:13.516509Z","iopub.execute_input":"2021-12-03T18:31:13.516851Z","iopub.status.idle":"2021-12-03T18:31:13.523155Z","shell.execute_reply.started":"2021-12-03T18:31:13.516816Z","shell.execute_reply":"2021-12-03T18:31:13.521950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"price_transformations = ['avg_price','med_price','typ_price','weight_price']\ndef get_price_transformations(df) :\n    \n    df['avg_price'] = talib.AVGPRICE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['med_price'] = talib.MEDPRICE(df['High'], df['Low'])\n    df['typ_price'] = talib.TYPPRICE(df['High'], df['Low'], df['Close'])\n    df['weight_price'] = talib.WCLPRICE(df['High'], df['Low'], df['Close'])","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:31:17.883073Z","iopub.execute_input":"2021-12-03T18:31:17.884203Z","iopub.status.idle":"2021-12-03T18:31:17.891434Z","shell.execute_reply.started":"2021-12-03T18:31:17.884142Z","shell.execute_reply":"2021-12-03T18:31:17.890809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pattern_recognition_columns = ['cdl2crows',\n       'cdl3blackcrows', 'cdl3inside', 'cdl3linestrike', 'cdl3outside',\n       'cdl3starsinsouth', 'cdl3whitesoldiers', 'cdlabandonedbaby',\n       'cdladvanceblock', 'cdlbelthold', 'cdlbreakaway', 'cdlclosingmarubozu',\n       'cdlconcealbabyswall', 'cdlcounterattack', 'cdldarkcloudcover',\n       'cdldoji', 'cdldojistar', 'cdldragonflydoji', 'cdlengulfing',\n       'cdleveningdojistar', 'cdleveningstar', 'cdlgapsidesidewhite',\n       'cdlgravestonedoji', 'cdlhammer', 'cdlhangingman', 'cdlharami',\n       'cdlharamicross', 'cdlhighwave', 'cdlhikkake', 'cdlhikkakemod',\n       'cdlhomingpigeon', 'cdlidentical3crows', 'cdlinneck',\n       'cdlinvertedhammer', 'cdlkicking', 'cdlkickingbylength',\n       'cdlladderbottom', 'cdllongleggeddoji', 'cdllongline', 'cdlmarubozu',\n       'cdlmatchinglow', 'cdlmathold', 'cdlmorningdojistar', 'cdlmorningstar',\n       'cdlonneck', 'cdlpiercing', 'cdlrickshawman', 'cdlrisefall3methods',\n       'cdlseparatinglines', 'cdlshootingstar', 'cdlshortline',\n       'cdlspinningtop', 'cdlstalledpattern', 'cdlsticksandwich', 'cdltakuri',\n       'cdltasukigap', 'cdlthrusting', 'cdltristar', 'cdlunique3river',\n       'cdlupsidegap2crows', 'cdlxsidegap3methods']\n\ndef get_pattern_recognition(df) :\n    \n    df['cdl2crows'] = talib.CDL2CROWS(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdl3blackcrows'] = talib.CDL3BLACKCROWS(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdl3blackcrows'] = talib.CDL3BLACKCROWS(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdl3inside'] = talib.CDL3INSIDE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdl3linestrike'] = talib.CDL3LINESTRIKE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdl3outside'] = talib.CDL3OUTSIDE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdl3starsinsouth'] = talib.CDL3STARSINSOUTH(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdl3whitesoldiers'] = talib.CDL3WHITESOLDIERS(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlabandonedbaby'] = talib.CDLABANDONEDBABY(df['Open'], df['High'], df['Low'], df['Close'], penetration=0)\n    df['cdladvanceblock'] = talib.CDLADVANCEBLOCK(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlbelthold'] = talib.CDLBELTHOLD(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlbreakaway'] = talib.CDLBREAKAWAY(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlclosingmarubozu'] = talib.CDLCLOSINGMARUBOZU(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlconcealbabyswall'] = talib.CDLCONCEALBABYSWALL(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlcounterattack'] = talib.CDLCOUNTERATTACK(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdldarkcloudcover'] = talib.CDLDARKCLOUDCOVER(df['Open'], df['High'], df['Low'], df['Close'], penetration=0)\n    df['cdldoji'] = talib.CDLDOJI(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdldojistar'] = talib.CDLDOJISTAR(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdldragonflydoji'] = talib.CDLDRAGONFLYDOJI(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlengulfing'] = talib.CDLENGULFING(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdleveningdojistar'] = talib.CDLEVENINGDOJISTAR(df['Open'], df['High'], df['Low'], df['Close'], penetration=0)\n    df['cdleveningstar'] = talib.CDLEVENINGSTAR(df['Open'], df['High'], df['Low'], df['Close'], penetration=0)\n    df['cdlgapsidesidewhite'] = talib.CDLGAPSIDESIDEWHITE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlgravestonedoji'] = talib.CDLGRAVESTONEDOJI(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlhammer'] = talib.CDLHAMMER(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlhangingman'] = talib.CDLHANGINGMAN(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlharami'] = talib.CDLHARAMI(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlharamicross'] = talib.CDLHARAMICROSS(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlhighwave'] = talib.CDLHIGHWAVE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlhikkake'] = talib.CDLHIKKAKE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlhikkakemod'] = talib.CDLHIKKAKEMOD(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlhomingpigeon'] = talib.CDLHOMINGPIGEON(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlidentical3crows'] = talib.CDLIDENTICAL3CROWS(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlinneck'] = talib.CDLINNECK(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlinvertedhammer'] = talib.CDLINVERTEDHAMMER(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlkicking'] = talib.CDLKICKING(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlkickingbylength'] = talib.CDLKICKINGBYLENGTH(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlladderbottom'] = talib.CDLLADDERBOTTOM(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdllongleggeddoji'] = talib.CDLLONGLEGGEDDOJI(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdllongline'] = talib.CDLLONGLINE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlmarubozu'] = talib.CDLMARUBOZU(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlmatchinglow'] = talib.CDLMATCHINGLOW(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlmathold'] = talib.CDLMATHOLD(df['Open'], df['High'], df['Low'], df['Close'], penetration=0)\n    df['cdlmorningdojistar'] = talib.CDLMORNINGDOJISTAR(df['Open'], df['High'], df['Low'], df['Close'], penetration=0)\n    df['cdlmorningstar'] = talib.CDLMORNINGSTAR(df['Open'], df['High'], df['Low'], df['Close'], penetration=0)\n    df['cdlonneck'] = talib.CDLONNECK(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlpiercing'] = talib.CDLPIERCING(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlrickshawman'] = talib.CDLRICKSHAWMAN(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlrisefall3methods'] = talib.CDLRISEFALL3METHODS(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlseparatinglines'] = talib.CDLSEPARATINGLINES(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlshootingstar'] = talib.CDLSHOOTINGSTAR(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlshortline'] = talib.CDLSHORTLINE(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlspinningtop'] = talib.CDLSPINNINGTOP(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlstalledpattern'] = talib.CDLSTALLEDPATTERN(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlsticksandwich'] = talib.CDLSTICKSANDWICH(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdltakuri'] = talib.CDLTAKURI(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdltasukigap'] = talib.CDLTASUKIGAP(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlthrusting'] = talib.CDLTHRUSTING(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdltristar'] = talib.CDLTRISTAR(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlunique3river'] = talib.CDLUNIQUE3RIVER(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlupsidegap2crows'] = talib.CDLUPSIDEGAP2CROWS(df['Open'], df['High'], df['Low'], df['Close'])\n    df['cdlxsidegap3methods'] = talib.CDLXSIDEGAP3METHODS(df['Open'], df['High'], df['Low'], df['Close'])","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:31:32.569342Z","iopub.execute_input":"2021-12-03T18:31:32.569663Z","iopub.status.idle":"2021-12-03T18:31:32.619178Z","shell.execute_reply.started":"2021-12-03T18:31:32.569632Z","shell.execute_reply":"2021-12-03T18:31:32.618175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in list(reduced_dataframes) :\n    get_momentum_indicators(reduced_dataframes[i])\n    get_volatility_indicators(reduced_dataframes[i])\n    get_volume_indicators(reduced_dataframes[i])\n    get_cycle_indicators(reduced_dataframes[i])\n    get_statitics_functions(reduced_dataframes[i])\n    get_overlap_functions(reduced_dataframes[i])\n    get_price_transformations(reduced_dataframes[i])\n    get_pattern_recognition(reduced_dataframes[i])","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2021-12-03T18:31:40.905342Z","iopub.execute_input":"2021-12-03T18:31:40.905612Z","iopub.status.idle":"2021-12-03T18:31:42.538617Z","shell.execute_reply.started":"2021-12-03T18:31:40.905584Z","shell.execute_reply":"2021-12-03T18:31:42.537734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Instead of using all the KPIs let's use only what is useful,below we will stard by deleting the indecators that are either zeros or null across all the dataset","metadata":{}},{"cell_type":"code","source":"reduced_dataframes_full = reduced_dataframes\nreduced_dataframes_full[1].info()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:32:16.906132Z","iopub.execute_input":"2021-12-03T18:32:16.906424Z","iopub.status.idle":"2021-12-03T18:32:16.929169Z","shell.execute_reply.started":"2021-12-03T18:32:16.906396Z","shell.execute_reply":"2021-12-03T18:32:16.928408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Here we will remove the irrelevant features that does not have any predicting power\n# if a features is non-variant we will remove it\n# then we will use only pttrn_cols_keep\n\npttrn_cols_remove = {}\npttrn_cols_keep = {}\nfor i in list(reduced_dataframes) :\n    inter = []\n    inter = reduced_dataframes[i].loc[:,pattern_recognition_columns].fillna(0).astype(bool).sum().astype(bool)\n    pttrn_cols_remove[i] = inter.loc[inter.values == False ].index.tolist()\n    pttrn_cols_keep[i] = inter.loc[inter.values == True ].index.tolist()\n    reduced_dataframes[i].drop(columns=pttrn_cols_remove[i], inplace = True)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:32:39.935287Z","iopub.execute_input":"2021-12-03T18:32:39.936648Z","iopub.status.idle":"2021-12-03T18:32:40.169860Z","shell.execute_reply.started":"2021-12-03T18:32:39.936584Z","shell.execute_reply":"2021-12-03T18:32:40.169175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduced_dataframes_full[1]","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:33:14.579001Z","iopub.execute_input":"2021-12-03T18:33:14.579308Z","iopub.status.idle":"2021-12-03T18:33:14.611792Z","shell.execute_reply.started":"2021-12-03T18:33:14.579276Z","shell.execute_reply":"2021-12-03T18:33:14.610939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop the rows with null target and start the EDA\nfor i in list(reduced_dataframes) :\n    reduced_dataframes[i].drop(reduced_dataframes[i][original_columns].loc[pd.isnull(reduced_dataframes[i][original_columns]).Target == True ].index, inplace=True)\n    reduced_dataframes[i].drop(reduced_dataframes[i][cycle_indicators_columns].loc[pd.isnull(reduced_dataframes[i][cycle_indicators_columns]).htsn_cyc == True ].index, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T15:00:14.415484Z","iopub.execute_input":"2021-12-03T15:00:14.415737Z","iopub.status.idle":"2021-12-03T15:00:14.551795Z","shell.execute_reply.started":"2021-12-03T15:00:14.415709Z","shell.execute_reply":"2021-12-03T15:00:14.550850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check that we don't have nulls\n\nsum_nulls = []\nfor i in list(reduced_dataframes) :\n    sum_nulls.append(reduced_dataframes[i].isnull().sum().sum())\nsum_nulls","metadata":{"execution":{"iopub.status.busy":"2021-12-03T18:33:26.661838Z","iopub.execute_input":"2021-12-03T18:33:26.662180Z","iopub.status.idle":"2021-12-03T18:33:26.720355Z","shell.execute_reply.started":"2021-12-03T18:33:26.662144Z","shell.execute_reply":"2021-12-03T18:33:26.719535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"studied_columns = ['Target'] + momentum_indicators_columns + statitics_functions_columns + volatility_indicators_columns + price_transformations","metadata":{"execution":{"iopub.status.busy":"2021-12-03T15:27:06.789144Z","iopub.execute_input":"2021-12-03T15:27:06.789826Z","iopub.status.idle":"2021-12-03T15:27:06.793924Z","shell.execute_reply.started":"2021-12-03T15:27:06.789792Z","shell.execute_reply":"2021-12-03T15:27:06.792994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduced_dataframes[1].loc[:,studied_columns].describe()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T15:00:14.614453Z","iopub.execute_input":"2021-12-03T15:00:14.614852Z","iopub.status.idle":"2021-12-03T15:00:14.744418Z","shell.execute_reply.started":"2021-12-03T15:00:14.614768Z","shell.execute_reply":"2021-12-03T15:00:14.743484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-12-03T15:00:14.745859Z","iopub.execute_input":"2021-12-03T15:00:14.746217Z","iopub.status.idle":"2021-12-03T15:00:15.821215Z","shell.execute_reply.started":"2021-12-03T15:00:14.746182Z","shell.execute_reply":"2021-12-03T15:00:15.820397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"sns.kdeplot(\n    data=reduced_dataframes[1], x=\"ad_vol\",\n    cumulative=False, common_norm=False, common_grid=True,\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:29:14.25085Z","iopub.execute_input":"2021-12-03T12:29:14.251564Z","iopub.status.idle":"2021-12-03T12:29:14.500245Z","shell.execute_reply.started":"2021-12-03T12:29:14.251518Z","shell.execute_reply":"2021-12-03T12:29:14.499387Z"}}},{"cell_type":"markdown","source":"sns.pairplot(reduced_dataframes[1][studied_columns],   \n    x_vars= studied_columns,\n    y_vars=['Target'], size = 10)\n\nplt.savefig('output.png')","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:26:17.176432Z","iopub.execute_input":"2021-12-03T13:26:17.176714Z","iopub.status.idle":"2021-12-03T13:26:38.155331Z","shell.execute_reply.started":"2021-12-03T13:26:17.176684Z","shell.execute_reply":"2021-12-03T13:26:38.154461Z"}}},{"cell_type":"markdown","source":"\nsns.relplot(\n    data=reduced_dataframes[1], x=\"obv_vol\", y=\"Target\"\n    ,palette=[\"b\", \"r\"], sizes=(10, 100)\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T12:30:41.845273Z","iopub.execute_input":"2021-12-03T12:30:41.845618Z","iopub.status.idle":"2021-12-03T12:30:42.229303Z","shell.execute_reply.started":"2021-12-03T12:30:41.845586Z","shell.execute_reply":"2021-12-03T12:30:42.228294Z"}}},{"cell_type":"code","source":"import plotly.graph_objects as go\n\nfig = go.Figure(data=[go.Candlestick(x=reduced_dataframes[1].index, open=reduced_dataframes[1]['Open'], high=reduced_dataframes[1]['High'], low=reduced_dataframes[1]['Low'], close=reduced_dataframes[1]['Close'])])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T15:03:26.078045Z","iopub.execute_input":"2021-12-03T15:03:26.078392Z","iopub.status.idle":"2021-12-03T15:03:26.102835Z","shell.execute_reply.started":"2021-12-03T15:03:26.078355Z","shell.execute_reply":"2021-12-03T15:03:26.102127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot( data=reduced_dataframes[1], x=\"Target\", cumulative=False, common_norm=False, common_grid=True, )","metadata":{"execution":{"iopub.status.busy":"2021-12-03T15:20:41.681108Z","iopub.execute_input":"2021-12-03T15:20:41.682282Z","iopub.status.idle":"2021-12-03T15:20:41.990398Z","shell.execute_reply.started":"2021-12-03T15:20:41.682235Z","shell.execute_reply":"2021-12-03T15:20:41.989496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\n#print(scaler.fit(studied_columns))\n\nX = reduced_dataframes[1][studied_columns].copy()\ny = X.pop('Target')\nX_scaled = scaler.fit_transform(X)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:04:45.035244Z","iopub.execute_input":"2021-12-03T16:04:45.036223Z","iopub.status.idle":"2021-12-03T16:04:45.055639Z","shell.execute_reply.started":"2021-12-03T16:04:45.036176Z","shell.execute_reply":"2021-12-03T16:04:45.054725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2021-12-03T17:21:02.607834Z","iopub.execute_input":"2021-12-03T17:21:02.608299Z","iopub.status.idle":"2021-12-03T17:21:02.645750Z","shell.execute_reply.started":"2021-12-03T17:21:02.608257Z","shell.execute_reply":"2021-12-03T17:21:02.644974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.cluster import KMeans\nkmeans = KMeans(n_clusters=30, n_init=10, random_state=0, verbose=1, )\nX[\"Cluster\"] = kmeans.fit_predict(X_scaled)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:56:03.660671Z","iopub.execute_input":"2021-12-03T16:56:03.661024Z","iopub.status.idle":"2021-12-03T16:56:07.835156Z","shell.execute_reply.started":"2021-12-03T16:56:03.660990Z","shell.execute_reply":"2021-12-03T16:56:07.834443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xy = X.copy()\nXy[\"Cluster\"] = Xy.Cluster.astype(\"category\")\nXy[\"Target\"] = y\n#sns.relplot(\n#    x=\"value\", y=\"Target\", hue=\"Cluster\", col=\"variable\",\n#    height=4, aspect=1, facet_kws={'sharex': False}, col_wrap=3,\n#    data=Xy.melt(\n#        value_vars=studied_columns, id_vars=[\"Target\", \"Cluster\"],\n#    ),\n#);","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:58:19.055120Z","iopub.execute_input":"2021-12-03T16:58:19.056215Z","iopub.status.idle":"2021-12-03T16:58:19.063550Z","shell.execute_reply.started":"2021-12-03T16:58:19.056163Z","shell.execute_reply":"2021-12-03T16:58:19.062905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"good_clusters = []\nfor i in Xy.Cluster.value_counts().index :\n    lup = len(Xy.loc[(Xy.Cluster == i) & (Xy.Target > 0)]['Target'])\n    ldown = len(Xy.loc[(Xy.Cluster == i) & (Xy.Target <= 0)]['Target'])\n    m = (lup - ldown) / (lup + ldown)\n    good_clusters.append(m)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:59:04.187597Z","iopub.execute_input":"2021-12-03T16:59:04.187923Z","iopub.status.idle":"2021-12-03T16:59:04.245607Z","shell.execute_reply.started":"2021-12-03T16:59:04.187892Z","shell.execute_reply":"2021-12-03T16:59:04.244737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X['Cluster'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T17:17:56.564364Z","iopub.execute_input":"2021-12-03T17:17:56.565393Z","iopub.status.idle":"2021-12-03T17:17:56.573486Z","shell.execute_reply.started":"2021-12-03T17:17:56.565339Z","shell.execute_reply":"2021-12-03T17:17:56.572880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[good_clusters,Xy.Cluster.value_counts().index.tolist()]","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:59:27.848751Z","iopub.execute_input":"2021-12-03T16:59:27.849054Z","iopub.status.idle":"2021-12-03T16:59:27.859533Z","shell.execute_reply.started":"2021-12-03T16:59:27.849024Z","shell.execute_reply":"2021-12-03T16:59:27.858543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xy = X.copy()\nXy[\"Cluster\"] = Xy.Cluster.astype(\"category\")\nXy[\"Target\"] = y\nsns.relplot(\n    x=\"value\", y=\"Target\", hue=\"Cluster\", col=\"variable\",\n    height=4, aspect=1, facet_kws={'sharex': False}, col_wrap=3,\n    data=Xy.loc[Xy.Cluster == 14].melt(\n        value_vars=studied_columns, id_vars=[\"Target\", \"Cluster\"],\n    ),\n);","metadata":{"execution":{"iopub.status.busy":"2021-12-03T17:18:36.852121Z","iopub.execute_input":"2021-12-03T17:18:36.852980Z","iopub.status.idle":"2021-12-03T17:18:46.875131Z","shell.execute_reply.started":"2021-12-03T17:18:36.852933Z","shell.execute_reply":"2021-12-03T17:18:46.874250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xy.loc[(Xy.Cluster == 2) & (Xy.Target >= 0)].","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:26:38.041163Z","iopub.execute_input":"2021-12-03T16:26:38.042067Z","iopub.status.idle":"2021-12-03T16:26:38.082339Z","shell.execute_reply.started":"2021-12-03T16:26:38.042013Z","shell.execute_reply":"2021-12-03T16:26:38.081425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot( data=Xy.loc[(Xy.Cluster == 2) & (Xy.Target >= 0)], x=\"Target\", cumulative=False, common_norm=False, common_grid=True, )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.kdeplot( data=Xy.loc[(Xy.Cluster == 3)], x=\"Target\", cumulative=False, common_norm=False, common_grid=True, )","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:32:42.307493Z","iopub.execute_input":"2021-12-03T16:32:42.307819Z","iopub.status.idle":"2021-12-03T16:32:42.546647Z","shell.execute_reply.started":"2021-12-03T16:32:42.307786Z","shell.execute_reply":"2021-12-03T16:32:42.545701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"good_clusters = []\nfor i in Xy.Cluster.value_counts().index :\n    lup = len(Xy.loc[(Xy.Cluster == i) & (Xy.Target > 0)]['Target'])\n    ldown = len(Xy.loc[(Xy.Cluster == i) & (Xy.Target <= 0)]['Target'])\n    m = (lup - ldown) / (lup + ldown)\n    good_clusters.append(m)\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:56:40.396603Z","iopub.execute_input":"2021-12-03T16:56:40.396945Z","iopub.status.idle":"2021-12-03T16:56:40.422831Z","shell.execute_reply.started":"2021-12-03T16:56:40.396912Z","shell.execute_reply":"2021-12-03T16:56:40.422011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xy.Cluster.value_counts().index.tolist()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:48:16.898736Z","iopub.execute_input":"2021-12-03T16:48:16.899449Z","iopub.status.idle":"2021-12-03T16:48:16.907554Z","shell.execute_reply.started":"2021-12-03T16:48:16.899402Z","shell.execute_reply":"2021-12-03T16:48:16.906745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[good_clusters,Xy.Cluster.value_counts().index.tolist()]","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:56:50.084174Z","iopub.execute_input":"2021-12-03T16:56:50.085242Z","iopub.status.idle":"2021-12-03T16:56:50.096032Z","shell.execute_reply.started":"2021-12-03T16:56:50.085183Z","shell.execute_reply":"2021-12-03T16:56:50.095315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-12-03T16:49:11.399564Z","iopub.execute_input":"2021-12-03T16:49:11.400589Z","iopub.status.idle":"2021-12-03T16:49:11.406069Z","shell.execute_reply.started":"2021-12-03T16:49:11.400550Z","shell.execute_reply":"2021-12-03T16:49:11.405213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}