{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gresearch_crypto\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport time\nimport datetime\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:16:43.898230Z","iopub.execute_input":"2022-03-07T15:16:43.898522Z","iopub.status.idle":"2022-03-07T15:16:43.903643Z","shell.execute_reply.started":"2022-03-07T15:16:43.898494Z","shell.execute_reply":"2022-03-07T15:16:43.902775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Importing Data","metadata":{"execution":{"iopub.status.busy":"2022-02-01T23:04:58.025296Z","iopub.execute_input":"2022-02-01T23:04:58.025893Z","iopub.status.idle":"2022-02-01T23:04:58.029573Z","shell.execute_reply.started":"2022-02-01T23:04:58.02585Z","shell.execute_reply":"2022-02-01T23:04:58.02872Z"}}},{"cell_type":"code","source":"df_train= pd.read_csv('../input/g-research-crypto-forecasting/train.csv', low_memory=False, \n                       dtype={'Asset_ID': 'int8', 'Count': 'int32', 'row_id': 'int32', 'Count': 'int32', \n                              'Open': 'float64', 'High': 'float64', 'Low': 'float64', 'Close': 'float64', \n                              'Volume': 'float64', 'VWAP': 'float64'\n                             }\n                      )","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:16:43.905000Z","iopub.execute_input":"2022-03-07T15:16:43.905794Z","iopub.status.idle":"2022-03-07T15:17:24.651183Z","shell.execute_reply.started":"2022-03-07T15:16:43.905738Z","shell.execute_reply":"2022-03-07T15:17:24.650291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['datetime'] = pd.to_datetime(df_train['timestamp'], unit='s')","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:24.652282Z","iopub.execute_input":"2022-03-07T15:17:24.652487Z","iopub.status.idle":"2022-03-07T15:17:26.280555Z","shell.execute_reply.started":"2022-03-07T15:17:24.652461Z","shell.execute_reply":"2022-03-07T15:17:26.279705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"timestamp\"][1]","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:26.282350Z","iopub.execute_input":"2022-03-07T15:17:26.282581Z","iopub.status.idle":"2022-03-07T15:17:26.290625Z","shell.execute_reply.started":"2022-03-07T15:17:26.282554Z","shell.execute_reply":"2022-03-07T15:17:26.290059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"timestamp\"][-1:].values[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:26.291729Z","iopub.execute_input":"2022-03-07T15:17:26.292677Z","iopub.status.idle":"2022-03-07T15:17:26.303601Z","shell.execute_reply.started":"2022-03-07T15:17:26.292628Z","shell.execute_reply":"2022-03-07T15:17:26.303079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dropping -inf, inf, and -values for VWAP.","metadata":{}},{"cell_type":"code","source":"df_train[\"VWAP\"].max()","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:26.304829Z","iopub.execute_input":"2022-03-07T15:17:26.305272Z","iopub.status.idle":"2022-03-07T15:17:26.500511Z","shell.execute_reply.started":"2022-03-07T15:17:26.305229Z","shell.execute_reply":"2022-03-07T15:17:26.499946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"VWAP\"].min()","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:26.501660Z","iopub.execute_input":"2022-03-07T15:17:26.502039Z","iopub.status.idle":"2022-03-07T15:17:26.697184Z","shell.execute_reply.started":"2022-03-07T15:17:26.501992Z","shell.execute_reply":"2022-03-07T15:17:26.696266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train[\"VWAP\"]<0]","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:26.698586Z","iopub.execute_input":"2022-03-07T15:17:26.698972Z","iopub.status.idle":"2022-03-07T15:17:26.816720Z","shell.execute_reply.started":"2022-03-07T15:17:26.698925Z","shell.execute_reply":"2022-03-07T15:17:26.816076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train[\"VWAP\"]==np.inf]","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:26.817888Z","iopub.execute_input":"2022-03-07T15:17:26.818315Z","iopub.status.idle":"2022-03-07T15:17:26.862139Z","shell.execute_reply.started":"2022-03-07T15:17:26.818274Z","shell.execute_reply":"2022-03-07T15:17:26.861171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if df_train[\"VWAP\"].max()==np.inf:\n    df_train=df_train.drop(df_train[df_train[\"VWAP\"]==np.inf].index,axis=0)\nif df_train[\"VWAP\"].min()<=0:\n    df_train=df_train.drop(df_train[df_train[\"VWAP\"]<0].index,axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:26.866468Z","iopub.execute_input":"2022-03-07T15:17:26.866721Z","iopub.status.idle":"2022-03-07T15:17:33.977611Z","shell.execute_reply.started":"2022-03-07T15:17:26.866691Z","shell.execute_reply":"2022-03-07T15:17:33.976875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr_1d=np.arange(df_train[\"timestamp\"][1],df_train[\"timestamp\"][-1:].values[0],60*60*24)","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:33.978668Z","iopub.execute_input":"2022-03-07T15:17:33.978882Z","iopub.status.idle":"2022-03-07T15:17:34.041739Z","shell.execute_reply.started":"2022-03-07T15:17:33.978857Z","shell.execute_reply":"2022-03-07T15:17:34.040953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr_1d.shape","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:34.042732Z","iopub.execute_input":"2022-03-07T15:17:34.042944Z","iopub.status.idle":"2022-03-07T15:17:34.048258Z","shell.execute_reply.started":"2022-03-07T15:17:34.042907Z","shell.execute_reply":"2022-03-07T15:17:34.047512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_df=pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:34.049309Z","iopub.execute_input":"2022-03-07T15:17:34.049509Z","iopub.status.idle":"2022-03-07T15:17:34.059674Z","shell.execute_reply.started":"2022-03-07T15:17:34.049483Z","shell.execute_reply":"2022-03-07T15:17:34.058940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for A in arr_1d:\n    daily_df=daily_df.append(df_train[df_train[\"timestamp\"]==A])\n    ","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:17:34.060748Z","iopub.execute_input":"2022-03-07T15:17:34.061102Z","iopub.status.idle":"2022-03-07T15:18:00.858158Z","shell.execute_reply.started":"2022-03-07T15:17:34.061074Z","shell.execute_reply":"2022-03-07T15:18:00.857249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"daily_df","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:00.860088Z","iopub.execute_input":"2022-03-07T15:18:00.860669Z","iopub.status.idle":"2022-03-07T15:18:00.882461Z","shell.execute_reply.started":"2022-03-07T15:18:00.860621Z","shell.execute_reply":"2022-03-07T15:18:00.881918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=daily_df","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:00.883394Z","iopub.execute_input":"2022-03-07T15:18:00.883924Z","iopub.status.idle":"2022-03-07T15:18:00.893072Z","shell.execute_reply.started":"2022-03-07T15:18:00.883892Z","shell.execute_reply":"2022-03-07T15:18:00.892501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:00.894253Z","iopub.execute_input":"2022-03-07T15:18:00.894616Z","iopub.status.idle":"2022-03-07T15:18:00.922870Z","shell.execute_reply.started":"2022-03-07T15:18:00.894590Z","shell.execute_reply":"2022-03-07T15:18:00.922243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"datetime2\"]=df_train[\"datetime\"]","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:00.924067Z","iopub.execute_input":"2022-03-07T15:18:00.924435Z","iopub.status.idle":"2022-03-07T15:18:00.929111Z","shell.execute_reply.started":"2022-03-07T15:18:00.924402Z","shell.execute_reply":"2022-03-07T15:18:00.928267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=df_train.set_index(\"datetime2\").to_period(\"D\") ","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:00.930437Z","iopub.execute_input":"2022-03-07T15:18:00.930667Z","iopub.status.idle":"2022-03-07T15:18:00.947410Z","shell.execute_reply.started":"2022-03-07T15:18:00.930613Z","shell.execute_reply":"2022-03-07T15:18:00.946694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details=pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\").sort_values(\"Asset_ID\")","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:00.950096Z","iopub.execute_input":"2022-03-07T15:18:00.951102Z","iopub.status.idle":"2022-03-07T15:18:00.964042Z","shell.execute_reply.started":"2022-03-07T15:18:00.951052Z","shell.execute_reply":"2022-03-07T15:18:00.963506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:00.964995Z","iopub.execute_input":"2022-03-07T15:18:00.965256Z","iopub.status.idle":"2022-03-07T15:18:00.975560Z","shell.execute_reply.started":"2022-03-07T15:18:00.965227Z","shell.execute_reply":"2022-03-07T15:18:00.975007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#splitting to assets\nbinance=df_train[df_train[\"Asset_ID\"]==0].sort_values(\"timestamp\")\nbitcoin=df_train[df_train[\"Asset_ID\"]==1].sort_values(\"timestamp\")\nbtccash=df_train[df_train[\"Asset_ID\"]==2].sort_values(\"timestamp\")\ncardona=df_train[df_train[\"Asset_ID\"]==3].sort_values(\"timestamp\")\ndoge =  df_train[df_train[\"Asset_ID\"]==4].sort_values(\"timestamp\")\neos  =  df_train[df_train[\"Asset_ID\"]==5].sort_values(\"timestamp\")\neth  =  df_train[df_train[\"Asset_ID\"]==6].sort_values(\"timestamp\")\nethcls =df_train[df_train[\"Asset_ID\"]==7].sort_values(\"timestamp\")\niota   =df_train[df_train[\"Asset_ID\"]==8].sort_values(\"timestamp\")\nltc   = df_train[df_train[\"Asset_ID\"]==9].sort_values(\"timestamp\")\nmaker = df_train[df_train[\"Asset_ID\"]==10].sort_values(\"timestamp\")\nmonero =df_train[df_train[\"Asset_ID\"]==11].sort_values(\"timestamp\")\nstellar=df_train[df_train[\"Asset_ID\"]==12].sort_values(\"timestamp\")\ntron  = df_train[df_train[\"Asset_ID\"]==13].sort_values(\"timestamp\")","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:00.976592Z","iopub.execute_input":"2022-03-07T15:18:00.977080Z","iopub.status.idle":"2022-03-07T15:18:01.010148Z","shell.execute_reply.started":"2022-03-07T15:18:00.977047Z","shell.execute_reply":"2022-03-07T15:18:01.009462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bitcoin","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.011359Z","iopub.execute_input":"2022-03-07T15:18:01.011647Z","iopub.status.idle":"2022-03-07T15:18:01.034220Z","shell.execute_reply.started":"2022-03-07T15:18:01.011609Z","shell.execute_reply":"2022-03-07T15:18:01.033621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coinlist=[\"binance\",\"bitcoin\",\"btccash\",\"cardona\",\"doge\",\n          \"eos\",\"eth\",\"ethcls\",\"iota\",\"ltc\",\"maker\",\"monero\",\"stellar\",\"tron\"]","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.035301Z","iopub.execute_input":"2022-03-07T15:18:01.035911Z","iopub.status.idle":"2022-03-07T15:18:01.039535Z","shell.execute_reply.started":"2022-03-07T15:18:01.035880Z","shell.execute_reply":"2022-03-07T15:18:01.038981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coindict={\"binance\":0,\"bitcoin\":1,\"btccash\":2,\"cardona\":3,\"doge\":4,\n                    \"eos\":5,\"eth\":6,\"ethcls\":7,\"iota\":8,\"ltc\":9,\"maker\":10,\"monero\":11,\"stellar\":12,\"tron\":13}","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.040860Z","iopub.execute_input":"2022-03-07T15:18:01.041277Z","iopub.status.idle":"2022-03-07T15:18:01.052204Z","shell.execute_reply.started":"2022-03-07T15:18:01.041242Z","shell.execute_reply":"2022-03-07T15:18:01.051589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coindict[\"binance\"]","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.053398Z","iopub.execute_input":"2022-03-07T15:18:01.054369Z","iopub.status.idle":"2022-03-07T15:18:01.066267Z","shell.execute_reply.started":"2022-03-07T15:18:01.054337Z","shell.execute_reply":"2022-03-07T15:18:01.065540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Indicator features","metadata":{}},{"cell_type":"code","source":"for name in coinlist:\n    str=name\n    \n    \n    locals()[str]=locals()[str].copy()\n    \n    locals()[str][\"returns\"] = np.log(locals()[str].Close / locals()[str].Close.shift(1)) # daily return\n    \n    # Simple Moving Averages\n    locals()[str][\"MA12\"] = locals()[str].Close.rolling(window = 12).mean() \n    locals()[str][\"MA26\"] = locals()[str].Close.rolling(window = 26).mean()\n    locals()[str][\"MA60\"] = locals()[str].Close.rolling(window = 60).mean()\n    locals()[str][\"MA99\"] = locals()[str].Close.rolling(window = 99).mean()\n    locals()[str][\"MA200\"] = locals()[str].Close.rolling(window = 200).mean()\n    \n    #Moving average convergence divergence (MACD)\n    locals()[str][\"smaMACD\"] = locals()[str].MA12-locals()[str].MA26 #fast SMA - slow SMA\n    locals()[str][\"smaMACDs\"] = locals()[str].smaMACD.rolling(window = 9).mean() # for 9 day period MACD signal\n    locals()[str][\"smaMACDh\"] = locals()[str].smaMACD - locals()[str].smaMACDs # MACD histogram\n    \n    #Bull Bear Power\n    locals()[str][\"BearBullPower2\"] = locals()[str][\"High\"]-locals()[str][\"MA12\"] + locals()[str][\"Low\"]-locals()[str][\"MA12\"]\n    \n    #Bollinger Bands\n    locals()[str][\"BollingerBasis\"] = locals()[str].Close.rolling(window = 20).mean()\n    locals()[str][\"BollingerUpper\"] = locals()[str][\"BollingerBasis\"]+2*(locals()[str][\"Close\"].rolling(20).std())\n    locals()[str][\"BollingerLower\"] = locals()[str][\"BollingerBasis\"]-2*(locals()[str][\"Close\"].rolling(20).std())\n    \n    #Dochian Channel\n    locals()[str][\"DCup\"] = locals()[str].High.rolling(window = 20).max() #Highest High in Last 20 days Periods\n    locals()[str][\"DClow\"] = locals()[str].Low.rolling(window = 20).min() #Lowest Low in Last 20 days Periods\n    locals()[str][\"DCmid\"] = (locals()[str][\"DCup\"]+locals()[str][\"DClow\"])/2\n    \n    print(name+\"'s shape: \",locals()[str].shape[0])\n    print(name+\" is Done.\")\n    \nprint(\"All done.\")","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.067583Z","iopub.execute_input":"2022-03-07T15:18:01.067831Z","iopub.status.idle":"2022-03-07T15:18:01.287689Z","shell.execute_reply.started":"2022-03-07T15:18:01.067802Z","shell.execute_reply":"2022-03-07T15:18:01.287142Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name in coinlist:\n    str=name\n    locals()[str]=locals()[str].dropna()\n    print(name+\"'s shape: \",locals()[str].shape[0])\n    print(name+\" is Done.\")\n    \nprint(\"All done.\")","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-07T15:18:01.291504Z","iopub.execute_input":"2022-03-07T15:18:01.292011Z","iopub.status.idle":"2022-03-07T15:18:01.370076Z","shell.execute_reply.started":"2022-03-07T15:18:01.291979Z","shell.execute_reply":"2022-03-07T15:18:01.369268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bitcoin.columns","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.371570Z","iopub.execute_input":"2022-03-07T15:18:01.372282Z","iopub.status.idle":"2022-03-07T15:18:01.379226Z","shell.execute_reply.started":"2022-03-07T15:18:01.372236Z","shell.execute_reply":"2022-03-07T15:18:01.378261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_columns=['Count','Open','High', 'Low', 'Close','Volume','VWAP',\"Target\" ,'returns', 'MA26', 'MA12', 'smaMACD',\n            'smaMACDs', 'smaMACDh', 'MA99', 'MA200', 'MA60',\"BollingerBasis\",\"BollingerUpper\",\n              \"BollingerLower\",'DCup', 'DClow','DCmid']","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.380434Z","iopub.execute_input":"2022-03-07T15:18:01.380713Z","iopub.status.idle":"2022-03-07T15:18:01.388183Z","shell.execute_reply.started":"2022-03-07T15:18:01.380674Z","shell.execute_reply":"2022-03-07T15:18:01.387652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bitcoin","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.389283Z","iopub.execute_input":"2022-03-07T15:18:01.389563Z","iopub.status.idle":"2022-03-07T15:18:01.430851Z","shell.execute_reply.started":"2022-03-07T15:18:01.389525Z","shell.execute_reply":"2022-03-07T15:18:01.430029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bitcoin.index.values","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.433116Z","iopub.execute_input":"2022-03-07T15:18:01.433435Z","iopub.status.idle":"2022-03-07T15:18:01.445770Z","shell.execute_reply.started":"2022-03-07T15:18:01.433394Z","shell.execute_reply":"2022-03-07T15:18:01.445242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Analysis","metadata":{}},{"cell_type":"code","source":"from warnings import simplefilter\nsimplefilter(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.446620Z","iopub.execute_input":"2022-03-07T15:18:01.446841Z","iopub.status.idle":"2022-03-07T15:18:01.455333Z","shell.execute_reply.started":"2022-03-07T15:18:01.446806Z","shell.execute_reply":"2022-03-07T15:18:01.454770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name in coinlist:\n    str=name\n    f,ax=plt.subplots(nrows=3,ncols=1,figsize=(16,8),gridspec_kw={'height_ratios': [2, 1,1]})\n    ax[0].set_title(name.title())\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"Close\"].values,label=\"close\",ax=ax[0])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"BollingerUpper\"].values,label=\"BBup\",ax=ax[0])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"BollingerBasis\"].values,label=\"BBmid\",ax=ax[0])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"BollingerLower\"].values,label=\"BBlow\",ax=ax[0])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"MA26\"].values,label=\"MA26\",ax=ax[0])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"BearBullPower2\"].values,label=\"BullBearPower\",ax=ax[1])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"returns\"].values,label=\"daily returns\",ax=ax[2])\n    plt.show()\n    print(\" \"*57+name.title()+\"'s\"+' plot'.title())\n    print(\" \")\n    print(\" \")\n","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:01.457031Z","iopub.execute_input":"2022-03-07T15:18:01.457339Z","iopub.status.idle":"2022-03-07T15:18:17.254851Z","shell.execute_reply.started":"2022-03-07T15:18:01.457299Z","shell.execute_reply":"2022-03-07T15:18:17.251702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name in coinlist:\n    str=name\n    f,ax=plt.subplots(nrows=2,ncols=1,figsize=(16,8),gridspec_kw={'height_ratios': [2, 1]})\n    ax[0].set_title(name.title())\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"Close\"].values,label=\"close\",ax=ax[0])\n    #sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"MA200\"].values,label=\"MA200\",ax=ax[0])\n    #sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"MA99\"].values,label=\"MA99\",ax=ax[0])\n    #sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"MA60\"].values,label=\"MA60\",ax=ax[0])\n    #sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"MA26\"].values,label=\"MA26\",ax=ax[0])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"smaMACD\"].values,label=\"smaMACD\",ax=ax[1])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"smaMACDs\"].values,label=\"smaMACD signal\",ax=ax[1])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"smaMACDh\"].values,label=\"smaMACD histogram\",ax=ax[1])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"DCup\"].values,label=\"BBup\",ax=ax[0])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"DCmid\"].values,label=\"BBmid\",ax=ax[0])\n    sns.lineplot(locals()[str][\"datetime\"],locals()[str][\"DClow\"].values,label=\"BBlow\",ax=ax[0])\n    plt.show()\n    print(\" \"*57+name.title()+\"'s\"+' plot'.title())\n    print(\" \")\n    print(\" \")\n    ","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:17.256366Z","iopub.execute_input":"2022-03-07T15:18:17.256573Z","iopub.status.idle":"2022-03-07T15:18:30.719086Z","shell.execute_reply.started":"2022-03-07T15:18:17.256547Z","shell.execute_reply":"2022-03-07T15:18:30.718202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bitcoin","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:18:30.720393Z","iopub.execute_input":"2022-03-07T15:18:30.720661Z","iopub.status.idle":"2022-03-07T15:18:30.774144Z","shell.execute_reply.started":"2022-03-07T15:18:30.720626Z","shell.execute_reply":"2022-03-07T15:18:30.773208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model, fit, grahps","metadata":{}},{"cell_type":"code","source":"def fourier_features(index, freq, order):\n    time = np.arange(len(index), dtype=np.float32)\n    k = 2 * np.pi * (1 / freq) * time\n    features = {}\n    for i in range(1, order + 1):\n        features.update({\n            f\"sin_{freq}_{i}\": np.sin(i * k),\n            f\"cos_{freq}_{i}\": np.cos(i * k),\n        })\n    return pd.DataFrame(features, index=index)\n\n\n# Compute Fourier features to the 4th order (8 new features) for a\n# series y with daily observations and annual seasonality:\n#\n# fourier_features(y, freq=365.25, order=4)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-07T15:18:30.775506Z","iopub.execute_input":"2022-03-07T15:18:30.775710Z","iopub.status.idle":"2022-03-07T15:18:30.782668Z","shell.execute_reply.started":"2022-03-07T15:18:30.775685Z","shell.execute_reply":"2022-03-07T15:18:30.781661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from warnings import simplefilter\n\nsimplefilter(\"ignore\")\n\n# Set Matplotlib defaults\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True, figsize=(11, 5))\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=16,\n    titlepad=10,\n)\nplot_params = dict(\n    color=\"0.75\",\n    style=\".-\",\n    markeredgecolor=\"0.25\",\n    markerfacecolor=\"0.25\",\n    legend=False,\n)\n%config InlineBackend.figure_format = 'retina'\n\n\n# annotations: https://stackoverflow.com/a/49238256/5769929\ndef seasonal_plot(X, y, period, freq, ax=None):\n    if ax is None:\n        _, ax = plt.subplots()\n    palette = sns.color_palette(\"husl\", n_colors=X[period].nunique(),)\n    ax = sns.lineplot(\n        x=freq,\n        y=y,\n        hue=period,\n        data=X,\n        ci=False,\n        ax=ax,\n        palette=palette,\n        legend=False,\n    )\n    ax.set_title(f\"Seasonal Plot ({period}/{freq})\")\n    for line, name in zip(ax.lines, X[period].unique()):\n        y_ = line.get_ydata()[-1]\n        ax.annotate(\n            name,\n            xy=(1, y_),\n            xytext=(6, 0),\n            color=line.get_color(),\n            xycoords=ax.get_yaxis_transform(),\n            textcoords=\"offset points\",\n            size=14,\n            va=\"center\",\n        )\n    return ax\n\n\ndef plot_periodogram(ts, detrend='linear', ax=None):\n    from scipy.signal import periodogram\n    fs = pd.Timedelta(\"1Y\") / pd.Timedelta(\"1D\")\n    freqencies, spectrum = periodogram(\n        ts,\n        fs=fs,\n        detrend=detrend,\n        window=\"boxcar\",\n        scaling='spectrum',\n    )\n    if ax is None:\n        _, ax = plt.subplots()\n    ax.step(freqencies, spectrum, color=\"purple\")\n    ax.set_xscale(\"log\")\n    ax.set_xticks([1, 2, 4, 6, 12, 26, 52, 104])\n    ax.set_xticklabels(\n        [\n            \"Annual (1)\",\n            \"Semiannual (2)\",\n            \"Quarterly (4)\",\n            \"Bimonthly (6)\",\n            \"Monthly (12)\",\n            \"Biweekly (26)\",\n            \"Weekly (52)\",\n            \"Semiweekly (104)\",\n        ],\n        rotation=30,\n    )\n    ax.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0, 0))\n    ax.set_ylabel(\"Variance\")\n    ax.set_title(\"Periodogram\")\n    return ax","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2022-03-07T15:18:30.786012Z","iopub.execute_input":"2022-03-07T15:18:30.788243Z","iopub.status.idle":"2022-03-07T15:18:30.836365Z","shell.execute_reply.started":"2022-03-07T15:18:30.788050Z","shell.execute_reply":"2022-03-07T15:18:30.835535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.tree import ExtraTreeRegressor\nfrom statsmodels.tsa.deterministic import CalendarFourier, DeterministicProcess","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:26:04.843773Z","iopub.execute_input":"2022-03-07T15:26:04.844048Z","iopub.status.idle":"2022-03-07T15:26:04.848710Z","shell.execute_reply.started":"2022-03-07T15:26:04.844020Z","shell.execute_reply":"2022-03-07T15:26:04.847929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"regression_model=ExtraTreeRegressor()","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:30:03.354483Z","iopub.execute_input":"2022-03-07T15:30:03.355200Z","iopub.status.idle":"2022-03-07T15:30:03.358922Z","shell.execute_reply.started":"2022-03-07T15:30:03.355142Z","shell.execute_reply":"2022-03-07T15:30:03.358368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fourier = CalendarFourier(freq=\"A\", order=100)  #100 sin/cos pairs for \"A\"nnual seasonality","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:26:21.379762Z","iopub.execute_input":"2022-03-07T15:26:21.380267Z","iopub.status.idle":"2022-03-07T15:26:21.384934Z","shell.execute_reply.started":"2022-03-07T15:26:21.380218Z","shell.execute_reply":"2022-03-07T15:26:21.384218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_absolute_error, mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:26:21.882328Z","iopub.execute_input":"2022-03-07T15:26:21.883116Z","iopub.status.idle":"2022-03-07T15:26:21.887008Z","shell.execute_reply.started":"2022-03-07T15:26:21.883077Z","shell.execute_reply":"2022-03-07T15:26:21.886248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name in coinlist:\n    str=name\n    X=name+\"_X\"\n    yy=name+\"_y\"\n    y_pred=name+\"_y_pred\"\n    X_fore=name+\"_X_fore\"\n    y_fore=name+\"_y_fore\"\n    model =name+\"_XGB_model\"\n    df_pred=name+\"_df_pred\"\n    df_fore=name+\"_df_fore\"\n    print(\"1.\"+name+\" Starting\")\n    \n    dp = DeterministicProcess(\n    index=locals()[str].index,\n    constant=False,               # dummy feature for bias (y-intercept)\n    order=1,                     # trend (order 1 means linear)\n    seasonal=True,               # weekly seasonality (indicators)\n    additional_terms=[fourier], # annual seasonality (fourier)\n    drop=False                  # drop terms to avoid collinearity\n                      \n    )\n    \n    locals()[X] = locals()[str][test_columns]\n\n    # days within a week\n    locals()[X][\"day\"] = locals()[X].index.dayofweek  # the x-axis (freq)\n    locals()[X][\"week\"] = locals()[X].index.week  # the seasonal period (period)\n    \n    # days within a year\n    locals()[X][\"dayofyear\"] = locals()[X].index.dayofyear\n    locals()[X][\"year\"] = locals()[X].index.year\n    print(\"2. first plot\")\n    \n    fig, (ax0, ax1) = plt.subplots(2, 1, figsize=(24, 8))\n    seasonal_plot(locals()[X], y=\"Target\", period=\"week\", freq=\"day\", ax=ax0)\n    seasonal_plot(locals()[X], y=\"Target\", period=\"year\", freq=\"dayofyear\", ax=ax1);\n    plt.show()\n    print(\"3. creting dp\")\n    locals()[X] = dp.in_sample()\n    print(\"4. dp finished\")\n    print(name,\"'s shape: \",locals()[X].shape)\n\n    locals()[yy] = locals()[str][\"Target\"]\n    print(\"5. \",name,\"'s shape: \",locals()[yy].shape)\n\n    locals()[model] = regression_model\n    print(\"6. \",name+\"'s fitting...\")\n    _ = locals()[model].fit(locals()[X], locals()[yy])\n    print(\"7. predict\")\n    locals()[y_pred] = pd.Series(locals()[model].predict(locals()[X]), index=locals()[yy].index)\n    locals()[X_fore] = dp.out_of_sample(steps=240)\n    locals()[y_fore] = pd.Series(locals()[model].predict(locals()[X_fore]), index=locals()[X_fore].index)\n    \n    plt.figure(figsize=(8,3))\n    sns.regplot(x=locals()[y_pred],y=locals()[str][\"Target\"],line_kws={\"color\":\"black\"})\n    plt.show()\n    plt.figure(figsize=(8,3))\n    sns.histplot((locals()[y_pred]-locals()[str][\"Target\"]),kde=True,bins=180)\n    plt.show()\n    \n    print(name+'_MAE:', mean_absolute_error(locals()[y_pred],locals()[str][\"Target\"]))\n    print(name+'_MSE:', mean_squared_error(locals()[y_pred],locals()[str][\"Target\"]))\n    print(\"8. second plot\")\n    plt.figure(figsize=(24,6))\n    ax = locals()[yy].plot(color='0.25', style='.', title=name+\" Target - Seasonal Forecast\")\n    ax = locals()[y_pred].plot(ax=ax, label=name+\" Seasonal\")\n    ax = locals()[y_fore].plot(ax=ax, label=name+\" Seasonal Forecast\", color='C3')\n    _ = ax.legend()\n    plt.show()\n    \n    locals()[df_pred]=pd.DataFrame(locals()[y_pred],columns=[\"Target\"])\n    locals()[df_pred][\"Asset_ID\"] = coindict[name]\n    locals()[df_fore]=pd.DataFrame(locals()[y_fore],columns=[\"Target\"])\n    locals()[df_fore][\"Asset_ID\"] = coindict[name]\n    print(\"9.\"+name+\" finished\")\nprint(\"All Done.\")","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:26:22.331493Z","iopub.execute_input":"2022-03-07T15:26:22.332459Z","iopub.status.idle":"2022-03-07T15:29:30.972093Z","shell.execute_reply.started":"2022-03-07T15:26:22.332399Z","shell.execute_reply":"2022-03-07T15:29:30.971334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict and Forecast Dataframes","metadata":{}},{"cell_type":"code","source":"df_predict=pd.concat([cardona_df_pred,btccash_df_pred,binance_df_pred,bitcoin_df_pred,doge_df_pred,eos_df_pred, ethcls_df_pred,\n                       eth_df_pred,iota_df_pred,ltc_df_pred,maker_df_pred,tron_df_pred,stellar_df_pred,monero_df_pred])\ndf_predict[\"Target\"]=df_predict[\"Target\"].astype(np.float64)\ndf_predict","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:29:41.792550Z","iopub.execute_input":"2022-03-07T15:29:41.792852Z","iopub.status.idle":"2022-03-07T15:29:41.811587Z","shell.execute_reply.started":"2022-03-07T15:29:41.792823Z","shell.execute_reply":"2022-03-07T15:29:41.810739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_predict.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:29:43.107998Z","iopub.execute_input":"2022-03-07T15:29:43.108325Z","iopub.status.idle":"2022-03-07T15:29:43.121935Z","shell.execute_reply.started":"2022-03-07T15:29:43.108281Z","shell.execute_reply":"2022-03-07T15:29:43.121005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_forecast=pd.concat([cardona_df_fore,btccash_df_fore,binance_df_fore,bitcoin_df_fore,doge_df_fore,eos_df_fore, ethcls_df_fore,\n                       eth_df_fore,iota_df_fore,ltc_df_fore,maker_df_fore,tron_df_fore,stellar_df_fore,monero_df_fore])\n","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:29:43.825883Z","iopub.execute_input":"2022-03-07T15:29:43.826209Z","iopub.status.idle":"2022-03-07T15:29:43.833794Z","shell.execute_reply.started":"2022-03-07T15:29:43.826154Z","shell.execute_reply":"2022-03-07T15:29:43.833067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_forecast","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:29:44.664031Z","iopub.execute_input":"2022-03-07T15:29:44.664331Z","iopub.status.idle":"2022-03-07T15:29:44.676768Z","shell.execute_reply.started":"2022-03-07T15:29:44.664291Z","shell.execute_reply":"2022-03-07T15:29:44.676092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_forecast.info()","metadata":{"execution":{"iopub.status.busy":"2022-03-07T15:29:48.828084Z","iopub.execute_input":"2022-03-07T15:29:48.828565Z","iopub.status.idle":"2022-03-07T15:29:48.840875Z","shell.execute_reply.started":"2022-03-07T15:29:48.828521Z","shell.execute_reply":"2022-03-07T15:29:48.839951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## References\n\n##### [Seasonality lesson from Kaggle's Time Series Course](http://www.kaggle.com/ryanholbrook/seasonality)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-01T23:12:57.895741Z","iopub.execute_input":"2022-02-01T23:12:57.896151Z","iopub.status.idle":"2022-02-01T23:12:57.901231Z","shell.execute_reply.started":"2022-02-01T23:12:57.89611Z","shell.execute_reply":"2022-02-01T23:12:57.900177Z"}}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}