{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Visualise and compare prices of any 2 assets on a day-by-day fashion","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport dask.dataframe as dd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set_style('whitegrid')\nsns.set(font_scale = 1.4)\n\nfrom datetime import datetime\n\nfrom scipy.fft import fft, fftfreq\nfrom sklearn import preprocessing\nfrom scipy import signal\n\nimport gc ","metadata":{"execution":{"iopub.status.busy":"2022-01-13T19:25:11.337111Z","iopub.execute_input":"2022-01-13T19:25:11.337574Z","iopub.status.idle":"2022-01-13T19:25:13.016204Z","shell.execute_reply.started":"2022-01-13T19:25:11.337530Z","shell.execute_reply":"2022-01-13T19:25:13.015461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since we care about short term price movements, lets study the properties of the time series in short, logical, time frames such as day-by-day.","metadata":{}},{"cell_type":"code","source":"DATA = '../input/g-research-crypto-forecasting/train.csv'\nDATA_ASSETS = '../input/g-research-crypto-forecasting/train.csv'","metadata":{"execution":{"iopub.status.busy":"2022-01-13T19:27:24.429741Z","iopub.execute_input":"2022-01-13T19:27:24.430050Z","iopub.status.idle":"2022-01-13T19:27:24.434401Z","shell.execute_reply.started":"2022-01-13T19:27:24.430014Z","shell.execute_reply":"2022-01-13T19:27:24.433561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # # # # # # # # Engineering Functions # # # # # # # # # # # \ndef downcast_floats (df):\n    before = df.memory_usage().sum()/1073741824\n    for c in df.select_dtypes(include=['float']).columns.tolist():\n        if (df[str(c)].max() < np.finfo('float16').max) & (df[str(c)].min() > np.finfo('float16').min):\n             df[str(c)] = df[str(c)].astype('float16')\n        elif (df[str(c)].max() < np.finfo('float32').max) & (df[str(c)].min() > np.finfo('float32').min):\n            df[str(c)] = df[str(c)].astype('float32')\n    \n    after = df.memory_usage().sum()/1073741824\n    print ('Memory usage reduced by '+str(np.round((after/before)*100,1))+'% to '+str(after)+' Gb')\n    return df\n\n\n# # # # # # # # # Data Selection Functions # # # # # # # # # # # \ndef select_assets(a1, a2):\n    asset_ID_1 = asset_details.loc[a1].Asset_ID\n    asset1 = df[df['Asset_ID']==asset_ID_1[0]].set_index(\"timestamp\").copy()\n    print(a1+ ' contains '+ str(asset1.shape[0])+ ' entries and ' + str(asset1.isna().sum().sum())+ ' missing values')\n    asset1 = asset1.drop('Asset_ID',axis =1)\n\n    \n    asset_ID_2 = asset_details.loc[a2].Asset_ID\n    asset2 = df[df['Asset_ID']==asset_ID_2[0]].set_index(\"timestamp\").copy()\n    print(a2+ ' contains '+ str(asset2.shape[0])+ ' entries and ' + str(asset2.isna().sum().sum())+ ' missing values')\n    asset2 = asset2.drop('Asset_ID',axis =1)\n    return asset1, asset2\n\n\n# match row for row two dataframes\ndef clean_match(df1, df2):\n    df1 = df1.dropna()\n    df2 = df2.dropna()\n    #find intersection of indexes \n    idx = df1.index.intersection(df2.index)\n    \n    df1 = df1.loc[idx]\n    df2 = df2.loc[idx]\n    \n    print('assets matched: intersection of '+ str(len(idx)) + ' common rows')\n    \n    return df1, df2\n\n\n# create df with the average volume on  each day\n# acceptes a df with rows trading data \ndef daily_vol (df, verbose = False):\n    vol = []\n    d = []\n    for day in df.one_day.unique().tolist():\n        d.append(day)\n        vol.append(df[(df['one_day'] == day)].Volume.mean())\n    df_dvol = pd.DataFrame(vol, columns=['day_volume'], index = d)\n    \n    if verbose == True:\n        df_dvol.hist(bins=100)\n\n    return df_dvol\n\n\n# return the days in whcich volume was low and high \n# accepts the daily volume df as input\ndef split_days_volume(df):\n    criterion=df.median()\n    return df[df['day_volume']<= float(criterion)].index, df[df['day_volume']> float(criterion)].index\n\n# # # # # # # # # Date time Functions # # # # # # # # # # # \n\n# Fill gaps in timeseries by padding - NOT USED CURRENTLY\ndef fill_time(df):\n    if len((df.index[1:]-df.index[:-1]).value_counts().tolist()) > 1:\n        df = df.reindex(range(df.index[0],df.index[-1]+60,60),method='pad')\n    else:\n        print('No gaps found in time series')\n    return df\n\n# convert unix to datetime data    \ndef unix2datetime (df):\n    df.index = pd.to_datetime(df.index,unit='s')\n    return df\n\n\ndef add_time_labels (df):\n    df['year'] = df.index.strftime('%Y')\n    df['month'] = df.index.strftime('%b')\n    df['date'] = df.index.strftime('%d')\n    df['hour'] = df.index.strftime('%H')\n    df['Day_of_week'] = df.index.strftime('%a')\n    df['one_day'] = df['date']+ df['month'] + df['year']\n    \n    return df\n\n# # # # # # # # # # # Feature engineering functions # # # # # # # # # # # \n\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)\n\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef spread(df):\n    return df['High'] - df['Low']\n\ndef mean_trade(df):\n    return df['Volume']/df['Count']\n\ndef log_change(series1, series2):\n    return np.log(series1/series2)\n\n# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)\n                ","metadata":{"execution":{"iopub.status.busy":"2022-01-13T19:40:36.936005Z","iopub.execute_input":"2022-01-13T19:40:36.937033Z","iopub.status.idle":"2022-01-13T19:40:36.985134Z","shell.execute_reply.started":"2022-01-13T19:40:36.936973Z","shell.execute_reply":"2022-01-13T19:40:36.983969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(DATA)\ndf = downcast_floats(df)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T19:40:37.785889Z","iopub.execute_input":"2022-01-13T19:40:37.786878Z","iopub.status.idle":"2022-01-13T19:41:27.397658Z","shell.execute_reply.started":"2022-01-13T19:40:37.786814Z","shell.execute_reply":"2022-01-13T19:41:27.396656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nasset_details = pd.read_csv(DATA_ASSETS)\nt = {0: 'BNB', 1: 'BTC', 2: 'BCH', 3: 'ADA',4: 'DOGE',5: 'EOS',6: 'ETH',7: 'ETC',8: 'MIOTA',9: 'LTC'\n    , 10: 'MKR',11: 'XMR',12: 'XLM',13: 'TRX'}\nasset_details['ticker'] = asset_details['Asset_ID'].map(t)\nasset_details.set_index('ticker',inplace=True)\nbtc, eth = select_assets('BTC', 'ETH' )\ndel df\ngc.collect()\n# In stead of filling in missing values,we choose to work on the intersection of common time series data\n# between the two coins which are examined.\nbtc, eth = clean_match(btc, eth)\n#btc = fill_time(btc)\n#eth = fill_time(eth)\n\n\nbtc['upperShadow'] = upper_shadow (btc)\nbtc['lowerShadow'] = lower_shadow (btc)\n\neth['upperShadow'] = upper_shadow (eth)\neth['lowerShadow'] = lower_shadow (eth)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T19:41:27.399627Z","iopub.execute_input":"2022-01-13T19:41:27.400196Z","iopub.status.idle":"2022-01-13T19:42:12.918629Z","shell.execute_reply.started":"2022-01-13T19:41:27.400143Z","shell.execute_reply":"2022-01-13T19:42:12.917772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc =  unix2datetime(btc)\neth =  unix2datetime(eth)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T19:45:14.439416Z","iopub.execute_input":"2022-01-13T19:45:14.439847Z","iopub.status.idle":"2022-01-13T19:45:14.509073Z","shell.execute_reply.started":"2022-01-13T19:45:14.439793Z","shell.execute_reply":"2022-01-13T19:45:14.508180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc = add_time_labels(btc)\neth = add_time_labels(eth)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T19:45:16.108869Z","iopub.execute_input":"2022-01-13T19:45:16.109746Z","iopub.status.idle":"2022-01-13T19:47:42.366467Z","shell.execute_reply.started":"2022-01-13T19:45:16.109658Z","shell.execute_reply":"2022-01-13T19:47:42.365327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc_dvol = daily_vol(btc, verbose = True)\neth_dvol = daily_vol(eth, verbose = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T19:47:42.368638Z","iopub.execute_input":"2022-01-13T19:47:42.368959Z","iopub.status.idle":"2022-01-13T20:01:47.113495Z","shell.execute_reply.started":"2022-01-13T19:47:42.368916Z","shell.execute_reply":"2022-01-13T20:01:47.112696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split days by relative daily volume (high / low)  // not used for now\n#dlvol, dhvol = split_days_volume(btc_dvol)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T20:01:47.115040Z","iopub.execute_input":"2022-01-13T20:01:47.115282Z","iopub.status.idle":"2022-01-13T20:01:47.119325Z","shell.execute_reply.started":"2022-01-13T20:01:47.115251Z","shell.execute_reply":"2022-01-13T20:01:47.118443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot day (for example day224) for the examined assets\nday = btc_dvol.index[224] # we can use btc_dvol or eth_dvol the same since they were matched earlier on. \n\n\nasset1 = 'BTC'\nasset2 = 'ETH'\ndf1 = btc\ndf2 = eth\n\nfeature = 'Close'\n\n\ndf_focus1 = df1[(df1['one_day'] == day)]\ndf_focus2 = df2[(df2['one_day'] == day)]\n\ndf_plot = pd.concat([df_focus1[feature], df_focus2[feature]],axis=1)\n#rename the columns\ndf_plot.columns = [feature+asset1, feature+asset2]\n\n\nfig, ax = plt.subplots(figsize=(30,15)) # Sample figsize in inches\nsns.lineplot(data=df_plot[feature+asset1], color=\"g\", linewidth=1.5)\nsns.lineplot(data=df_plot[feature+asset1].rolling('h').mean(), color=\"g\", linewidth=4.5, label=asset1)\n\nax2 = plt.twinx()\nsns.lineplot(data=df_plot[feature+asset2], color=\"b\", linewidth=1.5)\nsns.lineplot(data=df_plot[feature+asset2].rolling('h').mean(), color=\"b\", linewidth=4.5,label=asset2)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-13T20:01:47.121035Z","iopub.execute_input":"2022-01-13T20:01:47.121259Z","iopub.status.idle":"2022-01-13T20:01:48.865580Z","shell.execute_reply.started":"2022-01-13T20:01:47.121231Z","shell.execute_reply":"2022-01-13T20:01:48.864905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}