{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction to Crypto Forecasting\n\nOn-going work.","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\nimport pandas as pd\nimport numpy as np\nimport os\n\nimport time\nfrom datetime import datetime\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\npd.options.display.max_rows = 999\n%config InlineBackend.figure_format = 'retina'\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:51:04.925730Z","iopub.execute_input":"2021-11-08T16:51:04.926137Z","iopub.status.idle":"2021-11-08T16:51:05.882986Z","shell.execute_reply.started":"2021-11-08T16:51:04.926095Z","shell.execute_reply":"2021-11-08T16:51:05.881944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training data is in the competition dataset as usual","metadata":{"papermill":{"duration":0.011622,"end_time":"2020-10-16T21:06:02.516712","exception":false,"start_time":"2020-10-16T21:06:02.50509","status":"completed"},"tags":[]}},{"cell_type":"code","source":"dtype={'Asset_ID': 'int8', 'Count': 'int32', 'row_id': 'int32', 'Count': 'int32',\n       'Open': 'float32', 'High': 'float32', 'Low': 'float32', 'Close': 'float32',\n       'Volume': 'float32', 'VWAP': 'float32'}\n\ntrain_df = pd.read_csv('../input/g-research-crypto-forecasting/train.csv', low_memory=False, dtype=dtype)\nasset_details = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')","metadata":{"papermill":{"duration":0.273123,"end_time":"2020-10-16T21:06:02.800188","exception":false,"start_time":"2020-10-16T21:06:02.527065","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-11-08T16:51:05.885092Z","iopub.execute_input":"2021-11-08T16:51:05.885450Z","iopub.status.idle":"2021-11-08T16:52:00.840537Z","shell.execute_reply.started":"2021-11-08T16:51:05.885407Z","shell.execute_reply":"2021-11-08T16:52:00.839587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\n# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:00.841894Z","iopub.execute_input":"2021-11-08T16:52:00.842112Z","iopub.status.idle":"2021-11-08T16:52:00.849976Z","shell.execute_reply.started":"2021-11-08T16:52:00.842087Z","shell.execute_reply":"2021-11-08T16:52:00.849032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Asset Details","metadata":{}},{"cell_type":"code","source":"len(train_df.Asset_ID.unique())","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:00.852301Z","iopub.execute_input":"2021-11-08T16:52:00.852757Z","iopub.status.idle":"2021-11-08T16:52:01.040086Z","shell.execute_reply.started":"2021-11-08T16:52:00.852725Z","shell.execute_reply":"2021-11-08T16:52:01.039235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:01.041266Z","iopub.execute_input":"2021-11-08T16:52:01.041503Z","iopub.status.idle":"2021-11-08T16:52:01.058119Z","shell.execute_reply.started":"2021-11-08T16:52:01.041475Z","shell.execute_reply":"2021-11-08T16:52:01.057131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# time stamps","metadata":{}},{"cell_type":"code","source":"len(train_df.timestamp.unique())","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:01.059425Z","iopub.execute_input":"2021-11-08T16:52:01.059813Z","iopub.status.idle":"2021-11-08T16:52:01.665419Z","shell.execute_reply.started":"2021-11-08T16:52:01.059782Z","shell.execute_reply":"2021-11-08T16:52:01.664571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"min date\", datetime.fromtimestamp(min(train_df.timestamp.unique())))\nprint(\"max date\", datetime.fromtimestamp(max(train_df.timestamp.unique())))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:01.666518Z","iopub.execute_input":"2021-11-08T16:52:01.667165Z","iopub.status.idle":"2021-11-08T16:52:03.280753Z","shell.execute_reply.started":"2021-11-08T16:52:01.667122Z","shell.execute_reply":"2021-11-08T16:52:03.279616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(np.diff(train_df.timestamp.unique()))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:03.282132Z","iopub.execute_input":"2021-11-08T16:52:03.282641Z","iopub.status.idle":"2021-11-08T16:52:04.485705Z","shell.execute_reply.started":"2021-11-08T16:52:03.282585Z","shell.execute_reply":"2021-11-08T16:52:04.484834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"esp = train_df.timestamp.unique()[np.insert(np.diff(train_df.timestamp.unique())>60,False,0)]\nval = np.diff(train_df.timestamp.unique())[np.diff(train_df.timestamp.unique())>60]\n\npd.DataFrame({'day':[datetime.fromtimestamp(timestamp).strftime(\"%d/%m/%y\") for timestamp in esp],'delta':val}).groupby('day').agg([np.size,np.mean])","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:04.487257Z","iopub.execute_input":"2021-11-08T16:52:04.487776Z","iopub.status.idle":"2021-11-08T16:52:06.738427Z","shell.execute_reply.started":"2021-11-08T16:52:04.487732Z","shell.execute_reply":"2021-11-08T16:52:06.737579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"esp = train_df.timestamp.unique()[np.insert(np.diff(train_df.timestamp.unique())>120,False,0)]\nval = np.diff(train_df.timestamp.unique())[np.diff(train_df.timestamp.unique())>120]\n\npd.DataFrame({'day':[datetime.fromtimestamp(timestamp).strftime(\"%d/%m/%y\") for timestamp in esp],'delta':val}).groupby('day').count()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:06.740758Z","iopub.execute_input":"2021-11-08T16:52:06.740992Z","iopub.status.idle":"2021-11-08T16:52:09.084561Z","shell.execute_reply.started":"2021-11-08T16:52:06.740963Z","shell.execute_reply":"2021-11-08T16:52:09.083610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Weird things happened on 16/10/2019 and 23/10/2019","metadata":{}},{"cell_type":"markdown","source":"# outliers","metadata":{}},{"cell_type":"markdown","source":"remove target na ?","metadata":{}},{"cell_type":"code","source":"#train_df = train_df[~train_df.Target.isna()]","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:09.085931Z","iopub.execute_input":"2021-11-08T16:52:09.086160Z","iopub.status.idle":"2021-11-08T16:52:09.089867Z","shell.execute_reply.started":"2021-11-08T16:52:09.086136Z","shell.execute_reply":"2021-11-08T16:52:09.089101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# data distribution","metadata":{}},{"cell_type":"code","source":"train_df['timestamp'].hist(by=train_df['Asset_ID']);","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:09.090852Z","iopub.execute_input":"2021-11-08T16:52:09.091717Z","iopub.status.idle":"2021-11-08T16:52:13.400561Z","shell.execute_reply.started":"2021-11-08T16:52:09.091672Z","shell.execute_reply":"2021-11-08T16:52:13.399727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some data appears with less history. Esp. Asset #10 and #11.","metadata":{}},{"cell_type":"code","source":"train_df['Count'].hist(by=train_df['Asset_ID'], log=True);","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:13.402049Z","iopub.execute_input":"2021-11-08T16:52:13.404841Z","iopub.status.idle":"2021-11-08T16:52:17.648672Z","shell.execute_reply.started":"2021-11-08T16:52:13.404794Z","shell.execute_reply":"2021-11-08T16:52:17.647747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Open'].hist(by=train_df['Asset_ID'], log=True);","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:17.650315Z","iopub.execute_input":"2021-11-08T16:52:17.650856Z","iopub.status.idle":"2021-11-08T16:52:24.070403Z","shell.execute_reply.started":"2021-11-08T16:52:17.650811Z","shell.execute_reply":"2021-11-08T16:52:24.069514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Volume'].hist(by=train_df['Asset_ID'], log=True);","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:24.071711Z","iopub.execute_input":"2021-11-08T16:52:24.072270Z","iopub.status.idle":"2021-11-08T16:52:28.087062Z","shell.execute_reply.started":"2021-11-08T16:52:24.072228Z","shell.execute_reply":"2021-11-08T16:52:28.086198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[~np.isinf(train_df['VWAP'])].hist(by=train_df[~np.isinf(train_df['VWAP'])]['Asset_ID'], log=True);","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:28.088573Z","iopub.execute_input":"2021-11-08T16:52:28.088894Z","iopub.status.idle":"2021-11-08T16:52:58.205094Z","shell.execute_reply.started":"2021-11-08T16:52:28.088862Z","shell.execute_reply":"2021-11-08T16:52:58.204485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Seems to have outliers (ultra low volume ?)","metadata":{}},{"cell_type":"markdown","source":"# Normalized prices","metadata":{}},{"cell_type":"code","source":"for i in train_df.Asset_ID.unique():\n    df_p = train_df[['Close','timestamp']][train_df.Asset_ID==i]\n    df_p.index = df_p.timestamp\n    df_p.Close  = df_p.Close/df_p.Close.iloc[0]\n    df_p = df_p.drop(columns=['timestamp'])\n    plt.plot(df_p,label=asset_details[asset_details.Asset_ID == i].Asset_Name.values[0])\n    \nplt.legend(bbox_to_anchor=(1.05, 1),fontsize='x-small')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:52:58.206133Z","iopub.execute_input":"2021-11-08T16:52:58.206916Z","iopub.status.idle":"2021-11-08T16:53:24.130366Z","shell.execute_reply.started":"2021-11-08T16:52:58.206880Z","shell.execute_reply":"2021-11-08T16:53:24.129780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Returns\n\nReturns in time frame v.s. return between time frames ?","metadata":{}},{"cell_type":"code","source":"for i in train_df.Asset_ID.unique():\n    df_p = train_df[['Close','Open','timestamp']][train_df.Asset_ID==i]\n    df_p.index = df_p.timestamp\n    df_p.Close  = np.log(df_p.Close/df_p.Open)\n    df_p = df_p.drop(columns=['timestamp','Open'])\n    plt.plot(df_p,label=asset_details[asset_details.Asset_ID == i].Asset_Name.values[0])\n    \nplt.legend(loc='upper left',fontsize='x-small')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:53:24.131483Z","iopub.execute_input":"2021-11-08T16:53:24.132274Z","iopub.status.idle":"2021-11-08T16:53:33.145007Z","shell.execute_reply.started":"2021-11-08T16:53:24.132241Z","shell.execute_reply":"2021-11-08T16:53:33.144399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_df.Asset_ID.unique():\n    \n    df_p = train_df[['Close','timestamp']][train_df.Asset_ID==i]\n    df_p.index = df_p.timestamp\n\n    df_p.Close  = np.log(df_p.Close/df_p.Close.shift())\n    \n    #train_df.loc[train_df.Asset_ID==i,'log_ret_diff'] = df_p.Close \n    \n    df_p = df_p.drop(columns=['timestamp'])\n    plt.plot(df_p,label=asset_details[asset_details.Asset_ID == i].Asset_Name.values[0])\n    \nplt.legend(loc='upper left',fontsize='x-small')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:53:33.146042Z","iopub.execute_input":"2021-11-08T16:53:33.146472Z","iopub.status.idle":"2021-11-08T16:53:41.331737Z","shell.execute_reply.started":"2021-11-08T16:53:33.146437Z","shell.execute_reply":"2021-11-08T16:53:41.330897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# volume","metadata":{}},{"cell_type":"code","source":"for i in train_df.Asset_ID.unique():\n    df_p = train_df[['Volume','timestamp']][train_df.Asset_ID==i]\n    df_p.index = df_p.timestamp\n    df_p = df_p.drop(columns=['timestamp'])\n    plt.plot(df_p,label=asset_details[asset_details.Asset_ID == i].Asset_Name.values[0])\n    \nplt.legend(loc='upper left',fontsize='x-small')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:53:41.332919Z","iopub.execute_input":"2021-11-08T16:53:41.333164Z","iopub.status.idle":"2021-11-08T16:53:48.075939Z","shell.execute_reply.started":"2021-11-08T16:53:41.333136Z","shell.execute_reply":"2021-11-08T16:53:48.074831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Volatility","metadata":{}},{"cell_type":"markdown","source":"Garmann-Klass estimator for volatility - smoothed. Maybe better with inter time volatility ?","metadata":{}},{"cell_type":"code","source":"for i in train_df.Asset_ID.unique():\n    df_p = train_df[['Open','High','Low','Close','timestamp']][train_df.Asset_ID==i]\n    df_p.index = df_p.timestamp\n    df_p['GK_vol'] = (1 / 2 * np.log(df_p.High / df_p.Low) ** 2 - (2 * np.log(2) - 1) * np.log(df_p.Close / df_p.Open) ** 2).astype('float32')\n    df_p = df_p.drop(columns=['Open','High','Low','Close','timestamp'])\n    plt.plot(np.log(df_p.rolling(2400).mean()),label=asset_details[asset_details.Asset_ID == i].Asset_Name.values[0])\n    \nplt.legend(loc='upper left',fontsize='x-small')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:53:48.077188Z","iopub.execute_input":"2021-11-08T16:53:48.077519Z","iopub.status.idle":"2021-11-08T16:53:59.399271Z","shell.execute_reply.started":"2021-11-08T16:53:48.077485Z","shell.execute_reply":"2021-11-08T16:53:59.398373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\n\nsns.color_palette(\"tab10\")\nsns.kdeplot(data=train_df, x='Target', hue='Asset_ID', fill=True, common_norm=False, alpha=0.4)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:53:59.400709Z","iopub.execute_input":"2021-11-08T16:53:59.401004Z","iopub.status.idle":"2021-11-08T16:55:35.693571Z","shell.execute_reply.started":"2021-11-08T16:53:59.400967Z","shell.execute_reply":"2021-11-08T16:55:35.692570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Market","metadata":{}},{"cell_type":"code","source":"%%time\n\nfeatures_to_aggregate = ['log_ret']\ntrain_df['log_ret'] = np.log(train_df.Close/train_df.Open)\n\ndict_weights = {}\n\nfor i in range(asset_details.shape[0]):\n    dict_weights[asset_details.iloc[i,0]] = asset_details.iloc[i,1]\n    \ntrain_df['weights'] = train_df.Asset_ID.map(dict_weights).astype('float32')\n\nt, w, A_id = (train_df[col].values for col in ['timestamp','weights','Asset_ID'])\nids, index = np.unique(t, return_index=True)\n\nValues = train_df[features_to_aggregate].values\nsplits = np.split(Values, index[1:])\nsplits_w = np.split(w, index[1:])\nsplits_A_id = np.split(A_id, index[1:])\n\nret_small = []\nret = []\n\nfor time_id, x, w, A_id in zip(ids.tolist(), splits, splits_w, splits_A_id):\n    outputs = np.float32(np.sum((x.T*w),axis=1)/sum(w))\n    ret_small.append(outputs)\n    ret.append(np.tile(outputs, (len(w), 1)))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:55:35.694795Z","iopub.execute_input":"2021-11-08T16:55:35.695011Z","iopub.status.idle":"2021-11-08T16:56:39.675332Z","shell.execute_reply.started":"2021-11-08T16:55:35.694987Z","shell.execute_reply":"2021-11-08T16:56:39.674368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"market_ret = pd.DataFrame({'timestamp':ids,'log_ret_M':np.concatenate(ret_small,axis=0).flatten()})","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:57:25.436739Z","iopub.execute_input":"2021-11-08T16:57:25.437036Z","iopub.status.idle":"2021-11-08T16:57:26.201132Z","shell.execute_reply.started":"2021-11-08T16:57:25.437006Z","shell.execute_reply":"2021-11-08T16:57:26.200105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df['log_ret_M'] = np.concatenate(ret,axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:56:40.445734Z","iopub.execute_input":"2021-11-08T16:56:40.446125Z","iopub.status.idle":"2021-11-08T16:56:40.450476Z","shell.execute_reply.started":"2021-11-08T16:56:40.446082Z","shell.execute_reply":"2021-11-08T16:56:40.449557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"market_ret.index = market_ret.timestamp","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:56:40.453203Z","iopub.execute_input":"2021-11-08T16:56:40.453577Z","iopub.status.idle":"2021-11-08T16:56:40.471327Z","shell.execute_reply.started":"2021-11-08T16:56:40.453545Z","shell.execute_reply":"2021-11-08T16:56:40.470643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Price on log scale / compared to market","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:56:40.472754Z","iopub.execute_input":"2021-11-08T16:56:40.473221Z","iopub.status.idle":"2021-11-08T16:56:40.498046Z","shell.execute_reply.started":"2021-11-08T16:56:40.473180Z","shell.execute_reply":"2021-11-08T16:56:40.496862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cumulated log returns","metadata":{}},{"cell_type":"code","source":"for i in train_df.Asset_ID.unique():\n    df_p = train_df[['log_ret','timestamp']][train_df.Asset_ID==i]\n    df_p.index = df_p.timestamp\n    df_p = df_p.drop(columns=['timestamp'])\n    plt.plot(df_p.cumsum(),label=asset_details[asset_details.Asset_ID == i].Asset_Name.values[0])\n    \nplt.plot(market_ret.log_ret_M.cumsum(),label='Market',c='Black')\nplt.legend(loc='upper left',fontsize='x-small')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:56:40.502619Z","iopub.execute_input":"2021-11-08T16:56:40.503372Z","iopub.status.idle":"2021-11-08T16:56:47.907719Z","shell.execute_reply.started":"2021-11-08T16:56:40.503329Z","shell.execute_reply":"2021-11-08T16:56:47.906665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation with target - Beta\n\nResidualised Log returns appears generally negatively correlated with the target. I exploit this idea in this ultra fast baseline notebook: https://www.kaggle.com/lucasmorin/minimal-fast-baseline","metadata":{}},{"cell_type":"code","source":"train_df['log_ret_M'] = np.concatenate(ret,axis=0).flatten()\ntrain_df['log_ret_M2'] = train_df['log_ret_M']**2\ntrain_df['log_ret_Mr'] = train_df['log_ret_M'] * train_df['log_ret']","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:57:58.817505Z","iopub.execute_input":"2021-11-08T16:57:58.817830Z","iopub.status.idle":"2021-11-08T16:57:59.861396Z","shell.execute_reply.started":"2021-11-08T16:57:58.817796Z","shell.execute_reply":"2021-11-08T16:57:59.860692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_df.Asset_ID.unique():\n    df_p = train_df[['Asset_ID','log_ret_Mr','log_ret_M2','timestamp']][train_df.Asset_ID==i]\n    df_p['mr_rolling'] = df_p['log_ret_Mr'].rolling(window=3750, min_periods=3750).mean()\n    df_p['m2_rolling'] = df_p['log_ret_M2'].rolling(window=3750, min_periods=3750).mean()\n    df_p['beta'] = df_p['mr_rolling'] / df_p['m2_rolling']\n    \n    plt.plot(df_p['beta'],label=asset_details[asset_details.Asset_ID == i].Asset_Name.values[0])\n    \nplt.legend(loc='upper left',fontsize='x-small')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T17:02:15.988707Z","iopub.execute_input":"2021-11-08T17:02:15.989009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_df.Asset_ID.unique():\n    \n    print('Asset: '+asset_details[asset_details.Asset_ID == i].Asset_Name.values[0])\n    \n    \n    df_p = train_df[['log_ret','log_ret_M','Target','timestamp']][train_df.Asset_ID==i]\n    \n    df_p.index = df_p.timestamp\n    \n    df_p['log_ret_res'] = df_p['log_ret'] - df_p['log_ret_M']\n    \n    #corr_time = df_p.groupby(df_p.index//(3750*60)).corr().loc[:,\"Target\"].loc[:,'log_ret']\n    #corr_time2 = df_p.groupby(df_p.index//(3750*60)).corr().loc[:,\"Target\"].loc[:,'log_ret_M']\n    corr_time3 = df_p.groupby(df_p.index//(3750*60)).corr().loc[:,'log_ret'].loc[:,'log_ret_M']\n    #corr_time4 = df_p.groupby(df_p.index//(3750*60)).corr().loc[:,\"Target\"].loc[:,'log_ret_res']\n    \n    #print('Asset: Correlation between target and timestamp log returns: '+str(corr_time.mean()))\n    #print('Asset: Correlation between target and timestamp market log returns: '+str(corr_time2.mean()))\n    print('Asset: Beta: '+str(corr_time3.mean()))\n    #print('Asset: Correlation between target and timestamp residualisez log returns: '+str(corr_time4.mean()))\n    \n    \"\"\"\n    \n    corr_time.plot();\n    plt.xticks([])\n    plt.ylabel(\"Correlation\")\n    plt.title(\"Correlation between target and timestamp log returns\");\n    plt.show();\n    \n    corr_time2.plot();\n    plt.xticks([])\n    plt.ylabel(\"Correlation\")\n    plt.title(\"Correlation between target and timestamp market log returns\");\n    plt.show();\"\"\"\n    \n    corr_time3.plot();\n    plt.xticks([])\n    plt.ylabel(\"Correlation\")\n    plt.title(\"Correlation between timestamp log return and timestamp market log return\");\n    #plt.show();\n    \n    \"\"\"\n    corr_time4.plot();\n    plt.xticks([])\n    plt.ylabel(\"Correlation\")\n    plt.title(\"Correlation between target and timestamp residualisez log returns\");\n    plt.show();\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:56:49.027630Z","iopub.status.idle":"2021-11-08T16:56:49.028092Z","shell.execute_reply.started":"2021-11-08T16:56:49.027850Z","shell.execute_reply":"2021-11-08T16:56:49.027873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation\n\nTO DO: over time","metadata":{}},{"cell_type":"code","source":"#create dataframe with returns for all assets\nall_assets_2021 = pd.DataFrame([])\nfor asset_id, asset_name in zip(asset_details.Asset_ID, asset_details.Asset_Name):\n  asset = train_df[train_df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n  asset = asset.loc[totimestamp('01/01/2021'):totimestamp('01/05/2021')]\n  asset = asset.reindex(range(asset.index[0],asset.index[-1]+60,60),method='pad')\n  lret = log_return(asset.Close.fillna(0))[1:]\n  all_assets_2021 = all_assets_2021.join(lret, rsuffix=asset_name, how=\"outer\")\nplt.imshow(all_assets_2021.corr());\nplt.yticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values);\nplt.xticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values, rotation='vertical');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2021-11-08T16:56:49.029809Z","iopub.status.idle":"2021-11-08T16:56:49.030135Z","shell.execute_reply.started":"2021-11-08T16:56:49.029962Z","shell.execute_reply":"2021-11-08T16:56:49.029982Z"},"trusted":true},"execution_count":null,"outputs":[]}]}