{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Target Computation\n\nBased off of https://www.kaggle.com/vi2018/g-research-crypto-repro-target-computation\n\nThis notebook attempts to compute target as described here:\n\nhttps://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778\n\nVersion 2.0 improves readability by avoiding some unnecessary shift operations.","metadata":{}},{"cell_type":"markdown","source":"## Target Computation\n\nBased off of https://www.kaggle.com/vi2018/g-research-crypto-repro-target-computation\n\nThis notebook attempts to compute target as described here:\n\nhttps://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778\n\nVersion 2.0 improves readability by avoiding some unnecessary shift operations.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom datetime import datetime\nimport time\n\nINPUT=Path(\"../input/g-research-crypto-forecasting\")","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:38:30.579046Z","iopub.execute_input":"2021-11-17T02:38:30.579672Z","iopub.status.idle":"2021-11-17T02:38:30.585151Z","shell.execute_reply.started":"2021-11-17T02:38:30.579625Z","shell.execute_reply":"2021-11-17T02:38:30.584343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ResidualizeMarket(df, mktColumn, window):\n  if mktColumn not in df.columns:\n    return df\n\n  mkt = df[mktColumn]\n  num = df.multiply(mkt.values, axis=0).rolling(window).mean() #numerator of linear regression coefficient\n  denom = mkt.multiply(mkt.values, axis=0).rolling(window).mean()  #denominator of linear regression coefficient\n  beta = np.nan_to_num( num.values.T / denom.values, nan=0., posinf=0., neginf=0.)  #if regression fell over, use beta of 0\n\n  resultRet = df - (beta * mkt.values).T  #perform residualization\n  resultBeta = 0.*df + beta.T  #shape beta\n\n  return resultRet.drop(columns=[mktColumn]), resultBeta.drop(columns=[mktColumn])\n","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:57:48.810019Z","iopub.execute_input":"2021-11-17T02:57:48.810362Z","iopub.status.idle":"2021-11-17T02:57:48.818326Z","shell.execute_reply.started":"2021-11-17T02:57:48.810325Z","shell.execute_reply":"2021-11-17T02:57:48.817410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function log_return_ahead computes R_t = log(P_{t+16} / P_{t+1})\ndef log_return_ahead(series, periods=1):\n    return np.exp(-np.log(series).diff(periods=-periods).shift(-1)) - 1","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:38:30.603503Z","iopub.execute_input":"2021-11-17T02:38:30.604008Z","iopub.status.idle":"2021-11-17T02:38:30.617690Z","shell.execute_reply.started":"2021-11-17T02:38:30.603968Z","shell.execute_reply":"2021-11-17T02:38:30.616762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(INPUT/\"train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:38:30.619394Z","iopub.execute_input":"2021-11-17T02:38:30.619887Z","iopub.status.idle":"2021-11-17T02:39:34.915744Z","shell.execute_reply.started":"2021-11-17T02:38:30.619851Z","shell.execute_reply":"2021-11-17T02:39:34.914795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices = train_df.pivot(index=[\"timestamp\"], columns=[\"Asset_ID\"], values=[\"Close\"])\n\nprices.columns = [f\"A{a}\" for a in range(14)]\n\nprices = prices.reindex(range(prices.index[0], prices.index[-1]+60,60), method='ffill')\n\nprices.index = prices.index.map(lambda x: datetime.utcfromtimestamp(x))\nprices['mday'] = prices.index.day\nprices['minute'] = prices.index.minute\nprices['hour'] = prices.index.hour\n# remove first minute of each month\nprices = prices[~((prices['mday']==1) & (prices['minute']==0) & (prices['hour']==0))]\nprices = prices.drop(columns=['mday','minute','hour'])\nprices.sort_index(inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:39:34.917537Z","iopub.execute_input":"2021-11-17T02:39:34.918211Z","iopub.status.idle":"2021-11-17T02:39:48.379665Z","shell.execute_reply.started":"2021-11-17T02:39:34.918155Z","shell.execute_reply":"2021-11-17T02:39:48.378691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = train_df.pivot(index=[\"timestamp\"], columns=[\"Asset_ID\"], values=[\"Target\"])\ntarget.columns = [f\"A{a}\" for a in range(14)]\ntarget = target.reindex(range(target.index[0], target.index[-1]+60,60), method='ffill')\ntarget.index = target.index.map(lambda x: datetime.utcfromtimestamp(x))\ntarget.sort_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:39:48.381880Z","iopub.execute_input":"2021-11-17T02:39:48.382126Z","iopub.status.idle":"2021-11-17T02:40:00.553887Z","shell.execute_reply.started":"2021-11-17T02:39:48.382097Z","shell.execute_reply":"2021-11-17T02:40:00.553126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_returns_15min = log_return_ahead(prices, periods=15)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:40:00.554965Z","iopub.execute_input":"2021-11-17T02:40:00.555174Z","iopub.status.idle":"2021-11-17T02:40:01.873766Z","shell.execute_reply.started":"2021-11-17T02:40:00.555148Z","shell.execute_reply":"2021-11-17T02:40:01.872987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assets_df = pd.read_csv(INPUT/\"asset_details.csv\", index_col = \"Asset_ID\")\nassets_df.sort_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:40:01.874591Z","iopub.execute_input":"2021-11-17T02:40:01.874791Z","iopub.status.idle":"2021-11-17T02:40:01.890030Z","shell.execute_reply.started":"2021-11-17T02:40:01.874767Z","shell.execute_reply":"2021-11-17T02:40:01.889292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = assets_df.Weight.values\nweighted_avg_market_log_returns = log_returns_15min.mul(weights, axis='columns').div(log_returns_15min.notnull().mul(weights, axis='columns').sum(axis=1), axis=0).sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:40:01.891334Z","iopub.execute_input":"2021-11-17T02:40:01.891759Z","iopub.status.idle":"2021-11-17T02:40:02.377197Z","shell.execute_reply.started":"2021-11-17T02:40:01.891730Z","shell.execute_reply":"2021-11-17T02:40:02.376423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_returns_15min[\"market\"] = weighted_avg_market_log_returns\nresidualized_market_returns, beta = ResidualizeMarket(log_returns_15min, \"market\", window=3750)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:57:51.883568Z","iopub.execute_input":"2021-11-17T02:57:51.883921Z","iopub.status.idle":"2021-11-17T02:57:54.255596Z","shell.execute_reply.started":"2021-11-17T02:57:51.883878Z","shell.execute_reply":"2021-11-17T02:57:54.254911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_diffs = residualized_market_returns - target\n\n#target_diffs.dropna(inplace=True)\nprint(np.nanmean(np.abs(target_diffs.values)))\nprint(np.nanmax(np.abs(target_diffs.values)))","metadata":{"execution":{"iopub.status.busy":"2021-11-17T02:57:54.723256Z","iopub.execute_input":"2021-11-17T02:57:54.724202Z","iopub.status.idle":"2021-11-17T02:57:55.723410Z","shell.execute_reply.started":"2021-11-17T02:57:54.724152Z","shell.execute_reply":"2021-11-17T02:57:55.722405Z"},"trusted":true},"execution_count":null,"outputs":[]}]}