{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Target Computation\n\nThis notebook attempts to compute target as described here:\n\nhttps://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778\n\nVersion 2.0 improves readability by avoiding some unnecessary shift operations.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom datetime import datetime\nimport time\nimport matplotlib.pyplot as plt\nplt.rcParams[\"figure.figsize\"]=(20,8)\nINPUT=Path(\"../input/g-research-crypto-forecasting\")","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:05:05.074470Z","iopub.execute_input":"2021-11-15T20:05:05.075247Z","iopub.status.idle":"2021-11-15T20:05:05.103548Z","shell.execute_reply.started":"2021-11-15T20:05:05.075142Z","shell.execute_reply":"2021-11-15T20:05:05.102807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ResidualizeMarket(df, mktColumn, window):\n  if mktColumn not in df.columns:\n    return df\n\n  mkt = df[mktColumn]\n\n  num = df.multiply(mkt.values, axis=0).rolling(window).mean().values  #numerator of linear regression coefficient\n  denom = mkt.multiply(mkt.values, axis=0).rolling(window).mean().values  #denominator of linear regression coefficient\n  beta = np.nan_to_num( num.T / denom, nan=0., posinf=0., neginf=0.)  #if regression fell over, use beta of 0\n\n  resultRet = df - (beta * mkt.values).T  #perform residualization\n  resultBeta = 0.*df + beta.T  #shape beta\n\n  return resultRet.drop(columns=[mktColumn]), resultBeta.drop(columns=[mktColumn])","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:05:05.108087Z","iopub.execute_input":"2021-11-15T20:05:05.108345Z","iopub.status.idle":"2021-11-15T20:05:05.115793Z","shell.execute_reply.started":"2021-11-15T20:05:05.108312Z","shell.execute_reply":"2021-11-15T20:05:05.115085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function log_return_ahead computes R_t = log(P_{t+16} / P_{t+1})\ndef log_return_ahead(series, periods=1): \n    return -np.log(series).diff(periods=-periods).shift(-1)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:05:05.116868Z","iopub.execute_input":"2021-11-15T20:05:05.117177Z","iopub.status.idle":"2021-11-15T20:05:05.129149Z","shell.execute_reply.started":"2021-11-15T20:05:05.117151Z","shell.execute_reply":"2021-11-15T20:05:05.128531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pd.read_csv(INPUT/\"train.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:05:05.131058Z","iopub.execute_input":"2021-11-15T20:05:05.131407Z","iopub.status.idle":"2021-11-15T20:06:04.702844Z","shell.execute_reply.started":"2021-11-15T20:05:05.131369Z","shell.execute_reply":"2021-11-15T20:06:04.701821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Price of assets\n$$P^a$$","metadata":{}},{"cell_type":"code","source":"prices = train_df.pivot(index=[\"timestamp\"], columns=[\"Asset_ID\"], values=[\"Close\"])","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:04.704197Z","iopub.execute_input":"2021-11-15T20:06:04.704404Z","iopub.status.idle":"2021-11-15T20:06:13.213450Z","shell.execute_reply.started":"2021-11-15T20:06:04.704379Z","shell.execute_reply":"2021-11-15T20:06:13.212530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.columns = [f\"A{a}\" for a in range(14)]","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:13.216373Z","iopub.execute_input":"2021-11-15T20:06:13.217336Z","iopub.status.idle":"2021-11-15T20:06:13.221762Z","shell.execute_reply.started":"2021-11-15T20:06:13.217248Z","shell.execute_reply":"2021-11-15T20:06:13.220866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices = prices.reindex(range(prices.index[0], prices.index[-1]+60,60), method='pad')","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:13.223396Z","iopub.execute_input":"2021-11-15T20:06:13.223828Z","iopub.status.idle":"2021-11-15T20:06:13.554642Z","shell.execute_reply.started":"2021-11-15T20:06:13.223787Z","shell.execute_reply":"2021-11-15T20:06:13.553834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:13.555955Z","iopub.execute_input":"2021-11-15T20:06:13.556579Z","iopub.status.idle":"2021-11-15T20:06:13.571099Z","shell.execute_reply.started":"2021-11-15T20:06:13.556543Z","shell.execute_reply":"2021-11-15T20:06:13.570287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.index = prices.index.map(lambda x: datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:13.573015Z","iopub.execute_input":"2021-11-15T20:06:13.573313Z","iopub.status.idle":"2021-11-15T20:06:18.502707Z","shell.execute_reply.started":"2021-11-15T20:06:13.573272Z","shell.execute_reply":"2021-11-15T20:06:18.501816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.sort_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:18.505541Z","iopub.execute_input":"2021-11-15T20:06:18.505798Z","iopub.status.idle":"2021-11-15T20:06:18.513282Z","shell.execute_reply.started":"2021-11-15T20:06:18.505767Z","shell.execute_reply":"2021-11-15T20:06:18.512489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.tail()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:18.514562Z","iopub.execute_input":"2021-11-15T20:06:18.514830Z","iopub.status.idle":"2021-11-15T20:06:18.538790Z","shell.execute_reply.started":"2021-11-15T20:06:18.514801Z","shell.execute_reply":"2021-11-15T20:06:18.538151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Log Returns over 15 Minutes\n\n$$R^a(t) = log (P^a(t+16)\\ /\\ P^a(t+1))$$\n","metadata":{}},{"cell_type":"code","source":"log_returns_15min = log_return_ahead(prices, periods=15)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:18.539958Z","iopub.execute_input":"2021-11-15T20:06:18.540324Z","iopub.status.idle":"2021-11-15T20:06:19.344115Z","shell.execute_reply.started":"2021-11-15T20:06:18.540275Z","shell.execute_reply":"2021-11-15T20:06:19.343338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_returns_15min.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:19.345310Z","iopub.execute_input":"2021-11-15T20:06:19.345636Z","iopub.status.idle":"2021-11-15T20:06:19.384113Z","shell.execute_reply.started":"2021-11-15T20:06:19.345598Z","shell.execute_reply":"2021-11-15T20:06:19.383211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_returns_15min.tail()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:19.385206Z","iopub.execute_input":"2021-11-15T20:06:19.385426Z","iopub.status.idle":"2021-11-15T20:06:19.403098Z","shell.execute_reply.started":"2021-11-15T20:06:19.385400Z","shell.execute_reply":"2021-11-15T20:06:19.402285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_returns_15min[-200:].plot(grid=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:19.404770Z","iopub.execute_input":"2021-11-15T20:06:19.405244Z","iopub.status.idle":"2021-11-15T20:06:19.971683Z","shell.execute_reply.started":"2021-11-15T20:06:19.405201Z","shell.execute_reply":"2021-11-15T20:06:19.970529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Weighted Average Market Returns\n\n$$M(t) = \\frac{\\sum_a w^a R^a(t)}{\\sum_a w^a}  $$","metadata":{}},{"cell_type":"code","source":"assets_df = pd.read_csv(INPUT/\"asset_details.csv\", index_col = \"Asset_ID\")\nassets_df.sort_index(inplace=True)\nassets_df","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:19.972989Z","iopub.execute_input":"2021-11-15T20:06:19.973941Z","iopub.status.idle":"2021-11-15T20:06:19.993900Z","shell.execute_reply.started":"2021-11-15T20:06:19.973897Z","shell.execute_reply":"2021-11-15T20:06:19.992857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = assets_df.Weight.values\nweights","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:19.995353Z","iopub.execute_input":"2021-11-15T20:06:19.996610Z","iopub.status.idle":"2021-11-15T20:06:20.003828Z","shell.execute_reply.started":"2021-11-15T20:06:19.996560Z","shell.execute_reply":"2021-11-15T20:06:20.002988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weighted_avg_market_log_returns = log_returns_15min.mul(weights, axis='columns').mean(axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:20.005109Z","iopub.execute_input":"2021-11-15T20:06:20.006040Z","iopub.status.idle":"2021-11-15T20:06:20.098097Z","shell.execute_reply.started":"2021-11-15T20:06:20.005993Z","shell.execute_reply":"2021-11-15T20:06:20.097447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_returns_15min.mul(weights, axis='columns')[-200:].plot()\nweighted_avg_market_log_returns[-200:].plot(style=\"k8\", grid=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:20.099544Z","iopub.execute_input":"2021-11-15T20:06:20.100128Z","iopub.status.idle":"2021-11-15T20:06:20.637890Z","shell.execute_reply.started":"2021-11-15T20:06:20.100087Z","shell.execute_reply":"2021-11-15T20:06:20.637028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_returns_15min[\"market\"] = weighted_avg_market_log_returns\nresidualized_market_returns, beta = ResidualizeMarket(log_returns_15min, \"market\", window=3750)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:20.639081Z","iopub.execute_input":"2021-11-15T20:06:20.639386Z","iopub.status.idle":"2021-11-15T20:06:22.934393Z","shell.execute_reply.started":"2021-11-15T20:06:20.639352Z","shell.execute_reply":"2021-11-15T20:06:22.933493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residualized_market_returns[-200:].plot(grid=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:22.935597Z","iopub.execute_input":"2021-11-15T20:06:22.935827Z","iopub.status.idle":"2021-11-15T20:06:23.526327Z","shell.execute_reply.started":"2021-11-15T20:06:22.935801Z","shell.execute_reply":"2021-11-15T20:06:23.525742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compare computed with provided target","metadata":{}},{"cell_type":"code","source":"target = train_df.pivot(index=[\"timestamp\"], columns=[\"Asset_ID\"], values=[\"Target\"])","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:23.527257Z","iopub.execute_input":"2021-11-15T20:06:23.527647Z","iopub.status.idle":"2021-11-15T20:06:31.861846Z","shell.execute_reply.started":"2021-11-15T20:06:23.527614Z","shell.execute_reply":"2021-11-15T20:06:31.860807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.columns = [f\"A{a}\" for a in range(14)]","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:31.863070Z","iopub.execute_input":"2021-11-15T20:06:31.863310Z","iopub.status.idle":"2021-11-15T20:06:31.868381Z","shell.execute_reply.started":"2021-11-15T20:06:31.863280Z","shell.execute_reply":"2021-11-15T20:06:31.867500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = target.reindex(range(target.index[0], target.index[-1]+60,60), method='pad')","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:31.869559Z","iopub.execute_input":"2021-11-15T20:06:31.869856Z","iopub.status.idle":"2021-11-15T20:06:32.181204Z","shell.execute_reply.started":"2021-11-15T20:06:31.869824Z","shell.execute_reply":"2021-11-15T20:06:32.180404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.index = target.index.map(lambda x: datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:32.182581Z","iopub.execute_input":"2021-11-15T20:06:32.182893Z","iopub.status.idle":"2021-11-15T20:06:36.995027Z","shell.execute_reply.started":"2021-11-15T20:06:32.182856Z","shell.execute_reply":"2021-11-15T20:06:36.994290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target.sort_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:36.996104Z","iopub.execute_input":"2021-11-15T20:06:36.996349Z","iopub.status.idle":"2021-11-15T20:06:37.003556Z","shell.execute_reply.started":"2021-11-15T20:06:36.996322Z","shell.execute_reply":"2021-11-15T20:06:37.002782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target[-200:].plot(grid=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:37.004651Z","iopub.execute_input":"2021-11-15T20:06:37.004904Z","iopub.status.idle":"2021-11-15T20:06:37.498200Z","shell.execute_reply.started":"2021-11-15T20:06:37.004876Z","shell.execute_reply":"2021-11-15T20:06:37.497333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residualized_market_returns[\"A0\"][-200:].plot(grid=True)\ntarget[\"A0\"][-200:].plot(style='r--', grid=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:37.502856Z","iopub.execute_input":"2021-11-15T20:06:37.503206Z","iopub.status.idle":"2021-11-15T20:06:37.832760Z","shell.execute_reply.started":"2021-11-15T20:06:37.503160Z","shell.execute_reply":"2021-11-15T20:06:37.831490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residualized_market_returns[\"A1\"][-200:].plot()\ntarget[\"A1\"][-200:].plot(style='r--',grid=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:37.834485Z","iopub.execute_input":"2021-11-15T20:06:37.835645Z","iopub.status.idle":"2021-11-15T20:06:38.145810Z","shell.execute_reply.started":"2021-11-15T20:06:37.835595Z","shell.execute_reply":"2021-11-15T20:06:38.144986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_diffs = residualized_market_returns - target","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:38.146933Z","iopub.execute_input":"2021-11-15T20:06:38.147248Z","iopub.status.idle":"2021-11-15T20:06:38.286086Z","shell.execute_reply.started":"2021-11-15T20:06:38.147219Z","shell.execute_reply":"2021-11-15T20:06:38.285478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_diffs.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:38.287112Z","iopub.execute_input":"2021-11-15T20:06:38.287407Z","iopub.status.idle":"2021-11-15T20:06:38.374791Z","shell.execute_reply.started":"2021-11-15T20:06:38.287370Z","shell.execute_reply":"2021-11-15T20:06:38.373863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.quantile(target_diffs, [0.025, 0.975])","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:38.375952Z","iopub.execute_input":"2021-11-15T20:06:38.376176Z","iopub.status.idle":"2021-11-15T20:06:38.487079Z","shell.execute_reply.started":"2021-11-15T20:06:38.376150Z","shell.execute_reply":"2021-11-15T20:06:38.486292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(target_diffs.values.reshape(-1), bins=1000)\nplt.xlim((-0.01,0.01))\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T20:06:38.488008Z","iopub.execute_input":"2021-11-15T20:06:38.488225Z","iopub.status.idle":"2021-11-15T20:06:40.698021Z","shell.execute_reply.started":"2021-11-15T20:06:38.488196Z","shell.execute_reply":"2021-11-15T20:06:40.697152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conclusion\n\nThe targets provided in `train.csv` are very close to the values computed by ```ResidualizeMarket``` function provided here https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778.\n","metadata":{}}]}