{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Target Computation\n\nThis notebook attempts to compute target as described here:\n\nhttps://www.kaggle.com/c/g-research-crypto-forecasting/discussion/286778\n\nVersion 2.0 improves readability by avoiding some unnecessary shift operations.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom datetime import datetime\nimport time\nimport matplotlib.pyplot as plt\nplt.rcParams[\"figure.figsize\"]=(20,8)\nINPUT=Path(\"../input/g-research-crypto-forecasting\")","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:04:20.217509Z","iopub.execute_input":"2022-01-31T05:04:20.218190Z","iopub.status.idle":"2022-01-31T05:04:20.248516Z","shell.execute_reply.started":"2022-01-31T05:04:20.218057Z","shell.execute_reply":"2022-01-31T05:04:20.247878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ResidualizeMarket(df, mktColumn, window):\n    if mktColumn not in df.columns:\n        return df\n\n    mkt = df[mktColumn]\n\n    num = df.multiply(mkt.values, axis=0).rolling(window).mean().values  #numerator of linear regression coefficient\n    denom = mkt.multiply(mkt.values, axis=0).rolling(window).mean().values  #denominator of linear regression coefficient\n    beta = np.nan_to_num( num.T / denom, nan=0., posinf=0., neginf=0.)  #if regression fell over, use beta of 0\n\n    resultRet = df - (beta * mkt.values).T  #perform residualization\n    resultBeta = 0.*df + beta.T  #shape beta \n\n    return resultRet.drop(columns=[mktColumn]), resultBeta.drop(columns=[mktColumn])\n\ndef reduce_memory(df):\n    before = df.memory_usage().sum()  \n    for col in df.columns:        \n        dtype = df[col].dtype\n        if dtype == 'float64':\n            c_min = df[col].min()\n            c_max = df[col].max()        \n            if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                df[col] = df[col].astype(np.float32)\n            else:\n                df[col] = df[col].astype(np.float64)\n\n    df['Asset_ID'] = df['Asset_ID'].astype('int8')\n    df['Count'] = df['Count'].astype('int32')\n    df['timestamp'] = df['timestamp'].astype('uint32')\n                    \n    after = df.memory_usage().sum()\n    \n    print('Memory taken before transformation : ', before)\n    print('Memory taken after transformation : ', after)\n    print('Memory taken reduced by : ',( before - after) * 100/ before, '%')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:04:20.249774Z","iopub.execute_input":"2022-01-31T05:04:20.250124Z","iopub.status.idle":"2022-01-31T05:04:20.262548Z","shell.execute_reply.started":"2022-01-31T05:04:20.250093Z","shell.execute_reply":"2022-01-31T05:04:20.261681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function log_return_ahead computes R_t = log(P_{t+16} / P_{t+1})\ndef log_return_ahead(series, periods=1): \n    return -np.log(series).diff(periods=-periods).shift(-1)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:04:20.263835Z","iopub.execute_input":"2022-01-31T05:04:20.264253Z","iopub.status.idle":"2022-01-31T05:04:20.276640Z","shell.execute_reply.started":"2022-01-31T05:04:20.264208Z","shell.execute_reply":"2022-01-31T05:04:20.275816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = reduce_memory(pd.read_csv(INPUT/\"train.csv\"))\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:04:20.278356Z","iopub.execute_input":"2022-01-31T05:04:20.278576Z","iopub.status.idle":"2022-01-31T05:05:22.552181Z","shell.execute_reply.started":"2022-01-31T05:04:20.278549Z","shell.execute_reply":"2022-01-31T05:05:22.551390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_train_df = reduce_memory(pd.read_csv(INPUT/\"supplemental_train.csv\"))\nsub_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:05:22.553382Z","iopub.execute_input":"2022-01-31T05:05:22.554132Z","iopub.status.idle":"2022-01-31T05:05:29.124583Z","shell.execute_reply.started":"2022-01-31T05:05:22.554102Z","shell.execute_reply":"2022-01-31T05:05:29.123794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# continuous\nsub_train_df.timestamp.min()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:05:29.126011Z","iopub.execute_input":"2022-01-31T05:05:29.126625Z","iopub.status.idle":"2022-01-31T05:05:29.135987Z","shell.execute_reply.started":"2022-01-31T05:05:29.126594Z","shell.execute_reply":"2022-01-31T05:05:29.135110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.timestamp.max()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:05:29.137643Z","iopub.execute_input":"2022-01-31T05:05:29.137923Z","iopub.status.idle":"2022-01-31T05:05:29.163966Z","shell.execute_reply.started":"2022-01-31T05:05:29.137886Z","shell.execute_reply":"2022-01-31T05:05:29.163120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.append(sub_train_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:05:29.165096Z","iopub.execute_input":"2022-01-31T05:05:29.165286Z","iopub.status.idle":"2022-01-31T05:05:29.717455Z","shell.execute_reply.started":"2022-01-31T05:05:29.165264Z","shell.execute_reply":"2022-01-31T05:05:29.716735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assets = pd.read_csv(INPUT/\"asset_details.csv\")\nassets = assets.set_index(\"Asset_ID\")","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:05:29.718601Z","iopub.execute_input":"2022-01-31T05:05:29.718805Z","iopub.status.idle":"2022-01-31T05:05:29.731708Z","shell.execute_reply.started":"2022-01-31T05:05:29.718783Z","shell.execute_reply":"2022-01-31T05:05:29.730852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop some data\ntrain_df[\"datetime\"] = train_df[\"timestamp\"].map(lambda x: datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:05:29.734748Z","iopub.execute_input":"2022-01-31T05:05:29.735066Z","iopub.status.idle":"2022-01-31T05:06:06.894490Z","shell.execute_reply.started":"2022-01-31T05:05:29.735039Z","shell.execute_reply":"2022-01-31T05:06:06.893819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Price of assets\n$$P^a$$","metadata":{}},{"cell_type":"code","source":"prices = train_df.pivot(index=[\"timestamp\"], columns=[\"Asset_ID\"], values=[\"Close\"])\nprices = prices.droplevel(0, 1)\nasset_names = assets[\"Asset_Name\"].to_dict()\nprices = prices.rename(columns=asset_names)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:06.895766Z","iopub.execute_input":"2022-01-31T05:06:06.896403Z","iopub.status.idle":"2022-01-31T05:06:16.942656Z","shell.execute_reply.started":"2022-01-31T05:06:06.896365Z","shell.execute_reply":"2022-01-31T05:06:16.941757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices = prices.reindex(range(prices.index[0], int(prices.index[-1]+60),60), method='ffill')","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:16.944210Z","iopub.execute_input":"2022-01-31T05:06:16.944524Z","iopub.status.idle":"2022-01-31T05:06:17.303699Z","shell.execute_reply.started":"2022-01-31T05:06:16.944483Z","shell.execute_reply":"2022-01-31T05:06:17.302742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fillna automatically\n# prices = prices.fillna(method=\"ffill\")","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.305130Z","iopub.execute_input":"2022-01-31T05:06:17.305598Z","iopub.status.idle":"2022-01-31T05:06:17.309480Z","shell.execute_reply.started":"2022-01-31T05:06:17.305552Z","shell.execute_reply":"2022-01-31T05:06:17.308751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prices.index = prices.index.map(lambda x: datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.310711Z","iopub.execute_input":"2022-01-31T05:06:17.311126Z","iopub.status.idle":"2022-01-31T05:06:17.319854Z","shell.execute_reply.started":"2022-01-31T05:06:17.311092Z","shell.execute_reply":"2022-01-31T05:06:17.319330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prices.sort_index(inplace=True)\nprices.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.320856Z","iopub.execute_input":"2022-01-31T05:06:17.321455Z","iopub.status.idle":"2022-01-31T05:06:17.344702Z","shell.execute_reply.started":"2022-01-31T05:06:17.321423Z","shell.execute_reply":"2022-01-31T05:06:17.343931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Log Returns over 15 Minutes\n\n$$R^a(t) = log (P^a(t+16)\\ /\\ P^a(t+1))$$\n","metadata":{}},{"cell_type":"code","source":"log_returns_15min = log_return_ahead(prices, periods=15)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.346125Z","iopub.execute_input":"2022-01-31T05:06:17.346571Z","iopub.status.idle":"2022-01-31T05:06:17.535113Z","shell.execute_reply.started":"2022-01-31T05:06:17.346541Z","shell.execute_reply":"2022-01-31T05:06:17.534397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Weighted Average Market Returns\n\n$$M(t) = \\frac{\\sum_a w^a R^a(t)}{\\sum_a w^a}  $$","metadata":{}},{"cell_type":"code","source":"assets_df = pd.read_csv(INPUT/\"asset_details.csv\", index_col = \"Asset_ID\")\nassets_df.sort_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.536433Z","iopub.execute_input":"2022-01-31T05:06:17.536916Z","iopub.status.idle":"2022-01-31T05:06:17.546241Z","shell.execute_reply.started":"2022-01-31T05:06:17.536871Z","shell.execute_reply":"2022-01-31T05:06:17.545386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = assets_df.Weight.values\nweights","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.547581Z","iopub.execute_input":"2022-01-31T05:06:17.547820Z","iopub.status.idle":"2022-01-31T05:06:17.555268Z","shell.execute_reply.started":"2022-01-31T05:06:17.547798Z","shell.execute_reply":"2022-01-31T05:06:17.554534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weighted_avg_market_log_returns = log_returns_15min.mul(weights, axis='columns').mean(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.558016Z","iopub.execute_input":"2022-01-31T05:06:17.558539Z","iopub.status.idle":"2022-01-31T05:06:17.654753Z","shell.execute_reply.started":"2022-01-31T05:06:17.558499Z","shell.execute_reply":"2022-01-31T05:06:17.653822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# log_returns_15min.mul(weights, axis='columns')[-200:].plot()\n# weighted_avg_market_log_returns[-200:].plot(style=\"k8\", grid=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.656191Z","iopub.execute_input":"2022-01-31T05:06:17.656535Z","iopub.status.idle":"2022-01-31T05:06:17.660629Z","shell.execute_reply.started":"2022-01-31T05:06:17.656492Z","shell.execute_reply":"2022-01-31T05:06:17.659796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log_returns_15min[\"market\"] = weighted_avg_market_log_returns\nresidualized_market_returns, beta = ResidualizeMarket(log_returns_15min, \"market\", window=3750)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:17.661889Z","iopub.execute_input":"2022-01-31T05:06:17.662116Z","iopub.status.idle":"2022-01-31T05:06:19.983692Z","shell.execute_reply.started":"2022-01-31T05:06:17.662081Z","shell.execute_reply":"2022-01-31T05:06:19.983013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# residualized_market_returns[-200:].plot(grid=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:19.984794Z","iopub.execute_input":"2022-01-31T05:06:19.985448Z","iopub.status.idle":"2022-01-31T05:06:19.989190Z","shell.execute_reply.started":"2022-01-31T05:06:19.985406Z","shell.execute_reply":"2022-01-31T05:06:19.988422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compare computed with provided target","metadata":{}},{"cell_type":"code","source":"target = train_df.pivot(index=[\"timestamp\"], columns=[\"Asset_ID\"], values=[\"Target\"])\ntarget = target.droplevel(0, 1)\ntarget = target.rename(columns=asset_names)\ntarget = target.reindex(range(target.index[0], int(target.index[-1]+60),60), method='pad')\n# target.index = target.index.map(lambda x: datetime.fromtimestamp(x))\ntarget.sort_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:19.990570Z","iopub.execute_input":"2022-01-31T05:06:19.990777Z","iopub.status.idle":"2022-01-31T05:06:30.279092Z","shell.execute_reply.started":"2022-01-31T05:06:19.990752Z","shell.execute_reply":"2022-01-31T05:06:30.278319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residualized_market_returns[\"Bitcoin\"][-500:].plot(grid=True)\ntarget[\"Bitcoin\"][-500:].plot(style='r--', grid=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:30.280382Z","iopub.execute_input":"2022-01-31T05:06:30.280588Z","iopub.status.idle":"2022-01-31T05:06:30.658152Z","shell.execute_reply.started":"2022-01-31T05:06:30.280563Z","shell.execute_reply":"2022-01-31T05:06:30.657291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residualized_market_returns[\"Bitcoin Cash\"][-500:].plot()\ntarget[\"Bitcoin Cash\"][-500:].plot(style='r--',grid=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:30.659511Z","iopub.execute_input":"2022-01-31T05:06:30.660427Z","iopub.status.idle":"2022-01-31T05:06:30.936186Z","shell.execute_reply.started":"2022-01-31T05:06:30.660377Z","shell.execute_reply":"2022-01-31T05:06:30.935350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_diffs = np.abs(residualized_market_returns - target)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:30.937766Z","iopub.execute_input":"2022-01-31T05:06:30.938152Z","iopub.status.idle":"2022-01-31T05:06:31.064584Z","shell.execute_reply.started":"2022-01-31T05:06:30.938110Z","shell.execute_reply":"2022-01-31T05:06:31.063718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(target_diffs.values.reshape(-1), bins=1000)\nplt.xlim((-0.01,0.01))\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:31.066115Z","iopub.execute_input":"2022-01-31T05:06:31.066403Z","iopub.status.idle":"2022-01-31T05:06:33.682141Z","shell.execute_reply.started":"2022-01-31T05:06:31.066364Z","shell.execute_reply":"2022-01-31T05:06:33.681107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(target_diffs.iloc[-100000:].values.reshape(-1), bins=1000)\nplt.xlim((-0.01,0.01))\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:33.687147Z","iopub.execute_input":"2022-01-31T05:06:33.687395Z","iopub.status.idle":"2022-01-31T05:06:35.809391Z","shell.execute_reply.started":"2022-01-31T05:06:33.687366Z","shell.execute_reply":"2022-01-31T05:06:35.808579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in target.columns:\n    print(c)\n    if c in residualized_market_returns.columns:\n        print(residualized_market_returns[c].corr(target[c]))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:35.810448Z","iopub.execute_input":"2022-01-31T05:06:35.810670Z","iopub.status.idle":"2022-01-31T05:06:36.387212Z","shell.execute_reply.started":"2022-01-31T05:06:35.810644Z","shell.execute_reply":"2022-01-31T05:06:36.386282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residualized_market_returns[\"Maker\"][-500:].plot()\ntarget[\"Maker\"][-500:].plot(style='r--',grid=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:36.388327Z","iopub.execute_input":"2022-01-31T05:06:36.388636Z","iopub.status.idle":"2022-01-31T05:06:36.636363Z","shell.execute_reply.started":"2022-01-31T05:06:36.388600Z","shell.execute_reply":"2022-01-31T05:06:36.635501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Now we replicate the Target and re-construct the datasets for easy access.","metadata":{}},{"cell_type":"code","source":"features = train_df.pivot(index=[\"timestamp\"], columns=[\"Asset_ID\"], values=[\"Close\", \"Count\", \"Open\", \"High\", \"Low\", \"Volume\", \"VWAP\"])","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:09:05.506723Z","iopub.execute_input":"2022-01-31T05:09:05.507405Z","iopub.status.idle":"2022-01-31T05:09:05.590711Z","shell.execute_reply.started":"2022-01-31T05:09:05.507316Z","shell.execute_reply":"2022-01-31T05:09:05.589372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residualized_market_returns.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:09:05.655358Z","iopub.execute_input":"2022-01-31T05:09:05.655667Z","iopub.status.idle":"2022-01-31T05:09:05.679623Z","shell.execute_reply.started":"2022-01-31T05:09:05.655633Z","shell.execute_reply":"2022-01-31T05:09:05.678775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = features.rename(columns=asset_names)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:09:05.876777Z","iopub.execute_input":"2022-01-31T05:09:05.877056Z","iopub.status.idle":"2022-01-31T05:09:12.329987Z","shell.execute_reply.started":"2022-01-31T05:09:05.877028Z","shell.execute_reply":"2022-01-31T05:09:12.329201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = features.reindex(range(features.index[0], int(features.index[-1]+60),60), method='ffill')\n# features.index = features.index.map(lambda x: datetime.fromtimestamp(x))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:09:12.331601Z","iopub.execute_input":"2022-01-31T05:09:12.331873Z","iopub.status.idle":"2022-01-31T05:09:13.186206Z","shell.execute_reply.started":"2022-01-31T05:09:12.331844Z","shell.execute_reply":"2022-01-31T05:09:13.185372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_fill_price = features[\"Close\"].copy()\nnon_fill_price.columns = pd.MultiIndex.from_tuples(((\"CloseN\", a) for a in non_fill_price.columns))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:53.248671Z","iopub.execute_input":"2022-01-31T05:06:53.248936Z","iopub.status.idle":"2022-01-31T05:06:53.416189Z","shell.execute_reply.started":"2022-01-31T05:06:53.248906Z","shell.execute_reply":"2022-01-31T05:06:53.415474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = features.fillna(method=\"ffill\")","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:53.417536Z","iopub.execute_input":"2022-01-31T05:06:53.418240Z","iopub.status.idle":"2022-01-31T05:06:54.510314Z","shell.execute_reply.started":"2022-01-31T05:06:53.418195Z","shell.execute_reply":"2022-01-31T05:06:54.509522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"residualized_market_returns.columns = pd.MultiIndex.from_tuples(((\"Target\", a) for a in residualized_market_returns.columns))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:54.600508Z","iopub.execute_input":"2022-01-31T05:06:54.601013Z","iopub.status.idle":"2022-01-31T05:06:54.608056Z","shell.execute_reply.started":"2022-01-31T05:06:54.600965Z","shell.execute_reply":"2022-01-31T05:06:54.607281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([features, residualized_market_returns, non_fill_price], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:54.609193Z","iopub.execute_input":"2022-01-31T05:06:54.609530Z","iopub.status.idle":"2022-01-31T05:06:55.548932Z","shell.execute_reply.started":"2022-01-31T05:06:54.609489Z","shell.execute_reply":"2022-01-31T05:06:55.547948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:06:55.550267Z","iopub.execute_input":"2022-01-31T05:06:55.550554Z","iopub.status.idle":"2022-01-31T05:06:55.587513Z","shell.execute_reply.started":"2022-01-31T05:06:55.550520Z","shell.execute_reply":"2022-01-31T05:06:55.586598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[(\"Close\", \"Maker\")].corr(train_df[(\"CloseN\", \"Maker\")])","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:08:05.150453Z","iopub.execute_input":"2022-01-31T05:08:05.150999Z","iopub.status.idle":"2022-01-31T05:08:05.237753Z","shell.execute_reply.started":"2022-01-31T05:08:05.150936Z","shell.execute_reply":"2022-01-31T05:08:05.236397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df[(\"Close\", \"Maker\")].corr(train_df[(\"CloseN\", \"Maker\")])","metadata":{"execution":{"iopub.status.busy":"2022-01-31T04:58:47.141911Z","iopub.execute_input":"2022-01-31T04:58:47.142225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[(\"CloseN\", \"Maker\")]","metadata":{"execution":{"iopub.status.busy":"2022-01-31T05:08:39.724285Z","iopub.execute_input":"2022-01-31T05:08:39.724631Z","iopub.status.idle":"2022-01-31T05:08:39.740431Z","shell.execute_reply.started":"2022-01-31T05:08:39.724596Z","shell.execute_reply":"2022-01-31T05:08:39.739412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del features\n# del residualized_market_returns\n# del prices\n# del target","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:51:25.822929Z","iopub.execute_input":"2022-01-30T01:51:25.823208Z","iopub.status.idle":"2022-01-30T01:51:25.827116Z","shell.execute_reply.started":"2022-01-30T01:51:25.823182Z","shell.execute_reply":"2022-01-30T01:51:25.826399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:51:26.121687Z","iopub.execute_input":"2022-01-30T01:51:26.122715Z","iopub.status.idle":"2022-01-30T01:51:26.275305Z","shell.execute_reply.started":"2022-01-30T01:51:26.122667Z","shell.execute_reply":"2022-01-30T01:51:26.2742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.index = train_df.index.values.astype(int) // int(1e9)\n# train_df.index.name = \"timestamp\"","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:29:53.753435Z","iopub.execute_input":"2022-01-30T01:29:53.753784Z","iopub.status.idle":"2022-01-30T01:29:53.761768Z","shell.execute_reply.started":"2022-01-30T01:29:53.753738Z","shell.execute_reply":"2022-01-30T01:29:53.761028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.stack(level=1, dropna=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:52:08.670737Z","iopub.execute_input":"2022-01-30T01:52:08.671126Z","iopub.status.idle":"2022-01-30T01:52:16.196383Z","shell.execute_reply.started":"2022-01-30T01:52:08.671082Z","shell.execute_reply":"2022-01-30T01:52:16.195683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.reset_index(drop=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T02:04:19.990549Z","iopub.execute_input":"2022-01-30T02:04:19.990818Z","iopub.status.idle":"2022-01-30T02:04:21.211886Z","shell.execute_reply.started":"2022-01-30T02:04:19.990791Z","shell.execute_reply":"2022-01-30T02:04:21.210996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop data before \"2020-08-04\" for Maker\ntrain_df.drop(train_df.loc[(train_df[\"timestamp\"] <= 1596513600) & (train_df[\"Asset_ID\"] == \"Maker\")].index, inplace=True)\n# drop data before \"xxx\" for Monero\ntrain_df.drop(train_df.loc[(train_df[\"timestamp\"] <= 1541394000) & (train_df[\"Asset_ID\"] == \"Monero\")].index, inplace=True)\n# drop data before \"2018-07-14\" for Stellar\ntrain_df.drop(train_df.loc[(train_df[\"timestamp\"] <= 1531540800) & (train_df[\"Asset_ID\"] == \"Stellar\")].index, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:53:40.30459Z","iopub.execute_input":"2022-01-30T01:53:40.304921Z","iopub.status.idle":"2022-01-30T01:54:11.756291Z","shell.execute_reply.started":"2022-01-30T01:53:40.30489Z","shell.execute_reply":"2022-01-30T01:54:11.755172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.reset_index(drop=True)\ntrain_df.to_feather(\"./train.feather\")","metadata":{"execution":{"iopub.status.busy":"2022-01-30T02:04:10.969059Z","iopub.execute_input":"2022-01-30T02:04:10.969329Z","iopub.status.idle":"2022-01-30T02:04:11.106021Z","shell.execute_reply.started":"2022-01-30T02:04:10.969301Z","shell.execute_reply":"2022-01-30T02:04:11.104583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-01-30T02:04:22.55673Z","iopub.execute_input":"2022-01-30T02:04:22.557021Z","iopub.status.idle":"2022-01-30T02:04:22.580136Z","shell.execute_reply.started":"2022-01-30T02:04:22.556985Z","shell.execute_reply":"2022-01-30T02:04:22.579088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maker = train_df[train_df.Asset_ID == \"Maker\"].set_index(\"timestamp\")","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:56:33.203237Z","iopub.execute_input":"2022-01-30T01:56:33.203524Z","iopub.status.idle":"2022-01-30T01:56:37.368098Z","shell.execute_reply.started":"2022-01-30T01:56:33.203496Z","shell.execute_reply":"2022-01-30T01:56:37.36717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maker.dropna()","metadata":{"execution":{"iopub.status.busy":"2022-01-30T01:56:49.380679Z","iopub.execute_input":"2022-01-30T01:56:49.381676Z","iopub.status.idle":"2022-01-30T01:56:49.538099Z","shell.execute_reply.started":"2022-01-30T01:56:49.381632Z","shell.execute_reply":"2022-01-30T01:56:49.536907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion\n\nSublement file is used to construct some rolling features.","metadata":{}}]}