{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Target reconstruction - quick summary of SOTA\n\nThis notebook summarises community contributions, many found in and around [alexfir's notebook](https://www.kaggle.com/alexfir/recreating-target) - upvote there if these insights help you in this competition!\n\n## Reconstruction statistics for the train set:\n\n* Mean abs error: 0.00000000000000082\n* Max abs error:  0.00000000000008532\n* Std abs error:  0.00000000000000276\n","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\ndata_dir = '../input/g-research-crypto-forecasting/'\ndtypes = {\n    'timestamp': np.int64,\n    'Asset_ID': np.int8,\n    'Close': np.float64,\n    'Target': np.float64,\n}\ncrypto_df = pd.read_csv(data_dir + 'train.csv', dtype=dtypes, usecols=list(dtypes.keys()))\nasset_details = pd.read_csv(data_dir + 'asset_details.csv')\ncrypto_df = crypto_df.merge(asset_details, on='Asset_ID')\ncrypto_df['timestamp'] = pd.to_datetime(crypto_df['timestamp'], unit='s')","metadata":{"execution":{"iopub.status.busy":"2022-01-15T12:12:04.757217Z","iopub.execute_input":"2022-01-15T12:12:04.757617Z","iopub.status.idle":"2022-01-15T12:12:50.518405Z","shell.execute_reply.started":"2022-01-15T12:12:04.757577Z","shell.execute_reply":"2022-01-15T12:12:50.514952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# essentially verbatim from https://www.kaggle.com/alexfir/recreating-target/\ndef reconstruct_targets(data: pd.DataFrame, details: pd.DataFrame, price_column: str, use_log=False):\n    ids = list(details.Asset_ID)\n    asset_names = list(details.Asset_Name)\n    weights = np.array(list(details.Weight))\n    \n    timestamp_series = pd.Series([pd.Timestamp(\"2018-01-01 00:00\")]).append(crypto_df['timestamp'], ignore_index=True)\n    all_timestamps = np.sort(timestamp_series.unique())\n\n    targets = pd.DataFrame(index=all_timestamps)\n\n    for i, id in enumerate(ids):\n        asset = data[data.Asset_ID == id].set_index(keys='timestamp')\n        price = pd.Series(index=all_timestamps, data=asset[price_column])\n        if use_log:\n            targets[asset_names[i]] = np.log(price.shift(periods=-16)) - np.log(price.shift(periods=-1))\n        else:\n            targets[asset_names[i]] = (\n                price.shift(periods=-16) /\n                price.shift(periods=-1)\n            ) - 1\n            \n    targets['m'] = np.average(targets.fillna(0), axis=1, weights=weights)\n    m = targets['m']\n    \n    num = targets.multiply(m.values, axis=0).rolling(3750).mean().values\n    denom = m.multiply(m.values, axis=0).rolling(3750).mean().values\n    beta = np.nan_to_num(num.T / denom, nan=0., posinf=0., neginf=0.)\n    \n    recon_targets = targets - (beta * m.values).T\n    recon_targets.drop('m', axis=1, inplace=True)\n    \n    return recon_targets","metadata":{"execution":{"iopub.status.busy":"2022-01-15T12:12:50.521143Z","iopub.execute_input":"2022-01-15T12:12:50.522001Z","iopub.status.idle":"2022-01-15T12:12:50.539504Z","shell.execute_reply.started":"2022-01-15T12:12:50.521957Z","shell.execute_reply":"2022-01-15T12:12:50.53817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge_recon_with_train():\n    recon_targets = reconstruct_targets(data=crypto_df, details=asset_details, price_column='Close')\n    recon_targets = pd.melt(recon_targets.reset_index(), id_vars='index')\n    recon_targets = recon_targets.rename(columns={'index':'timestamp', 'variable':'Asset_Name', 'value':'recon_Target'})\n    return crypto_df.merge(recon_targets, on=['Asset_Name', 'timestamp'])","metadata":{"execution":{"iopub.status.busy":"2022-01-15T12:12:50.541245Z","iopub.execute_input":"2022-01-15T12:12:50.541571Z","iopub.status.idle":"2022-01-15T12:12:50.559637Z","shell.execute_reply.started":"2022-01-15T12:12:50.541537Z","shell.execute_reply":"2022-01-15T12:12:50.558251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf = merge_recon_with_train()\n\nprint(\"Is it true that all NaN's match between Target and recon_Target?\")\nprint(f\"{all(df.Target.isna() == df.recon_Target.isna())}\\n\")\n\nprint(\"Is it true that none of the Targets have been dropped?\")\nprint(f\"{len(crypto_df.Target)==len(df.Target)}\\n\")\n\nprint(\"Statistics for absolute error between Target and reconstructed Target for the entirety of train data:\")\ndf['abserror'] = abs(df['Target'] - df['recon_Target'])\nprint(df.abserror.describe())","metadata":{"execution":{"iopub.status.busy":"2022-01-15T12:13:18.866468Z","iopub.execute_input":"2022-01-15T12:13:18.867451Z","iopub.status.idle":"2022-01-15T12:13:58.31862Z","shell.execute_reply.started":"2022-01-15T12:13:18.867394Z","shell.execute_reply":"2022-01-15T12:13:58.317371Z"},"trusted":true},"execution_count":null,"outputs":[]}]}