{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### In this notebook I outline the Target Engineering method i found useful for training more accurate models.","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nimport time\nimport datetime\n# visualize\nimport matplotlib.pyplot as plt\nfrom matplotlib import pyplot\nimport warnings\nwarnings.simplefilter('ignore')\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-22T10:53:20.803318Z","iopub.execute_input":"2022-02-22T10:53:20.803660Z","iopub.status.idle":"2022-02-22T10:53:20.831235Z","shell.execute_reply.started":"2022-02-22T10:53:20.803564Z","shell.execute_reply":"2022-02-22T10:53:20.830211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\ntrain2 = pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv')\ntrain = pd.concat([train,train2])\ntrain = train.drop(columns=['VWAP','Count','Open','Low','High'])\ntrain = train.sort_values(by=['Asset_ID','timestamp'])","metadata":{"execution":{"iopub.status.busy":"2022-02-22T10:53:20.833280Z","iopub.execute_input":"2022-02-22T10:53:20.833624Z","iopub.status.idle":"2022-02-22T10:54:46.274808Z","shell.execute_reply.started":"2022-02-22T10:53:20.833578Z","shell.execute_reply":"2022-02-22T10:54:46.273850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\nids = list(asset_details.Asset_ID)\nweights = np.array(list(asset_details.Weight))\nasset_details['Normalized_Weight'] = asset_details['Weight'] / asset_details['Weight'].sum()\n","metadata":{"execution":{"iopub.status.busy":"2022-02-22T10:54:55.031628Z","iopub.execute_input":"2022-02-22T10:54:55.031944Z","iopub.status.idle":"2022-02-22T10:54:55.046241Z","shell.execute_reply.started":"2022-02-22T10:54:55.031907Z","shell.execute_reply":"2022-02-22T10:54:55.045479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mintime = train['timestamp'].min()\nmaxtime = train['timestamp'].max()\nall_timestamps1 = np.array([i for i in range(mintime, maxtime,60)])\nall_timestamps2 = sorted(train['timestamp'][train['timestamp']>=mintime].unique())\ntargets1 = pd.DataFrame(index=all_timestamps1)\ntargets2 = pd.DataFrame(index=all_timestamps2)\nbetas1 = pd.DataFrame(index=all_timestamps1)","metadata":{"execution":{"iopub.status.busy":"2022-02-22T10:54:56.176756Z","iopub.execute_input":"2022-02-22T10:54:56.177277Z","iopub.status.idle":"2022-02-22T10:55:00.681469Z","shell.execute_reply.started":"2022-02-22T10:54:56.177224Z","shell.execute_reply":"2022-02-22T10:55:00.680369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets1 = pd.DataFrame(index=all_timestamps1)\ntargets2 = pd.DataFrame(index=all_timestamps2)\nprice_column = 'Close'\nfor i in train['Asset_ID'].unique():\n    print(i)\n    asset = train[train.Asset_ID == i].set_index(keys='timestamp')\n    asset = asset[~asset.index.duplicated()]\n    price2 = pd.Series(index=all_timestamps2, data=asset[price_column])\n    price1 = pd.Series(index=all_timestamps1, data=asset[price_column])\n    price1 = price1.astype(float).fillna(method='ffill')\n    weight_i = asset_details[asset_details['Asset_ID']==i]['Normalized_Weight'].iloc[0]\n    targets1[f'{i}'] = (\n        price1.shift(periods=-16) /\n        price1.shift(periods=-1)\n    ) - 1\n    targets1[f'w{i}'] = targets1[f'{i}'] * weight_i\n\n    targets2[f'{i}'] = (\n        price2.shift(periods=-16) /\n        price2.shift(periods=-1)\n    ) - 1\n    targets2[f'w{i}'] = targets2[f'{i}'] * weight_i\n    \ntargets1['m1'] = targets1[[f'w{i}' for i in train['Asset_ID'].unique()]].fillna(0).sum(axis=1)\ntargets2['m1'] = 0\nm1 = targets1['m1']\nm2 = targets2['m1']\n\ntargets1  = targets1[[f'{i}' for i in train['Asset_ID'].unique()]]\nnum1 = targets1.multiply(m1.values, axis=0).rolling(3750).mean().values\ndenom1 = m1.multiply(m1.values, axis=0).rolling(3750).mean().values\nbeta1 = np.nan_to_num(num1.T / denom1, nan=0., posinf=0., neginf=0.)\n\ntargets1 = targets1 - (beta1 * m1.values).T\n\ntargets2  = targets2[[f'{i}' for i in train['Asset_ID'].unique()]]\nnum2 = targets2.multiply(m2.values, axis=0).rolling(3750).mean().values\ndenom2 = m2.multiply(m2.values, axis=0).rolling(3750).mean().values\nbeta2 = np.nan_to_num(num2.T / denom2, nan=0., posinf=0., neginf=0.)\n\ntargets2 = targets2 - (beta2 * m2.values).T\n\nfor j,i in enumerate(train['Asset_ID'].unique()):\n    betas1[f'{i}'] = beta1[j,:]","metadata":{"execution":{"iopub.status.busy":"2022-02-22T10:55:00.683553Z","iopub.execute_input":"2022-02-22T10:55:00.683906Z","iopub.status.idle":"2022-02-22T10:55:39.686028Z","shell.execute_reply.started":"2022-02-22T10:55:00.683863Z","shell.execute_reply":"2022-02-22T10:55:39.685191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full = pd.DataFrame()\nfor i in train['Asset_ID'].unique():\n    \n    asset = train[train.Asset_ID == int(i)].set_index(keys='timestamp')\n    if len(asset) > 400000:\n        print(f'asset size {asset.shape[0]}')\n        recreated1 = pd.Series(index=asset.index, data=targets1[f'{i}'])\n        recreated2 = pd.Series(index=asset.index, data=targets2[f'{i}'])\n        recreatedbeta1 = pd.Series(index=asset.index, data=betas1[f'{i}'])\n        recreatedm1 = pd.Series(index=asset.index, data=m1)\n        #isna = pd.Series(index=asset.index, data=targets[f'isna'])\n        asset['m'] =  recreatedm1\n        asset['Beta'] = recreatedbeta1\n        asset['TargetBeta'] = recreated1\n        asset['TargetZero'] = recreated2\n\n        full = pd.concat([full, asset])\n","metadata":{"execution":{"iopub.status.busy":"2022-02-22T10:55:39.687682Z","iopub.execute_input":"2022-02-22T10:55:39.688050Z","iopub.status.idle":"2022-02-22T10:55:54.818384Z","shell.execute_reply.started":"2022-02-22T10:55:39.687983Z","shell.execute_reply":"2022-02-22T10:55:54.817418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We produce two targets\n- TargetBeta - we forward fill any missing close values, which allows up to create a beta for all assets at all times. We avoid random time periods where beta == 0 due to missing values.\n- TargetZero - we set beta & timestamp mean (m) component == 0.","metadata":{}},{"cell_type":"code","source":"full.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-22T10:55:54.820199Z","iopub.execute_input":"2022-02-22T10:55:54.820437Z","iopub.status.idle":"2022-02-22T10:55:54.838694Z","shell.execute_reply.started":"2022-02-22T10:55:54.820409Z","shell.execute_reply":"2022-02-22T10:55:54.837756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Difference between true target and reconstructed TargetBeta for Asset 4 (DOGE)')\nprint('In later time periods there is less missing data and so the difference between the TargetBeta and the true Target is close to 0.')\nabs(full['TargetBeta'][full['Asset_ID']==4]-full['Target'][full['Asset_ID']==4]).plot()","metadata":{"execution":{"iopub.status.busy":"2022-02-22T10:55:54.840279Z","iopub.execute_input":"2022-02-22T10:55:54.840940Z","iopub.status.idle":"2022-02-22T10:55:55.583714Z","shell.execute_reply.started":"2022-02-22T10:55:54.840886Z","shell.execute_reply":"2022-02-22T10:55:55.582748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Difference between true target and reconstructed TargetZero for Asset 4 (DOGE)')\nprint('In earlier time periods there is more missing data and so the difference between the TargetZero and the true Target is close to 0.')\nabs(full['TargetZero'][full['Asset_ID']==4]-full['Target'][full['Asset_ID']==4]).plot()","metadata":{"execution":{"iopub.status.busy":"2022-02-22T10:55:55.585262Z","iopub.execute_input":"2022-02-22T10:55:55.585617Z","iopub.status.idle":"2022-02-22T10:55:56.119903Z","shell.execute_reply.started":"2022-02-22T10:55:55.585569Z","shell.execute_reply":"2022-02-22T10:55:56.119072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}