{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":30894,"databundleVersionId":3147400,"sourceType":"competition"},{"sourceId":7936997,"sourceType":"datasetVersion","datasetId":4665908}],"dockerImageVersionId":30145,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport gc\n\ndirectory = '../input/g-research-crypto-forecasting'\nfile_path = '/kaggle/input/spermik2/train_spermik.csv'\ndtypes = {\n    'timestamp': np.int64,\n    'Asset_ID': np.int8,\n#     'Count': np.int32,\n#     'Open': np.float64,\n#     'High': np.float64,\n#     'Low': np.float64,\n    'Close': np.float64,\n#     'Volume': np.float64,\n#     'VWAP': np.float64,\n    'Target': np.float64,\n}\ndata = pd.read_csv(file_path, dtype=dtypes, usecols=list(dtypes.keys()))\ndata['Time'] = pd.to_datetime(data['timestamp'], unit='s')\n\nfile_path = os.path.join(directory, 'asset_details.csv')\ndetails = pd.read_csv(file_path)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T10:14:50.303224Z","iopub.execute_input":"2024-03-25T10:14:50.303595Z","iopub.status.idle":"2024-03-25T10:14:50.401186Z","shell.execute_reply.started":"2024-03-25T10:14:50.303553Z","shell.execute_reply":"2024-03-25T10:14:50.399924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = list(details.Asset_ID)\nasset_names = list(details.Asset_Name)\n\n# times = data['timestamp'].agg(['min', 'max']).to_dict()\n# all_timestamps = np.arange(times['min'], times['max'] + 60, 60)\nall_timestamps = np.sort(data['timestamp'].unique())\ntargets = pd.DataFrame(index=all_timestamps)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-25T08:30:26.560199Z","iopub.execute_input":"2024-03-25T08:30:26.561371Z","iopub.status.idle":"2024-03-25T08:30:27.299733Z","shell.execute_reply.started":"2024-03-25T08:30:26.561299Z","shell.execute_reply":"2024-03-25T08:30:27.298368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, id in enumerate(ids):\n    asset = data[data.Asset_ID == id].set_index(keys='timestamp')\n    price = pd.Series(index=all_timestamps, data=asset[price_column])\n#     targets[asset_names[i]] = np.log(\n#         price.shift(periods=-16) /\n#         price.shift(periods=-1)\n#     )\n    targets[asset_names[i]] = (\n        price.shift(periods=-16) /\n        price.shift(periods=-1)\n    ) - 1\n\n","metadata":{"execution":{"iopub.status.busy":"2024-03-25T08:30:27.30153Z","iopub.execute_input":"2024-03-25T08:30:27.301825Z","iopub.status.idle":"2024-03-25T08:30:43.203068Z","shell.execute_reply.started":"2024-03-25T08:30:27.301789Z","shell.execute_reply":"2024-03-25T08:30:43.202062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = np.array(list(details.Weight))\ntargets['m'] = np.average(targets.fillna(0), axis=1, weights=weights)","metadata":{"execution":{"iopub.status.busy":"2024-03-25T08:30:43.204863Z","iopub.execute_input":"2024-03-25T08:30:43.207307Z","iopub.status.idle":"2024-03-25T08:30:43.846768Z","shell.execute_reply.started":"2024-03-25T08:30:43.207157Z","shell.execute_reply":"2024-03-25T08:30:43.845603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = targets['m']\n\nnum = targets.multiply(m.values, axis=0).rolling(3750).mean().values\ndenom = m.multiply(m.values, axis=0).rolling(3750).mean().values\nbeta = np.nan_to_num(num.T / denom, nan=0., posinf=0., neginf=0.)\n\ntargets = targets - (beta * m.values).T","metadata":{"execution":{"iopub.status.busy":"2024-03-25T08:30:43.848389Z","iopub.execute_input":"2024-03-25T08:30:43.848697Z","iopub.status.idle":"2024-03-25T08:30:45.632529Z","shell.execute_reply.started":"2024-03-25T08:30:43.84866Z","shell.execute_reply":"2024-03-25T08:30:45.631641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"diffs = []\n\nfor i, id in enumerate(ids):\n    print(asset_names[i])\n    # type: pd.DataFrame\n    asset = data[data.Asset_ID == id].set_index(keys='timestamp')\n    print(f'asset size {asset.shape[0]}')\n    recreated = pd.Series(index=asset.index, data=targets[asset_names[i]])\n    diff = np.abs(asset['Target'] - recreated)\n    diffs.append(diff[~pd.isna(diff)].values)\n    print(f'Average absolute error {diff.mean():8.6f}')\n    print(f'Max absolute error {diff.max():8.6f}')\n    print(f'Standard deviation {diff.std():8.6f}')\n    print(f'Target na {pd.isna(asset.Target).sum()}')\n    print(f'Target_calculated na {pd.isna(recreated).sum()}')\n    print()\n\ndiffs = np.concatenate(diffs, axis=0)\nprint('For all assets')\nprint(f'Average absolute error {diffs.mean():8.6f}')\nprint(f'Max absolute error {diffs.max():8.6f}')\nprint(f'Standard deviation {diffs.std():8.6f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-03-25T08:30:45.633991Z","iopub.execute_input":"2024-03-25T08:30:45.634273Z","iopub.status.idle":"2024-03-25T08:30:56.035415Z","shell.execute_reply.started":"2024-03-25T08:30:45.634239Z","shell.execute_reply":"2024-03-25T08:30:56.033984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"beta_ = num.T / denom\n\nfor i, id in enumerate(ids):\n    print(asset_names[i])\n    print(f'Infiinte beta rows {np.isinf(beta_[i]).sum()}')\n    nan_sum = np.isnan(beta_[i]).sum()\n    print(f'NAN beta rows {nan_sum} ({100 * nan_sum / beta_.shape[1]:5.2f}%)')\n\n    eps = 1e-6\n    zero_sum = ((beta[i] > -eps) & (beta[i] < eps)).sum()\n    print(f'Zero beta rows {zero_sum} ({100 * zero_sum / beta.shape[1]:5.2f}%)')\n    print()","metadata":{"execution":{"iopub.status.busy":"2024-03-25T08:30:56.037741Z","iopub.execute_input":"2024-03-25T08:30:56.038117Z","iopub.status.idle":"2024-03-25T08:30:56.334492Z","shell.execute_reply.started":"2024-03-25T08:30:56.03807Z","shell.execute_reply":"2024-03-25T08:30:56.333312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_target(data: pd.DataFrame, details: pd.DataFrame, price_column: str):\n    ids = list(details.Asset_ID)\n    asset_names = list(details.Asset_Name)\n    weights = np.array(list(details.Weight))\n\n    all_timestamps = np.sort(data['timestamp'].unique())\n    targets = pd.DataFrame(index=all_timestamps)\n\n    for i, id in enumerate(ids):\n        asset = data[data.Asset_ID == id].set_index(keys='timestamp')\n        price = pd.Series(index=all_timestamps, data=asset[price_column])\n        targets[asset_names[i]] = (\n            price.shift(periods=-16) /\n            price.shift(periods=-1)\n        ) - 1\n    \n    targets['m'] = np.average(targets.fillna(0), axis=1, weights=weights)\n    \n    m = targets['m']\n\n    num = targets.multiply(m.values, axis=0).rolling(3750).mean().values\n    denom = m.multiply(m.values, axis=0).rolling(3750).mean().values\n    beta = np.nan_to_num(num.T / denom, nan=0., posinf=0., neginf=0.)\n\n    targets = targets - (beta * m.values).T\n    targets.drop('m', axis=1, inplace=True)\n    \n    return targets","metadata":{"execution":{"iopub.status.busy":"2024-03-25T08:30:56.33596Z","iopub.execute_input":"2024-03-25T08:30:56.336279Z","iopub.status.idle":"2024-03-25T08:30:56.351066Z","shell.execute_reply.started":"2024-03-25T08:30:56.336242Z","shell.execute_reply":"2024-03-25T08:30:56.34926Z"},"trusted":true},"execution_count":null,"outputs":[]}]}