{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":30894,"databundleVersionId":3147400,"sourceType":"competition"},{"sourceId":12307832,"sourceType":"datasetVersion","datasetId":7757728}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport time\nfrom datetime import datetime\n\nasset_details = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-28T04:41:42.067818Z","iopub.execute_input":"2025-06-28T04:41:42.068044Z","iopub.status.idle":"2025-06-28T04:41:42.360982Z","shell.execute_reply.started":"2025-06-28T04:41:42.068027Z","shell.execute_reply":"2025-06-28T04:41:42.360196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"asset_details","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T04:42:28.525915Z","iopub.execute_input":"2025-06-28T04:42:28.526894Z","iopub.status.idle":"2025-06-28T04:42:28.533933Z","shell.execute_reply.started":"2025-06-28T04:42:28.526859Z","shell.execute_reply":"2025-06-28T04:42:28.533307Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# dict_ticker = {'Bitcoin Cash':'BCH',\n# 'Binance Coin':'BNB',\n# 'Bitcoin':'BTC',\n# 'EOS.IO':'EOS',\n# 'Ethereum Classic':'ETC',\n# 'Ethereum':'ETH',\n# 'Litecoin':'LTC',\n# 'Monero':'XMR',\n# 'TRON':'TRX',\n# 'Stellar':'XLM',\n# 'Cardano':'ADA',\n# 'IOTA':'IOTA',\n# 'Maker':'MKR',\n# 'Dogecoin':'DOGE'}\n\n# chunks = []\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     if dirname == '/kaggle/input/g-research-binance-1m-data-btc':\n#         for filename in filenames:\n#             ticker = filename.split('-')[0]\n#             full_name = [i for i in dict_ticker if dict_ticker[i]==ticker][0]\n#             Asset_ID = asset_details[asset_details.Asset_Name == full_name].Asset_ID.values[0]\n            \n#             data = pd.read_parquet(dirname+'/'+filename, engine='pyarrow')\n#             data = data.reset_index()\n#             data = data.rename(columns={'open_time':'timestamp','open':'Open','high':'High','low':'Low','close':'Close','quote_asset_volume':'Volume','number_of_trades':'Count'})\n#             data.timestamp = (pd.to_datetime(data.timestamp).astype(int)/ 10**9).astype(int)\n\n#             #definition not okay so far\n#             data['VWAP'] = (data.Low * data.taker_buy_base_asset_volume + data.High * data.taker_buy_quote_asset_volume)/(data.taker_buy_base_asset_volume + data.taker_buy_quote_asset_volume)\n            \n#             data = data.drop('volume',axis=1)\n            \n#             data['Asset_ID'] = Asset_ID\n            \n#             price_column = 'Close'\n\n#             data['Time'] = pd.to_datetime(data['timestamp'], unit='s')\n#             data.sort_values(by='Time', inplace=True)\n#             data.set_index(keys='Time', inplace=True)\n#             data['p1'] = data[price_column].shift(freq='-1T')\n#             data['p16'] = data[price_column].shift(freq='-16T')\n#             data['r'] = np.log(data.p16/data.p1)\n#             data.drop(['p1', 'p16'], axis=1, inplace=True)\n#             data.reset_index(inplace=True)\n\n#             chunks.append(data)\n\n# data = pd.concat(chunks)\n# data.sort_values(by='timestamp', inplace=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dict_ticker = {'Bitcoin Cash':'BCH',\n'Binance Coin':'BNB',\n'Bitcoin':'BTC',\n'EOS.IO':'EOS',\n'Ethereum Classic':'ETC',\n'Ethereum':'ETH',\n'Litecoin':'LTC',\n'Monero':'XMR',\n'TRON':'TRX',\n'Stellar':'XLM',\n'Cardano':'ADA',\n'IOTA':'IOTA',\n'Maker':'MKR',\n'Dogecoin':'DOGE'}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T04:49:02.515669Z","iopub.execute_input":"2025-06-28T04:49:02.515885Z","iopub.status.idle":"2025-06-28T04:49:02.519820Z","shell.execute_reply.started":"2025-06-28T04:49:02.515870Z","shell.execute_reply":"2025-06-28T04:49:02.519216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    if dirname == '/kaggle/input/g-research-binance-1m-data-btc':\n        for filename in filenames:\n            ticker = filename.split('-')[0]\n            full_name = [i for i in dict_ticker if dict_ticker[i]==ticker][0]\n            Asset_ID = asset_details[asset_details.Asset_Name == full_name].Asset_ID.values[0]\n            \n            data = pd.read_parquet(dirname+'/'+filename, engine='pyarrow')\n            data = data.reset_index()\n            data = data.rename(columns={'open_time':'timestamp','open':'Open','high':'High','low':'Low','close':'Close','quote_asset_volume':'Volume','number_of_trades':'Count'})\n            data.timestamp = (pd.to_datetime(data.timestamp).astype(int)/ 10**9).astype(int)\n\n            #definition not okay so far\n            data['VWAP'] = (data.Low * data.taker_buy_base_asset_volume + data.High * data.taker_buy_quote_asset_volume)/(data.taker_buy_base_asset_volume + data.taker_buy_quote_asset_volume)\n            \n            data = data.drop('volume',axis=1)\n            \n            data['Asset_ID'] = Asset_ID\n            \n            price_column = 'Close'\n\n            data['Time'] = pd.to_datetime(data['timestamp'], unit='s')\n            data.sort_values(by='Time', inplace=True)\n            data.set_index(keys='Time', inplace=True)\n            data['p1'] = data[price_column].shift(freq='-1T')\n            data['p16'] = data[price_column].shift(freq='-16T')\n            data['r'] = np.log(data.p16/data.p1)\n            data.drop(['p1', 'p16'], axis=1, inplace=True)\n            data.reset_index(inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T04:49:05.247320Z","iopub.execute_input":"2025-06-28T04:49:05.247790Z","iopub.status.idle":"2025-06-28T04:49:09.690100Z","shell.execute_reply.started":"2025-06-28T04:49:05.247760Z","shell.execute_reply":"2025-06-28T04:49:09.689486Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.sort_values(by='timestamp', inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T04:51:05.924489Z","iopub.execute_input":"2025-06-28T04:51:05.924704Z","iopub.status.idle":"2025-06-28T04:51:05.993118Z","shell.execute_reply.started":"2025-06-28T04:51:05.924689Z","shell.execute_reply":"2025-06-28T04:51:05.992527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T04:51:08.541070Z","iopub.execute_input":"2025-06-28T04:51:08.541333Z","iopub.status.idle":"2025-06-28T04:51:08.552615Z","shell.execute_reply.started":"2025-06-28T04:51:08.541319Z","shell.execute_reply":"2025-06-28T04:51:08.551902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data['w'] = data['Asset_ID'].map(asset_details.set_index(keys='Asset_ID')['Weight'])\nweight_sum = asset_details.Weight.sum()\n\ndata['weighted_asset_r'] = data.w * data.r\ntime_group = data.groupby('Time')\n\nm = time_group['weighted_asset_r'].sum() / time_group['w'].sum()\n\ndata.set_index(keys=['Time'], inplace=True)\ndata['m'] = m\ndata.reset_index(inplace=True)\n\ndata['m2'] = data.m ** 2\ndata['mr'] = data.r * data.m\n\nids = list(asset_details.Asset_ID)\n\nchunks = []\nfor id in ids:\n    # type: pd.DataFrame\n    asset = data[data.Asset_ID == id].copy()\n    asset.sort_values(by='Time', inplace=True)\n    asset.set_index(keys='Time', inplace=True)\n    asset['mr_rolling'] = asset['mr'].rolling(window='3750T', min_periods=3750).mean()\n    asset['m2_rolling'] = asset['m2'].rolling(window='3750T', min_periods=3750).mean()\n    asset.reset_index(inplace=True)\n    chunks.append(asset)\n    debug = 1\n\ndata = pd.concat(chunks)\ndata.sort_values(by='Time', inplace=True)\ndata['beta'] = data['mr_rolling'] / data['m2_rolling']\n\ndata['Target'] = data['r'] - data['beta'] * data['m']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T04:59:18.313022Z","iopub.execute_input":"2025-06-28T04:59:18.313274Z","iopub.status.idle":"2025-06-28T04:59:20.144453Z","shell.execute_reply.started":"2025-06-28T04:59:18.313260Z","shell.execute_reply":"2025-06-28T04:59:20.143796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T05:01:54.936343Z","iopub.execute_input":"2025-06-28T05:01:54.936629Z","iopub.status.idle":"2025-06-28T05:01:54.958428Z","shell.execute_reply.started":"2025-06-28T05:01:54.936610Z","shell.execute_reply":"2025-06-28T05:01:54.957621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T04:59:48.458068Z","iopub.execute_input":"2025-06-28T04:59:48.458324Z","iopub.status.idle":"2025-06-28T04:59:48.462789Z","shell.execute_reply.started":"2025-06-28T04:59:48.458310Z","shell.execute_reply":"2025-06-28T04:59:48.461835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = data.drop(['taker_buy_base_asset_volume','taker_buy_quote_asset_volume','r','w','weighted_asset_r','m','m2','mr','mr_rolling','m2_rolling','beta','Time'],axis=1)\n\ncol_ordered = ['timestamp','Asset_ID','Count','Open','High','Low','Close','Volume','VWAP','Target']\ndata = data[col_ordered]\n\ndtype0 = {'Asset_ID': 'int8', 'Count': 'int32', 'Count': 'int32',\n       'Open': 'float32', 'High': 'float32', 'Low': 'float32', 'Close': 'float32',\n       'Volume': 'float32', 'VWAP': 'float32'}\n\ndata = data.astype(dtype0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T05:02:42.641629Z","iopub.execute_input":"2025-06-28T05:02:42.641846Z","iopub.status.idle":"2025-06-28T05:02:42.815543Z","shell.execute_reply.started":"2025-06-28T05:02:42.641831Z","shell.execute_reply":"2025-06-28T05:02:42.814443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T05:03:04.624253Z","iopub.execute_input":"2025-06-28T05:03:04.624470Z","iopub.status.idle":"2025-06-28T05:03:04.634649Z","shell.execute_reply.started":"2025-06-28T05:03:04.624456Z","shell.execute_reply":"2025-06-28T05:03:04.633957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.to_parquet('add_train.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T05:05:03.082640Z","iopub.execute_input":"2025-06-28T05:05:03.082846Z","iopub.status.idle":"2025-06-28T05:05:04.469988Z","shell.execute_reply.started":"2025-06-28T05:05:03.082831Z","shell.execute_reply":"2025-06-28T05:05:04.469268Z"}},"outputs":[],"execution_count":null}]}