{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Calculate the realized market return and the beta series of each coin.\n#The predicted target is based on the beta, so this process is necessary.\n#NOTICE: The realized beta are based on the history price,and different from the target's beta.","metadata":{"execution":{"iopub.status.busy":"2022-01-13T10:04:52.371734Z","iopub.execute_input":"2022-01-13T10:04:52.372991Z","iopub.status.idle":"2022-01-13T10:04:52.398313Z","shell.execute_reply.started":"2022-01-13T10:04:52.372842Z","shell.execute_reply":"2022-01-13T10:04:52.397289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom datetime import datetime\n\nfrom tqdm import tqdm\nimport pickle\n\nimport gresearch_crypto\n\ndata_folder = \"../input/g-research-crypto-forecasting/\"","metadata":{"execution":{"iopub.status.busy":"2022-01-13T10:24:31.923251Z","iopub.execute_input":"2022-01-13T10:24:31.924191Z","iopub.status.idle":"2022-01-13T10:24:31.929454Z","shell.execute_reply.started":"2022-01-13T10:24:31.924145Z","shell.execute_reply":"2022-01-13T10:24:31.928629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df = pd.read_csv(data_folder + 'train.csv')\nasset_details = pd.read_csv(data_folder + 'asset_details.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-13T10:04:52.437817Z","iopub.execute_input":"2022-01-13T10:04:52.438322Z","iopub.status.idle":"2022-01-13T10:05:56.290357Z","shell.execute_reply.started":"2022-01-13T10:04:52.438284Z","shell.execute_reply":"2022-01-13T10:05:56.289217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_list = crypto_df['timestamp'].unique()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T10:05:56.292362Z","iopub.execute_input":"2022-01-13T10:05:56.292655Z","iopub.status.idle":"2022-01-13T10:05:56.831392Z","shell.execute_reply.started":"2022-01-13T10:05:56.292618Z","shell.execute_reply":"2022-01-13T10:05:56.830434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#calculate the market r15 series\n\ndf_mkt = pd.DataFrame(index = index_list)\ndf_mkt[['r15','weight']] = 0\nfor i in tqdm(range(14)):\n    df_tmp = pd.DataFrame(index = index_list)\n    w = asset_details[asset_details['Asset_ID']==i].Weight.values[0]\n    df_tmp['Close'] = crypto_df[crypto_df['Asset_ID'] == i].set_index('timestamp')['Close']\n    df_tmp['r15'] = np.log(df_tmp['Close']/df_tmp['Close'].shift(15))*w\n    df_tmp['weight'] = w\n    df_mkt.loc[~df_tmp['r15'].isna(),['r15','weight']] += df_tmp[['r15','weight']].dropna()\nmkt_r15=df_mkt.r15/df_mkt.weight\nmkt_r15.to_pickle(f'/kaggle/working/mkt_r15.pkl')\ndel df_mkt","metadata":{"execution":{"iopub.status.busy":"2022-01-13T10:05:56.833840Z","iopub.execute_input":"2022-01-13T10:05:56.834135Z","iopub.status.idle":"2022-01-13T10:06:11.204675Z","shell.execute_reply.started":"2022-01-13T10:05:56.834095Z","shell.execute_reply":"2022-01-13T10:06:11.203617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#calculate the beta and save\n\ncols = ['Count','Open','High','Low','Close','Volume','VWAP','Target']\nfor i in tqdm(range(14)):\n    df_tmp = pd.DataFrame(index = index_list)\n    df_tmp[cols] = crypto_df[crypto_df['Asset_ID'] == i].set_index('timestamp')[cols]\n    df_tmp['r15'] = np.log(df_tmp['Close']/df_tmp['Close'].shift(15))\n    df_tmp['mkt_r15'] = mkt_r15\n    df_beta = df_tmp[['r15','mkt_r15']].dropna()\n    df_tmp['beta3750'] = (df_beta.r15*df_beta.mkt_r15).rolling(3750).mean()/(df_beta.mkt_r15**2).rolling(3750).mean()\n    pd.to_pickle(df_tmp, f'/kaggle/working/df_prefeatures{i}.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-01-13T10:06:11.205844Z","iopub.execute_input":"2022-01-13T10:06:11.206227Z","iopub.status.idle":"2022-01-13T10:06:42.451480Z","shell.execute_reply.started":"2022-01-13T10:06:11.206182Z","shell.execute_reply":"2022-01-13T10:06:42.450191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del crypto_df","metadata":{"execution":{"iopub.status.busy":"2022-01-13T10:06:42.452910Z","iopub.execute_input":"2022-01-13T10:06:42.453198Z","iopub.status.idle":"2022-01-13T10:06:42.463431Z","shell.execute_reply.started":"2022-01-13T10:06:42.453164Z","shell.execute_reply":"2022-01-13T10:06:42.462263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_pickle(f'/kaggle/working/df_prefeatures1.pkl')\ndf","metadata":{"execution":{"iopub.status.busy":"2022-01-13T10:06:42.465109Z","iopub.execute_input":"2022-01-13T10:06:42.465353Z","iopub.status.idle":"2022-01-13T10:06:42.605247Z","shell.execute_reply.started":"2022-01-13T10:06:42.465322Z","shell.execute_reply":"2022-01-13T10:06:42.604460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}