{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-01T04:23:25.018774Z","iopub.execute_input":"2022-02-01T04:23:25.019903Z","iopub.status.idle":"2022-02-01T04:23:25.150712Z","shell.execute_reply.started":"2022-02-01T04:23:25.019718Z","shell.execute_reply":"2022-02-01T04:23:25.149860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')\n#asset_details = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv')\n#example_test = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/example_test.csv')\n#example_sample_submission = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/example_sample_submission.csv')\nsupplemental_train = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv')\n\n#print('train', train.shape)\n#print('asset_details', asset_details.shape)\n#print('example_test', example_test.shape)\n#print('example_sample_submission', example_sample_submission.shape)\nprint('supplemental_train', supplemental_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T04:23:25.152734Z","iopub.execute_input":"2022-02-01T04:23:25.153207Z","iopub.status.idle":"2022-02-01T04:23:40.957816Z","shell.execute_reply.started":"2022-02-01T04:23:25.153164Z","shell.execute_reply":"2022-02-01T04:23:40.956675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hist_data_dict(hist_data):\n    \"\"\" \n        构建 hist_data_dict ： \n            1. 补足begin_t和end_t之间缺失的数据\n            2. 新增day和min字段\n    \"\"\"\n    hist_data_dict = {}\n    for s in hist_data['Asset_ID'].unique():\n        s_hist_data = hist_data[hist_data['Asset_ID']==s]\n        begin_t = s_hist_data['timestamp'].values[0]\n        end_t = s_hist_data['timestamp'].values[-1]\n        complete_ts = [t for t in range(begin_t, end_t+1, 60)]\n        complete_close_df = pd.merge(pd.DataFrame(complete_ts, columns=['timestamp']),\n                                     s_hist_data[['timestamp', 'Close', 'Target']],\n                                     how='left',\n                                     on='timestamp'\n                                    )\n        complete_close_df['Close'][complete_close_df['Close'].isnull()] = 0.5*(complete_close_df['Close'].fillna(method='ffill') + complete_close_df['Close'].fillna(method='bfill'))\n        complete_close_df['Target'][complete_close_df['Target'].isnull()] = 0.5*(complete_close_df['Target'].fillna(method='ffill') + complete_close_df['Target'].fillna(method='bfill'))\n        complete_close_df['Close'] = complete_close_df['Close'].fillna(method='bfill')\n        complete_close_df['Target'] = complete_close_df['Target'].fillna(method='bfill')\n        complete_close_df.dropna(subset=['Target'], inplace=True)\n        \n        complete_close_df['day'] = complete_close_df['timestamp']/60//1440\n        complete_close_df['min'] = complete_close_df['timestamp']/60%1440\n\n        hist_data_dict[s] = complete_close_df\n    return hist_data_dict","metadata":{"execution":{"iopub.status.busy":"2022-02-01T04:23:40.959433Z","iopub.execute_input":"2022-02-01T04:23:40.959749Z","iopub.status.idle":"2022-02-01T04:23:40.973168Z","shell.execute_reply.started":"2022-02-01T04:23:40.959708Z","shell.execute_reply":"2022-02-01T04:23:40.972154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_data_dict = get_hist_data_dict(supplemental_train)","metadata":{"execution":{"iopub.status.busy":"2022-02-01T04:23:40.974823Z","iopub.execute_input":"2022-02-01T04:23:40.975087Z","iopub.status.idle":"2022-02-01T04:23:44.209983Z","shell.execute_reply.started":"2022-02-01T04:23:40.975027Z","shell.execute_reply":"2022-02-01T04:23:44.208791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_period = 30\n\nclose_arr_dict = {s: hist_data_dict[s]['Close'][-map_period:].fillna(method='ffill').values for s in hist_data_dict}\nprint('close_arr_dict', len(close_arr_dict))","metadata":{"execution":{"iopub.status.busy":"2022-02-01T04:23:44.211698Z","iopub.execute_input":"2022-02-01T04:23:44.212079Z","iopub.status.idle":"2022-02-01T04:23:44.225148Z","shell.execute_reply.started":"2022-02-01T04:23:44.212012Z","shell.execute_reply":"2022-02-01T04:23:44.224258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nimport time \n\nenv = gresearch_crypto.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\n\nfor (test_df, pred_df) in iter_test:\n    for idx, row in enumerate(test_df.iterrows()):\n        start = time.time()\n        # 解析该行数据\n        t, symbol, cnt, o, h, l, c, vol, vwap, rowid = row[1]\n\n        # 构造最新的close_arr，更新close_arr_dict\n        close_arr = np.roll(close_arr_dict[symbol], -1)\n        close_arr[-1] = c\n        close_arr_dict[symbol] = close_arr\n\n        # close_arr进行一阶拟合和二阶拟合\n        params_d1 = np.polyfit(np.arange(1, map_period+1), close_arr, 1)\n        params_d1 = np.polyfit(np.arange(1, map_period+1), close_arr, 2)\n        # 预测pred_c\n        pred_t = map_period + 15\n        pred_c1 = params_d1[0]*pred_t + params_d1[1]\n        pred_c2 = params_d1[0]*pred_t**2 + params_d1[1]*pred_t + params_d1[2]\n        pred_c = (pred_c1 + pred_c2)/2     \n\n        # 预测target\n        try:\n            best_target = np.log(pred_c/c)\n        except:\n            best_target = 0.0\n\n        # 赋值给pref_df提交\n        pred_df.iloc[idx, 1] = best_target        \n        print(time.time()-start)\n    env.predict(pred_df)   # register your predictions","metadata":{"execution":{"iopub.status.busy":"2022-02-01T04:23:48.471176Z","iopub.execute_input":"2022-02-01T04:23:48.471480Z","iopub.status.idle":"2022-02-01T04:23:48.652739Z","shell.execute_reply.started":"2022-02-01T04:23:48.471447Z","shell.execute_reply":"2022-02-01T04:23:48.651469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-01-30T03:32:04.452648Z","iopub.execute_input":"2022-01-30T03:32:04.453646Z","iopub.status.idle":"2022-01-30T03:32:04.461906Z","shell.execute_reply.started":"2022-01-30T03:32:04.453594Z","shell.execute_reply":"2022-01-30T03:32:04.460529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}