{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook provides the absolute minimum code necessary for making a submission and is meant for copying into your own pipeline. If you're looking for a more detailed explanation of how the API works, take a look at [this notebook instead](https://www.kaggle.com/sohier/detailed-api-introduction).","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import RobustScaler\nfrom scipy.stats import pearsonr","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:49:03.838469Z","iopub.execute_input":"2021-11-03T14:49:03.838845Z","iopub.status.idle":"2021-11-03T14:49:06.265200Z","shell.execute_reply.started":"2021-11-03T14:49:03.838811Z","shell.execute_reply":"2021-11-03T14:49:06.264312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    nof_trees = 100","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:49:06.266691Z","iopub.execute_input":"2021-11-03T14:49:06.266906Z","iopub.status.idle":"2021-11-03T14:49:06.271425Z","shell.execute_reply.started":"2021-11-03T14:49:06.266881Z","shell.execute_reply":"2021-11-03T14:49:06.270573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"def process_data(df):\n    \n    \n    col_subset = ['High', 'Low', 'Close', 'Open', 'Count', 'Volume']\n    \n#     dfa = pd.get_dummies(df['Asset_ID'])\n#     dfa.columns = ['asset' + str(f) for f in range(dfa.shape[1])]    \n#     df = pd.concat([df[col_subset], dfa], axis = 1 )\n\n#     df['Upper_Shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n#     df['Lower_Shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n    \n#     # returns across different combinations\n    df['hl_ret'] = (df['High']/df['Low'] - 1)\n#     df['hc_ret'] = (df['High']/df['Close'] - 1)\n#     df['ho_ret'] = (df['High']/df['Open'] - 1)    \n#     df['lh_ret'] = (df['Low']/df['High'] - 1)\n#     df['lc_ret'] = (df['Low']/df['Close'] - 1)\n#     df['lo_ret'] = (df['Low']/df['Open'] - 1)\n#     df['ch_ret'] = (df['Close']/df['High'] - 1)\n#     df['cl_ret'] = (df['Close']/df['Low'] - 1)\n#     df['co_ret'] = (df['Close']/df['Open'] - 1)\n#     df['oh_ret'] = (df['Open']/df['High'] - 1)\n#     df['ol_ret'] = (df['Open']/df['Low'] - 1)\n    df['oc_ret'] = (df['Open']/df['Close'] - 1)\n    \n    # volume\n    df['Volume'] = df['Volume']/10**6\n    \n#     # VWAP\n#     df['VWAP'] = df['VWAP'].replace(-np.inf, 0)\n#     df['VWAP'] = df['VWAP'].replace(np.inf, 0)    \n#     df['VWAP'] = df['VWAP'].fillna(0)\n#     df['VWAP'] = df['VWAP']/10**4\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:49:06.272603Z","iopub.execute_input":"2021-11-03T14:49:06.272837Z","iopub.status.idle":"2021-11-03T14:49:06.286670Z","shell.execute_reply.started":"2021-11-03T14:49:06.272810Z","shell.execute_reply":"2021-11-03T14:49:06.285655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\n\n# xte = pd.read_csv('../input/g-research-crypto-forecasting/example_test.csv')\n","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:49:06.287954Z","iopub.execute_input":"2021-11-03T14:49:06.288490Z","iopub.status.idle":"2021-11-03T14:50:04.270659Z","shell.execute_reply.started":"2021-11-03T14:49:06.288454Z","shell.execute_reply":"2021-11-03T14:50:04.269829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = ['Asset_ID', 'Close', 'Count', 'High', 'Low', 'Open', 'VWAP',\n       'Volume', 'timestamp']","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:50:04.272488Z","iopub.execute_input":"2021-11-03T14:50:04.273118Z","iopub.status.idle":"2021-11-03T14:50:04.276759Z","shell.execute_reply.started":"2021-11-03T14:50:04.273075Z","shell.execute_reply":"2021-11-03T14:50:04.276180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.dropna(axis = 0, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:50:04.277757Z","iopub.execute_input":"2021-11-03T14:50:04.278288Z","iopub.status.idle":"2021-11-03T14:50:06.506916Z","shell.execute_reply.started":"2021-11-03T14:50:04.278257Z","shell.execute_reply":"2021-11-03T14:50:06.505949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\nTODO: fit model per asset","metadata":{}},{"cell_type":"code","source":"ytrain = df_train['Target'].copy()\nxtrain = df_train.copy()\n\ndel df_train","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:50:06.508222Z","iopub.execute_input":"2021-11-03T14:50:06.508479Z","iopub.status.idle":"2021-11-03T14:50:07.363444Z","shell.execute_reply.started":"2021-11-03T14:50:06.508450Z","shell.execute_reply":"2021-11-03T14:50:07.362661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = int(0.9 * xtrain.shape[0])\nx0, y0 = xtrain.iloc[:train_size],ytrain.iloc[:train_size]\nx1, y1 = xtrain.iloc[train_size:],ytrain.iloc[train_size:]\n\n\nx0 = process_data(x0)\nx1 = process_data(x1)\n\ndel xtrain, ytrain","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:50:07.364484Z","iopub.execute_input":"2021-11-03T14:50:07.365279Z","iopub.status.idle":"2021-11-03T14:50:13.233916Z","shell.execute_reply.started":"2021-11-03T14:50:07.365244Z","shell.execute_reply":"2021-11-03T14:50:13.232917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ridge","metadata":{}},{"cell_type":"code","source":"rs = RobustScaler()\nrs.fit(x0)\n\nx0 = rs.transform(x0)\nx1 = rs.transform(x1)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:50:49.346151Z","iopub.execute_input":"2021-11-03T14:50:49.346603Z","iopub.status.idle":"2021-11-03T14:51:24.924588Z","shell.execute_reply.started":"2021-11-03T14:50:49.346556Z","shell.execute_reply":"2021-11-03T14:51:24.922593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Ridge()\n\nmodel.fit(x0,y0)\n\npred = model.predict(x1)\n\n\nprint(np.corrcoef(y1,pred))\n\n# 0.0068182","metadata":{"execution":{"iopub.status.busy":"2021-11-03T14:51:44.404950Z","iopub.execute_input":"2021-11-03T14:51:44.405944Z","iopub.status.idle":"2021-11-03T14:51:52.930887Z","shell.execute_reply.started":"2021-11-03T14:51:44.405887Z","shell.execute_reply":"2021-11-03T14:51:52.929850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nTODO:\n* prediction per asset","metadata":{}},{"cell_type":"code","source":"\n# env = gresearch_crypto.make_env()\n# iter_test = env.iter_test()\n# for (test_df, sample_prediction_df) in iter_test:\n#     xte = process_data(test_df)\n#     sample_prediction_df['Target'] = model.predict(xte)\n#     env.predict(sample_prediction_df)","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":0.025192,"end_time":"2020-10-19T15:08:36.590409","exception":false,"start_time":"2020-10-19T15:08:36.565217","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}