{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gresearch_crypto\nfrom lightgbm import LGBMRegressor\nimport datetime as dt\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\nSEED = 202112","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:47:34.119557Z","iopub.execute_input":"2022-01-18T02:47:34.120424Z","iopub.status.idle":"2022-01-18T02:47:36.232771Z","shell.execute_reply.started":"2022-01-18T02:47:34.120304Z","shell.execute_reply":"2022-01-18T02:47:36.231785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_used = ['Asset_ID', 'Count', 'Open', 'Close', 'Volume', 'VWAP'\n             , 'weekday', 'hour', 'upper_shadow', 'lower_shadow'#, 'Mean'\n             , \"Close/Open\", 'hlco_ratio', 'spread', 'mean_trade'#, 'log_price_change'\n             , 'Count_g', 'Volume_g', 'VWAP_g']\n\ndef feature_engneering(df):\n    df['weekday'] = df.index.weekday\n    df['hour'] = df.index.hour\n    df['upper_shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n    df['lower_shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n    #df['Mean'] = df[['Open', 'High', 'Low', 'Close']].mean()\n    df[\"Close/Open\"] = df[\"Close\"] / df[\"Open\"] \n    df['hlco_ratio'] = (df['High'] - df['Low'])/(df['Close']-df['Open'])\n    df['spread'] = df['High'] - df['Low']\n    df['mean_trade'] = df['Volume']/df['Count']\n    #df['log_price_change'] = np.log(df['Close']/df['Open'])\n    df_X = df[cols_used]\n    return df_X\n\ndef model_train(df_X, df_y):\n    \n    model = LGBMRegressor(#boosting = 'dart', \n                          n_estimators = 800, \n                          learning_rate = 0.05, \n                          max_bin = 15, \n                          num_leaves = 4,\n                          random_state = SEED)\n    model.fit(df_X, df_y)\n    \n    return df_X, df_y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:47:36.234484Z","iopub.execute_input":"2022-01-18T02:47:36.235010Z","iopub.status.idle":"2022-01-18T02:47:36.247686Z","shell.execute_reply.started":"2022-01-18T02:47:36.234965Z","shell.execute_reply":"2022-01-18T02:47:36.246548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')\n#asset_dt = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv')\n\ndf['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\ndf['datetime_g'] = df['datetime'].dt.strftime(\"%Y-%m-%d\")\ndf = df.set_index('datetime').drop('timestamp', axis=1)\ndf = df[(df.index.year == 2021) & (df.index.month < 6)].dropna(subset=[\"Target\"])\n\ndf_g = df.groupby(by = [\"Asset_ID\", 'datetime_g']).mean()\ndf_g = df_g[['Count', 'Volume', 'VWAP']]\ndf_g.columns = ['Count_g', 'Volume_g', 'VWAP_g']\ndf_g = df_g.reset_index().set_index('datetime_g')\n\ndf_index = df.index\ndf = df.merge(df_g, how = 'left', on = [\"Asset_ID\", 'datetime_g'])\ndf.index = df_index\n\n#df = df.dropna(subset=[\"Target\"])\ndf_train_X = feature_engneering(df)\ndf_train_y = df['Target']\ndel df","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:47:36.249397Z","iopub.execute_input":"2022-01-18T02:47:36.249745Z","iopub.status.idle":"2022-01-18T02:52:02.426955Z","shell.execute_reply.started":"2022-01-18T02:47:36.249706Z","shell.execute_reply":"2022-01-18T02:52:02.426169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y, model = model_train(df_train_X, df_train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:52:02.428410Z","iopub.execute_input":"2022-01-18T02:52:02.428934Z","iopub.status.idle":"2022-01-18T02:53:16.528337Z","shell.execute_reply.started":"2022-01-18T02:52:02.428887Z","shell.execute_reply":"2022-01-18T02:53:16.527476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import time\n# time_start = time.time()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    df_test['datetime'] = pd.to_datetime(df_test['timestamp'], unit='s')\n    df_test['datetime_g'] = df_test['datetime'].dt.strftime(\"%Y-%m-%d\")\n    row_ls = pd.DataFrame()\n    for _, row in df_test.iterrows():\n        row_ID = row['Asset_ID']\n        try:\n            df_temp = df_g[df_g['Asset_ID'] == row_ID]\n        except:\n            df_temp = df_g[df_g['Asset_ID'] == 1]\n        df_temp.index = pd.to_datetime(df_temp.index, format = \"%Y-%m-%d\")\n        row_time = pd.to_datetime(row['datetime_g'], format = \"%Y-%m-%d\")\n        row = pd.DataFrame(row).T\n        try:\n            closest_train_sample = df_temp.iloc[df_temp.index.get_loc(row_time, method='nearest')]\n            row.loc[:, 'Count_g'],  row.loc[:, 'Volume_g'], row.loc[:, 'VWAP_g'] = closest_train_sample['Count_g'], closest_train_sample['Volume_g'], closest_train_sample['VWAP_g']\n        except:\n            print('New Asset')\n            row.loc[:, 'Count_g'],  row.loc[:, 'Volume_g'], row.loc[:, 'VWAP_g'] = row['Count'],  row['Volume'], row['VWAP']\n        #row = pd.DataFrame(row).T\n        row_ls = row_ls.append(row, ignore_index = True)\n    df_train = row_ls.set_index('datetime').drop('timestamp', axis=1)\n    df = feature_engneering(df_train)\n    col_float = df.columns.drop(['Asset_ID', 'weekday', 'hour']).to_list()\n    col_int = ['Asset_ID', 'weekday', 'hour']\n    df_t = df.copy()\n    df_t.loc[:, col_float] = df[col_float].astype(np.float64)\n    df_t.loc[:, col_int] = df[col_int].astype(int)\n    y_pred = model.predict(df_t)\n    df_pred['Target'] = y_pred\n    df_pred['Target'] = df_pred['Target'].fillna(0)\n    #print(df, df_pred)\n    env.predict(df_pred)\n    \n# time_end = time.time()\n# print('Time cost = %fs' % (time_end - time_start))","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:53:16.530814Z","iopub.execute_input":"2022-01-18T02:53:16.531415Z","iopub.status.idle":"2022-01-18T02:53:17.046584Z","shell.execute_reply.started":"2022-01-18T02:53:16.531354Z","shell.execute_reply":"2022-01-18T02:53:17.045806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:58:11.377853Z","iopub.execute_input":"2022-01-18T02:58:11.378882Z","iopub.status.idle":"2022-01-18T02:58:11.385390Z","shell.execute_reply.started":"2022-01-18T02:58:11.378838Z","shell.execute_reply":"2022-01-18T02:58:11.384800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for df_test, df_pred in iter_test:\n#     df_test['datetime'] = pd.to_datetime(df_test['timestamp'], unit='s')\n#     df_test['datetime_g'] = df_test['datetime'].dt.strftime(\"%Y-%m-%d\")\n#     for _, row in df_test.iterrows():\n#         row_ID = row['Asset_ID']\n#         try:\n#             df_temp = df_g[df_g['Asset_ID'] == row_ID]\n#         except:\n#             df_temp = df_g[df_g['Asset_ID'] == 1]\n            \n#         # closest_train_sample = df.iloc[df.index.get_loc(row['datetime'], method='nearest')]\n#         df_temp.index = pd.to_datetime(df_temp.index, format = \"%Y-%m-%d\")\n#         row_time = pd.to_datetime(row['datetime_g'], format = \"%Y-%m-%d\")\n#         row = pd.DataFrame(row).T\n#         try:\n#             closest_train_sample = df_temp.iloc[df_temp.index.get_loc(row_time, method='nearest')]\n#             row.loc[:, 'Count_g'],  row.loc[:, 'Volume_g'], row.loc[:, 'VWAP_g'] = closest_train_sample['Count_g'], closest_train_sample['Volume_g'], closest_train_sample['VWAP_g']\n#         except:\n#             #print('New Asset')\n#             row.loc[:, 'Count_g'],  row.loc[:, 'Volume_g'], row.loc[:, 'VWAP_g'] = row['Count'],  row['Volume'], row['VWAP']\n#     row = row.set_index('datetime').drop('timestamp', axis=1)\n#     row = feature_engneering(row)\n#     col_float = row.columns.drop(['Asset_ID', 'weekday', 'hour']).to_list()\n#     col_int = ['Asset_ID', 'weekday', 'hour']\n#     row.loc[:, col_float] = row[col_float].astype(np.float64)\n#     row.loc[:, col_int] = row[col_int].astype(int)\n#     df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = model.predict(row)\n#     df_pred['Target'] = df_pred['Target'].fillna(0)\n#     env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T02:53:17.048268Z","iopub.execute_input":"2022-01-18T02:53:17.048786Z","iopub.status.idle":"2022-01-18T02:53:17.057053Z","shell.execute_reply.started":"2022-01-18T02:53:17.048739Z","shell.execute_reply":"2022-01-18T02:53:17.055826Z"},"trusted":true},"execution_count":null,"outputs":[]}]}