{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport keras\nfrom keras.models import Sequential\nfrom sklearn.preprocessing import RobustScaler\n# from keras.preprocessing.sequence import TimeseriesGenerator\nfrom sklearn.model_selection import train_test_split\n# import tensorflow_probability as tfp\nimport gresearch_crypto\nimport gc\n","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:45:28.778922Z","iopub.execute_input":"2022-01-17T14:45:28.779507Z","iopub.status.idle":"2022-01-17T14:45:33.958093Z","shell.execute_reply.started":"2022-01-17T14:45:28.7794Z","shell.execute_reply":"2022-01-17T14:45:33.957228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir_path = '/kaggle/input/g-research-crypto-forecasting/'\ntrain = pd.read_csv(dir_path + 'train.csv').set_index('timestamp')\nassets = pd.read_csv(dir_path + 'asset_details.csv')\nassets_order = pd.read_csv(dir_path + 'supplemental_train.csv').Asset_ID[:14]","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:45:33.96323Z","iopub.execute_input":"2022-01-17T14:45:33.965428Z","iopub.status.idle":"2022-01-17T14:46:34.092891Z","shell.execute_reply.started":"2022-01-17T14:45:33.96539Z","shell.execute_reply":"2022-01-17T14:46:34.092135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train data set","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"np.random.seed(1)\ntf.random.set_seed(1)","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:46:34.095894Z","iopub.execute_input":"2022-01-17T14:46:34.096369Z","iopub.status.idle":"2022-01-17T14:46:34.100632Z","shell.execute_reply.started":"2022-01-17T14:46:34.096338Z","shell.execute_reply":"2022-01-17T14:46:34.099876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[:1000000]","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:34.103226Z","iopub.execute_input":"2022-01-17T14:46:34.103505Z","iopub.status.idle":"2022-01-17T14:46:34.11079Z","shell.execute_reply.started":"2022-01-17T14:46:34.103462Z","shell.execute_reply":"2022-01-17T14:46:34.110025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VWAP_MAX =np.max(train[np.isfinite(train.VWAP)].VWAP)\nVWAP_MIN =np.min(train[np.isfinite(train.VWAP)].VWAP)\nprint(VWAP_MAX)\nprint(VWAP_MIN)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:34.111729Z","iopub.execute_input":"2022-01-17T14:46:34.111974Z","iopub.status.idle":"2022-01-17T14:46:34.199416Z","shell.execute_reply.started":"2022-01-17T14:46:34.111941Z","shell.execute_reply":"2022-01-17T14:46:34.198636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_VWAP = np.nan_to_num(train.VWAP, posinf=VWAP_MAX, neginf=VWAP_MIN)\ndel train['VWAP']\ntrain['VWAP'] = tmp_VWAP\ntrain.shape","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:34.200579Z","iopub.execute_input":"2022-01-17T14:46:34.201344Z","iopub.status.idle":"2022-01-17T14:46:34.21896Z","shell.execute_reply.started":"2022-01-17T14:46:34.201307Z","shell.execute_reply":"2022-01-17T14:46:34.218219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"Data NULL: \\n\", train.isnull().sum())","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:34.220391Z","iopub.execute_input":"2022-01-17T14:46:34.220649Z","iopub.status.idle":"2022-01-17T14:46:34.22467Z","shell.execute_reply.started":"2022-01-17T14:46:34.220615Z","shell.execute_reply":"2022-01-17T14:46:34.223535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Target'] = train.groupby(['Asset_ID'], sort=False)['Target'].apply(lambda x: x.fillna(x.mean()))","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:34.226438Z","iopub.execute_input":"2022-01-17T14:46:34.226756Z","iopub.status.idle":"2022-01-17T14:46:55.564165Z","shell.execute_reply.started":"2022-01-17T14:46:34.226723Z","shell.execute_reply":"2022-01-17T14:46:55.563461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train[['Asset_ID','Target']].copy()\ntime = dict((i,j) for j,i in enumerate(df.index.unique()))\ndf['id'] = df.index.map(time)\n# df","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:55.565295Z","iopub.execute_input":"2022-01-17T14:46:55.56557Z","iopub.status.idle":"2022-01-17T14:46:55.8208Z","shell.execute_reply.started":"2022-01-17T14:46:55.565535Z","shell.execute_reply":"2022-01-17T14:46:55.820092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\n# df","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:55.823795Z","iopub.execute_input":"2022-01-17T14:46:55.824147Z","iopub.status.idle":"2022-01-17T14:46:58.300837Z","shell.execute_reply.started":"2022-01-17T14:46:55.82411Z","shell.execute_reply":"2022-01-17T14:46:58.300131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = df.id.copy()\ndel df","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:58.302045Z","iopub.execute_input":"2022-01-17T14:46:58.302309Z","iopub.status.idle":"2022-01-17T14:46:58.319172Z","shell.execute_reply.started":"2022-01-17T14:46:58.302276Z","shell.execute_reply":"2022-01-17T14:46:58.318494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(df):\n    df['upper'] = df['High'] - np.maximum(df['Close'],df['Open'])\n    df['lower'] = np.minimum(df['Close'],df['Open'])-df['Low']\n    df['range'] = df['High'] - df['Low']\n    df['mean_trade'] = df['Volume']/df['Count']\n    df['log_price_change'] = np.log(df['Close']/df['Open'])\n    return df\ntrain=add_features(train)\ntrain.shape","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:58.320549Z","iopub.execute_input":"2022-01-17T14:46:58.3213Z","iopub.status.idle":"2022-01-17T14:46:58.367906Z","shell.execute_reply.started":"2022-01-17T14:46:58.321246Z","shell.execute_reply":"2022-01-17T14:46:58.36723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.head()","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:58.368976Z","iopub.execute_input":"2022-01-17T14:46:58.369212Z","iopub.status.idle":"2022-01-17T14:46:58.373027Z","shell.execute_reply.started":"2022-01-17T14:46:58.369179Z","shell.execute_reply":"2022-01-17T14:46:58.372314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = train.columns.drop(['Asset_ID','Target'])\ntrain[features] = RobustScaler().fit_transform(train[features])\ntrain.head()","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:58.374143Z","iopub.execute_input":"2022-01-17T14:46:58.374811Z","iopub.status.idle":"2022-01-17T14:46:58.871869Z","shell.execute_reply.started":"2022-01-17T14:46:58.374774Z","shell.execute_reply":"2022-01-17T14:46:58.871129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = train.index.unique()\n# for i in range(index[0],index[-1]+60,60):\n#     print(i)\ndef reindex(df):\n    df = df.reindex(range(index[0],index[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\ntrain=train.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\ntrain.shape","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:58.873123Z","iopub.execute_input":"2022-01-17T14:46:58.873444Z","iopub.status.idle":"2022-01-17T14:46:59.430923Z","shell.execute_reply.started":"2022-01-17T14:46:58.873404Z","shell.execute_reply":"2022-01-17T14:46:59.430235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  把group_num and row_id 合併起來\ntrain['group_num'] = train.index.map(time)\ntrain = train.dropna(subset=['group_num'])\ntrain['group_num'] = train['group_num'].astype('int')\n\n# 標記產生出來的row為non-real\ntrain['id'] = train['group_num'].astype(str) + '_' + train['Asset_ID'].astype(str)\ntrain['is_real'] = train.id.isin(ids)*1\ntrain = train.drop('id', axis=1)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:46:59.433253Z","iopub.execute_input":"2022-01-17T14:46:59.433521Z","iopub.status.idle":"2022-01-17T14:47:03.650209Z","shell.execute_reply.started":"2022-01-17T14:46:59.433485Z","shell.execute_reply":"2022-01-17T14:47:03.649488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# non_real 的設為0\nfeatures = train.columns.drop(['Asset_ID','group_num','is_real'])\ntrain.loc[train.is_real==0, features]=0","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:47:03.651493Z","iopub.execute_input":"2022-01-17T14:47:03.651762Z","iopub.status.idle":"2022-01-17T14:47:03.728983Z","shell.execute_reply.started":"2022-01-17T14:47:03.651727Z","shell.execute_reply":"2022-01-17T14:47:03.728283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 將'supplemental_train.csv' 中的asset_order合併進來\ntrain['asset_order'] = train.Asset_ID.map(assets_order)\ntrain=train.sort_values(by=['group_num', 'asset_order'])\ntrain.head(5)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:47:03.730232Z","iopub.execute_input":"2022-01-17T14:47:03.73048Z","iopub.status.idle":"2022-01-17T14:47:04.000821Z","shell.execute_reply.started":"2022-01-17T14:47:03.730449Z","shell.execute_reply":"2022-01-17T14:47:04.000125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train.Target.copy()\ny_train.shape","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:47:04.002017Z","iopub.execute_input":"2022-01-17T14:47:04.003419Z","iopub.status.idle":"2022-01-17T14:47:04.011456Z","shell.execute_reply.started":"2022-01-17T14:47:04.003378Z","shell.execute_reply":"2022-01-17T14:47:04.010564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train.drop(['Target','Asset_ID','Open','High','Low','is_real','group_num','asset_order','Close','upper','lower'],axis = 1)\nprint(x_train.shape)\nprint(x_train.head(5))","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:47:04.013046Z","iopub.execute_input":"2022-01-17T14:47:04.013404Z","iopub.status.idle":"2022-01-17T14:47:04.055737Z","shell.execute_reply.started":"2022-01-17T14:47:04.013368Z","shell.execute_reply":"2022-01-17T14:47:04.054951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.to_numpy().reshape(-1,6,1)\nx_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:47:04.056777Z","iopub.execute_input":"2022-01-17T14:47:04.057015Z","iopub.status.idle":"2022-01-17T14:47:04.066028Z","shell.execute_reply.started":"2022-01-17T14:47:04.056981Z","shell.execute_reply":"2022-01-17T14:47:04.065239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x, val_x, train_y, val_y = train_test_split(x_train, y_train, test_size = 0.25, random_state=2)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:47:04.067395Z","iopub.execute_input":"2022-01-17T14:47:04.068002Z","iopub.status.idle":"2022-01-17T14:47:04.25374Z","shell.execute_reply.started":"2022-01-17T14:47:04.067968Z","shell.execute_reply":"2022-01-17T14:47:04.253006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.models.Sequential([\n      keras.layers.LSTM(64,return_sequences=True,input_shape=(6,1)),\n      keras.layers.ReLU(),\n      keras.layers.Dropout(0.2),\n      keras.layers.LSTM(32,return_sequences=True, activation='relu'),\n      keras.layers.Dropout(0.5),\n      keras.layers.LSTM(32),\n\n      keras.layers.Dropout(0.5),\n      keras.layers.Dense(1)#activation = \"linear\"\n])\nmodel.summary()","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:47:04.254906Z","iopub.execute_input":"2022-01-17T14:47:04.255631Z","iopub.status.idle":"2022-01-17T14:47:04.259759Z","shell.execute_reply.started":"2022-01-17T14:47:04.255592Z","shell.execute_reply":"2022-01-17T14:47:04.258837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from keras.layers import Dense, LSTM, Dropout, GRU\n# from keras.models import Sequential\n# from tensorflow.keras.optimizers import SGD\n# # The GRU architecture\n# regressorGRU = Sequential()\n# # First GRU layer with Dropout regularisation\n# regressorGRU.add(GRU(units=64, return_sequences=True, input_shape=(train_x.shape[1],1), activation='ReLU'))\n# regressorGRU.add(Dropout(0.2))\n# # Second GRU layer\n# regressorGRU.add(GRU(units=32, input_shape=(train_x.shape[1],1)))\n# regressorGRU.add(Dropout(0.5))\n# # # Third GRU layer\n# # regressorGRU.add(GRU(units=50, return_sequences=True, input_shape=(train_x.shape[1],1), activation='tanh'))\n# # regressorGRU.add(Dropout(0.2))\n# # # Fourth GRU layer\n# # regressorGRU.add(GRU(units=50, activation='tanh'))\n# # regressorGRU.add(Dropout(0.2))\n# # The output layer\n# regressorGRU.add(Dense(units=1))\n","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:47:04.261101Z","iopub.execute_input":"2022-01-17T14:47:04.261491Z","iopub.status.idle":"2022-01-17T14:47:07.109367Z","shell.execute_reply.started":"2022-01-17T14:47:04.261457Z","shell.execute_reply":"2022-01-17T14:47:07.108662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.metrics import accuracy\n# optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\noptimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\nmodel.compile(optimizer = optimizer, loss='mse', metrics = ['mae'])\n\nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",\n                          mode=\"min\",\n                          patience=5)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:47:07.11067Z","iopub.execute_input":"2022-01-17T14:47:07.110901Z","iopub.status.idle":"2022-01-17T14:47:07.12978Z","shell.execute_reply.started":"2022-01-17T14:47:07.110868Z","shell.execute_reply":"2022-01-17T14:47:07.128924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from keras.metrics import accuracy\n# from tensorflow.keras import layers, Model\n# # optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n# # model = LSTM_model()\n# optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n# model.compile(optimizer = optimizer, loss='mse', metrics = ['mae'])\n\n# early_stopping = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",\n#                           mode=\"min\",\n#                           patience=5)\n# # model.summary()","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:47:07.131254Z","iopub.execute_input":"2022-01-17T14:47:07.131514Z","iopub.status.idle":"2022-01-17T14:47:07.135902Z","shell.execute_reply.started":"2022-01-17T14:47:07.131482Z","shell.execute_reply":"2022-01-17T14:47:07.135062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history=model.fit(x_train,y_train, batch_size=64,epochs=5,validation_data=(val_x,val_y), callbacks=[early_stopping], shuffle=True)","metadata":{"collapsed":false,"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T14:47:07.137346Z","iopub.execute_input":"2022-01-17T14:47:07.138158Z","iopub.status.idle":"2022-01-17T14:47:44.723898Z","shell.execute_reply.started":"2022-01-17T14:47:07.138123Z","shell.execute_reply":"2022-01-17T14:47:44.722619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Diffining Figure\nimport matplotlib.pyplot as plt\nfigure = plt.figure(figsize=(20,7))\n\n#Adding Subplot 1 (For Accuracy)\nfigure.add_subplot(121)\n\nplt.plot(history.epoch,history.history['mae'],label = \"mae\") # Accuracy curve for training set\nplt.plot(history.epoch,history.history['val_mae'],label = \"val_mae\") # Accuracy curve for validation set\n\nplt.title(\"MAE Curve\",fontsize=18)\nplt.xlabel(\"Epochs\",fontsize=15)\nplt.ylabel(\"MAE\",fontsize=15)\nplt.grid(alpha=0.3)\nplt.legend()\n\n#Adding Subplot 1 (For Loss)\nfigure.add_subplot(122)\n\nplt.plot(history.epoch,history.history['loss'],label=\"loss\") # Loss curve for training set\nplt.plot(history.epoch,history.history['val_loss'],label=\"val_loss\") # Loss curve for validation set\n\nplt.title(\"Loss Curve\",fontsize=18)\nplt.xlabel(\"Epochs\",fontsize=15)\nplt.ylabel(\"Loss\",fontsize=15)\nplt.grid(alpha=0.3)\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:47:44.725107Z","iopub.status.idle":"2022-01-17T14:47:44.725678Z","shell.execute_reply.started":"2022-01-17T14:47:44.725434Z","shell.execute_reply":"2022-01-17T14:47:44.725458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## test data set","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"def add_features_test(df):\n    df['upper'] = df['High'] - np.maximum(df['Close'],df['Open'])\n    df['lower'] = np.minimum(df['Close'],df['Open'])-df['Low']\n    df['range'] = df['High'] - df['Low']\n    df['mean_trade'] = df['Volume']/df['Count']\n    df['log_price_change'] = np.log(df['Close']/df['Open'])\n    return df\n\ndef reindex_test(df):\n    df = df.reindex(range(index[0],index[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:47:44.726857Z","iopub.status.idle":"2022-01-17T14:47:44.727413Z","shell.execute_reply.started":"2022-01-17T14:47:44.727177Z","shell.execute_reply":"2022-01-17T14:47:44.727203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:47:44.72844Z","iopub.status.idle":"2022-01-17T14:47:44.72897Z","shell.execute_reply.started":"2022-01-17T14:47:44.728747Z","shell.execute_reply":"2022-01-17T14:47:44.728773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (test, submission) in enumerate(iter_test): \n#     submission['Target'] = test_y[i*14:i*14+14]  # make your predictions here\n    VWAP_MAX =np.max(test[np.isfinite(test.VWAP)].VWAP)\n    VWAP_MIN =np.min(test[np.isfinite(test.VWAP)].VWAP)\n    tmp_VWAP = np.nan_to_num(test.VWAP, posinf=VWAP_MAX, neginf=VWAP_MIN)\n    del test['VWAP']\n    test['VWAP'] = tmp_VWAP\n#     print(test.shape)\n    df = test[['Asset_ID']].copy()\n    time = dict((i,j) for j,i in enumerate(df.index.unique()))\n    df['id'] = df.index.map(time)\n\n    df['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\n    ids = df.id.copy()\n    del df\n    \n    test=add_features_test(test)\n#     print(test.shape)\n    \n    features = test.columns.drop(['Asset_ID','row_id'])\n    test[features] = RobustScaler().fit_transform(test[features])\n    \n    index = test.index.unique()\n    test=test.groupby('Asset_ID').apply(reindex_test).reset_index(0, drop=True).sort_index()\n#     print(test.shape)\n    \n    #  把group_num and row_id 合併起來\n    test['group_num'] = test.index.map(time)\n    test = test.dropna(subset=['group_num'])\n    test['group_num'] = test['group_num'].astype('int')\n\n    # 標記產生出來的row為non-real\n    test['id'] = test['group_num'].astype(str) + '_' + test['Asset_ID'].astype(str)\n    test['is_real'] = test.id.isin(ids)*1\n    test = test.drop('id', axis=1)\n    \n    # non_real 的設為0\n    features_test = test.columns.drop(['Asset_ID','group_num','is_real'])\n    test.loc[test.is_real==0, features_test]=0\n    \n    test['asset_order'] = test.Asset_ID.map(assets_order)\n    test=test.sort_values(by=['asset_order'])\n#     print(test.shape)\n    \n    del test['row_id']\n    del test['timestamp']\n#     print(test.shape)\n    test = test.drop(['Asset_ID','Open','High','Low', 'Close','group_num','is_real','asset_order','upper','lower'],axis = 1)\n#     print(test.shape)\n#     print(test.info())\n    test = test.to_numpy().reshape(-1,6,1)\n#     print(test.shape)\n\n    test_y = np.mean(model.predict(test),axis =1)\n#     submission['Target'] = 0\n    submission['Target'] = test_y\n    print(submission)\n    env.predict(submission)   # register your predictions","metadata":{"execution":{"iopub.status.busy":"2022-01-17T14:47:44.730064Z","iopub.status.idle":"2022-01-17T14:47:44.730616Z","shell.execute_reply.started":"2022-01-17T14:47:44.730377Z","shell.execute_reply":"2022-01-17T14:47:44.730404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}