{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport keras\nfrom keras.models import Sequential\nfrom sklearn.preprocessing import RobustScaler\n# from keras.preprocessing.sequence import TimeseriesGenerator\nfrom sklearn.model_selection import train_test_split\n# import tensorflow_probability as tfp\n# import gresearch_crypto\nimport gc","metadata":{"id":"fNCCFEKAQuWB","execution":{"iopub.status.busy":"2022-01-12T17:19:12.601581Z","iopub.execute_input":"2022-01-12T17:19:12.601964Z","iopub.status.idle":"2022-01-12T17:19:17.638154Z","shell.execute_reply.started":"2022-01-12T17:19:12.601855Z","shell.execute_reply":"2022-01-12T17:19:17.637433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir_path = '/kaggle/input/g-research-crypto-forecasting/'\ntrain = pd.read_csv(dir_path + 'train.csv').set_index('timestamp')\nassets = pd.read_csv(dir_path + 'asset_details.csv' )\nassets_order = pd.read_csv(dir_path + 'supplemental_train.csv').Asset_ID[:14]\nsubmission = pd.read_csv(dir_path + 'example_sample_submission.csv' )","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"3LmmdZumQuWE","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:19:17.641666Z","iopub.execute_input":"2022-01-12T17:19:17.641883Z","iopub.status.idle":"2022-01-12T17:20:16.539627Z","shell.execute_reply.started":"2022-01-12T17:19:17.641857Z","shell.execute_reply":"2022-01-12T17:20:16.538833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train data set","metadata":{"pycharm":{"name":"#%% md\n"},"id":"I0TYF_c-QuWF"}},{"cell_type":"code","source":"# train.info()\n# assets.info()","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"cT0mJwwcQuWG","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.540912Z","iopub.execute_input":"2022-01-12T17:20:16.541166Z","iopub.status.idle":"2022-01-12T17:20:16.546098Z","shell.execute_reply.started":"2022-01-12T17:20:16.541134Z","shell.execute_reply":"2022-01-12T17:20:16.545346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.describe()","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"TTzh-7H0QuWG","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.547226Z","iopub.execute_input":"2022-01-12T17:20:16.547510Z","iopub.status.idle":"2022-01-12T17:20:16.554702Z","shell.execute_reply.started":"2022-01-12T17:20:16.547471Z","shell.execute_reply":"2022-01-12T17:20:16.554014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[:1000000]","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"aQdnyZEgQuWH","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.557129Z","iopub.execute_input":"2022-01-12T17:20:16.557779Z","iopub.status.idle":"2022-01-12T17:20:16.563617Z","shell.execute_reply.started":"2022-01-12T17:20:16.557743Z","shell.execute_reply":"2022-01-12T17:20:16.562885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VWAP_MAX =np.max(train[np.isfinite(train.VWAP)].VWAP)\nVWAP_MIN =np.min(train[np.isfinite(train.VWAP)].VWAP)\nprint(VWAP_MAX)\nprint(VWAP_MIN)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"vZiHAH10QuWH","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.565017Z","iopub.execute_input":"2022-01-12T17:20:16.565530Z","iopub.status.idle":"2022-01-12T17:20:16.653018Z","shell.execute_reply.started":"2022-01-12T17:20:16.565495Z","shell.execute_reply":"2022-01-12T17:20:16.652301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_VWAP = np.nan_to_num(train.VWAP, posinf=VWAP_MAX, neginf=VWAP_MIN)\ndel train['VWAP']\ntrain['VWAP'] = tmp_VWAP\ntrain.shape","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"rZxgJzIOQuWI","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.654519Z","iopub.execute_input":"2022-01-12T17:20:16.654785Z","iopub.status.idle":"2022-01-12T17:20:16.672077Z","shell.execute_reply.started":"2022-01-12T17:20:16.654749Z","shell.execute_reply":"2022-01-12T17:20:16.671254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"Data NULL: \\n\", train.isnull().sum())","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"WPh1S5voQuWJ","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.674279Z","iopub.execute_input":"2022-01-12T17:20:16.674557Z","iopub.status.idle":"2022-01-12T17:20:16.677695Z","shell.execute_reply.started":"2022-01-12T17:20:16.674521Z","shell.execute_reply":"2022-01-12T17:20:16.677037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Target'] = train['Target'].fillna(0)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"uug6ZQ58QuWJ","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.679136Z","iopub.execute_input":"2022-01-12T17:20:16.679602Z","iopub.status.idle":"2022-01-12T17:20:16.694943Z","shell.execute_reply.started":"2022-01-12T17:20:16.679566Z","shell.execute_reply":"2022-01-12T17:20:16.694285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train[['Asset_ID','Target']].copy()\ntime = dict((i,j) for j,i in enumerate(df.index.unique()))\ndf['id'] = df.index.map(time)\n# df","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"3g2YWWe8QuWK","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.696368Z","iopub.execute_input":"2022-01-12T17:20:16.696603Z","iopub.status.idle":"2022-01-12T17:20:16.957127Z","shell.execute_reply.started":"2022-01-12T17:20:16.696572Z","shell.execute_reply":"2022-01-12T17:20:16.956371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\n# df","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"fOku6R7AQuWK","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:16.958633Z","iopub.execute_input":"2022-01-12T17:20:16.958896Z","iopub.status.idle":"2022-01-12T17:20:19.734175Z","shell.execute_reply.started":"2022-01-12T17:20:16.958858Z","shell.execute_reply":"2022-01-12T17:20:19.733379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = df.id.copy()\ndel df","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"CwkQ0vepQuWK","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:19.735452Z","iopub.execute_input":"2022-01-12T17:20:19.735725Z","iopub.status.idle":"2022-01-12T17:20:19.754577Z","shell.execute_reply.started":"2022-01-12T17:20:19.735687Z","shell.execute_reply":"2022-01-12T17:20:19.753645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(df):\n    df['upper'] = df['High'] - np.maximum(df['Close'],df['Open'])\n    df['lower'] = np.minimum(df['Close'],df['Open'])-df['Low']\n    df['range'] = df['High'] - df['Low']\n    df['mean_trade'] = df['Volume']/df['Count']\n    df['log_price_change'] = np.log(df['Close']/df['Open'])\n    return df\ntrain=add_features(train)\ntrain.shape","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"C9c5GicgQuWL","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:19.756076Z","iopub.execute_input":"2022-01-12T17:20:19.756475Z","iopub.status.idle":"2022-01-12T17:20:19.804738Z","shell.execute_reply.started":"2022-01-12T17:20:19.756433Z","shell.execute_reply":"2022-01-12T17:20:19.803915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.head()","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"ioC1VzS8QuWL","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:19.808798Z","iopub.execute_input":"2022-01-12T17:20:19.810506Z","iopub.status.idle":"2022-01-12T17:20:19.814300Z","shell.execute_reply.started":"2022-01-12T17:20:19.810457Z","shell.execute_reply":"2022-01-12T17:20:19.813341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = train.columns.drop(['Asset_ID','Target'])\ntrain[features] = RobustScaler().fit_transform(train[features])\ntrain.head()","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"O9fkPAM_QuWL","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:19.815998Z","iopub.execute_input":"2022-01-12T17:20:19.816446Z","iopub.status.idle":"2022-01-12T17:20:20.310797Z","shell.execute_reply.started":"2022-01-12T17:20:19.816404Z","shell.execute_reply":"2022-01-12T17:20:20.310136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = train.index.unique()\n# for i in range(index[0],index[-1]+60,60):\n#     print(i)\ndef reindex(df):\n    df = df.reindex(range(index[0],index[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\ntrain=train.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\ntrain.shape","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"vVNBcbJ9QuWM","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:20.312264Z","iopub.execute_input":"2022-01-12T17:20:20.312749Z","iopub.status.idle":"2022-01-12T17:20:20.871131Z","shell.execute_reply.started":"2022-01-12T17:20:20.312701Z","shell.execute_reply":"2022-01-12T17:20:20.870295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  把group_num and row_id 合併起來\ntrain['group_num'] = train.index.map(time)\ntrain = train.dropna(subset=['group_num'])\ntrain['group_num'] = train['group_num'].astype('int')\n\n# 標記產生出來的row為non-real\ntrain['id'] = train['group_num'].astype(str) + '_' + train['Asset_ID'].astype(str)\ntrain['is_real'] = train.id.isin(ids)*1\ntrain = train.drop('id', axis=1)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"W5PXFO3tQuWM","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:20.872479Z","iopub.execute_input":"2022-01-12T17:20:20.872792Z","iopub.status.idle":"2022-01-12T17:20:24.861176Z","shell.execute_reply.started":"2022-01-12T17:20:20.872753Z","shell.execute_reply":"2022-01-12T17:20:24.860440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# non_real 的設為0\nfeatures = train.columns.drop(['Asset_ID','group_num','is_real'])\ntrain.loc[train.is_real==0, features]=0","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"n-pjeT7NQuWM","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:24.862516Z","iopub.execute_input":"2022-01-12T17:20:24.862767Z","iopub.status.idle":"2022-01-12T17:20:24.937819Z","shell.execute_reply.started":"2022-01-12T17:20:24.862734Z","shell.execute_reply":"2022-01-12T17:20:24.937091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 將'supplemental_train.csv' 中的asset_order合併進來\ntrain['asset_order'] = train.Asset_ID.map(assets_order)\ntrain=train.sort_values(by=['group_num', 'asset_order'])\ntrain.head(5)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"Guxbwfh0QuWN","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:24.939340Z","iopub.execute_input":"2022-01-12T17:20:24.939596Z","iopub.status.idle":"2022-01-12T17:20:25.238835Z","shell.execute_reply.started":"2022-01-12T17:20:24.939562Z","shell.execute_reply":"2022-01-12T17:20:25.238115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.shape","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"56Cu_6tRQuWN","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:25.240065Z","iopub.execute_input":"2022-01-12T17:20:25.241776Z","iopub.status.idle":"2022-01-12T17:20:25.244538Z","shell.execute_reply.started":"2022-01-12T17:20:25.241746Z","shell.execute_reply":"2022-01-12T17:20:25.243894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train.Target.copy()\ny_train.shape","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"uoY5TDF6QuWP","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:25.245846Z","iopub.execute_input":"2022-01-12T17:20:25.246245Z","iopub.status.idle":"2022-01-12T17:20:25.258934Z","shell.execute_reply.started":"2022-01-12T17:20:25.246206Z","shell.execute_reply":"2022-01-12T17:20:25.258108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train.drop(['Target'],axis = 1)\nprint(x_train.shape)\nprint(x_train.head(5))","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"9EQikdAeQuWP","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:25.260785Z","iopub.execute_input":"2022-01-12T17:20:25.261522Z","iopub.status.idle":"2022-01-12T17:20:25.339219Z","shell.execute_reply.started":"2022-01-12T17:20:25.261483Z","shell.execute_reply":"2022-01-12T17:20:25.338513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.to_numpy().reshape(-1,16,1)\nx_train.shape","metadata":{"id":"9gtsqH3EQuWP","execution":{"iopub.status.busy":"2022-01-12T17:20:25.340436Z","iopub.execute_input":"2022-01-12T17:20:25.340707Z","iopub.status.idle":"2022-01-12T17:20:25.400886Z","shell.execute_reply.started":"2022-01-12T17:20:25.340671Z","shell.execute_reply":"2022-01-12T17:20:25.400149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x, val_x, train_y, val_y = train_test_split(x_train, y_train, test_size = 0.25, random_state=2)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"Sr3VoEh-QuWP","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:25.402176Z","iopub.execute_input":"2022-01-12T17:20:25.402630Z","iopub.status.idle":"2022-01-12T17:20:25.697764Z","shell.execute_reply.started":"2022-01-12T17:20:25.402590Z","shell.execute_reply":"2022-01-12T17:20:25.696939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = keras.models.Sequential([\n#       keras.layers.LSTM(64,return_sequences=True,input_shape=(16,1)),\n#       keras.layers.ReLU(),\n#       keras.layers.Dropout(0.2),\n#       # keras.layers.LSTM(40,return_sequences=True, activation='relu'),\n#       # keras.layers.Dropout(0.5),\n#       keras.layers.LSTM(32),\n\n#       keras.layers.Dropout(0.5),\n#       keras.layers.Dense(1,activation = 'linear' )#activation = \"linear\"\n# ])\n# model.summary()","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"LudBQL3dQuWQ","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:25.699336Z","iopub.execute_input":"2022-01-12T17:20:25.699831Z","iopub.status.idle":"2022-01-12T17:20:25.703841Z","shell.execute_reply.started":"2022-01-12T17:20:25.699791Z","shell.execute_reply":"2022-01-12T17:20:25.702795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.layers import Dense, LSTM, Dropout, GRU\nfrom tensorflow.keras.optimizers import RMSprop\n# The GRU architecture\nregressorGRU = Sequential()\n# First GRU layer with Dropout regularisation\nregressorGRU.add(GRU(units=50, return_sequences=True, input_shape=(train_x.shape[1],1), activation='tanh'))\nregressorGRU.add(Dropout(0.2))\n# Second GRU layer\nregressorGRU.add(GRU(units=50, return_sequences=True, input_shape=(train_x.shape[1],1), activation='tanh'))\nregressorGRU.add(Dropout(0.2))\n# Third GRU layer\nregressorGRU.add(GRU(units=50, return_sequences=True, input_shape=(train_x.shape[1],1), activation='tanh'))\nregressorGRU.add(Dropout(0.2))\n# Fourth GRU layer\nregressorGRU.add(GRU(units=50, activation='tanh'))\nregressorGRU.add(Dropout(0.2))\n# The output layer\nregressorGRU.add(Dense(units=1))","metadata":{"execution":{"iopub.status.busy":"2022-01-12T17:20:25.705401Z","iopub.execute_input":"2022-01-12T17:20:25.705895Z","iopub.status.idle":"2022-01-12T17:20:29.289429Z","shell.execute_reply.started":"2022-01-12T17:20:25.705854Z","shell.execute_reply":"2022-01-12T17:20:29.288692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# regressorGRU.compile(optimizer=SGD(lr=0.001, decay=1e-7, momentum=0.9, nesterov=False),loss='mse', metrics = ['mae'])\nregressorGRU.compile(optimizer=RMSprop(lr=0.001),loss='mse', metrics = ['mae'])\nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",mode=\"min\",patience=5)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"m4M0ZzZWQuWQ","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:29.290865Z","iopub.execute_input":"2022-01-12T17:20:29.291104Z","iopub.status.idle":"2022-01-12T17:20:29.309759Z","shell.execute_reply.started":"2022-01-12T17:20:29.291072Z","shell.execute_reply":"2022-01-12T17:20:29.308921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gru = regressorGRU.fit(x_train,y_train, batch_size=64,epochs=5,validation_data=(val_x,val_y), callbacks=[early_stopping], shuffle=True)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"l6Gsq3QBQuWQ","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:20:29.311512Z","iopub.execute_input":"2022-01-12T17:20:29.312140Z","iopub.status.idle":"2022-01-12T17:36:55.999470Z","shell.execute_reply.started":"2022-01-12T17:20:29.312101Z","shell.execute_reply":"2022-01-12T17:36:55.998487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## test data set","metadata":{"pycharm":{"name":"#%% md\n"},"id":"08PRREpaQuWQ"}},{"cell_type":"code","source":"test = pd.read_csv(dir_path + 'example_test.csv').set_index('timestamp')\nRow_Id = test['row_id']\nGroup_Num = test['group_num']","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"CipUxVKAQuWQ","executionInfo":{"status":"error","timestamp":1641956988402,"user_tz":-480,"elapsed":436,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"outputId":"82c0fab0-19bd-48e0-c763-ba19bd79757c","collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.001237Z","iopub.execute_input":"2022-01-12T17:36:56.001606Z","iopub.status.idle":"2022-01-12T17:36:56.017666Z","shell.execute_reply.started":"2022-01-12T17:36:56.001566Z","shell.execute_reply":"2022-01-12T17:36:56.016869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VWAP_MAX =np.max(test[np.isfinite(test.VWAP)].VWAP)\nVWAP_MIN =np.min(test[np.isfinite(test.VWAP)].VWAP)\ntmp_VWAP = np.nan_to_num(test.VWAP, posinf=VWAP_MAX, neginf=VWAP_MIN)\ndel test['VWAP']\ntest['VWAP'] = tmp_VWAP\ntest.shape","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"DENzu5gfQuWR","executionInfo":{"status":"aborted","timestamp":1641956988394,"user_tz":-480,"elapsed":9,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.018948Z","iopub.execute_input":"2022-01-12T17:36:56.019192Z","iopub.status.idle":"2022-01-12T17:36:56.030830Z","shell.execute_reply.started":"2022-01-12T17:36:56.019157Z","shell.execute_reply":"2022-01-12T17:36:56.029996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test[['Asset_ID']].copy()\ntime = dict((i,j) for j,i in enumerate(df.index.unique()))\n# df['id'] = df.index.map(time)\n#\n# df['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\n# ids = df.id.copy()\n# del df","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"zB4tjS4XQuWR","executionInfo":{"status":"aborted","timestamp":1641956988396,"user_tz":-480,"elapsed":11,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.032357Z","iopub.execute_input":"2022-01-12T17:36:56.032605Z","iopub.status.idle":"2022-01-12T17:36:56.042921Z","shell.execute_reply.started":"2022-01-12T17:36:56.032571Z","shell.execute_reply":"2022-01-12T17:36:56.042154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features_test(df):\n    df['upper'] = df['High'] - np.maximum(df['Close'],df['Open'])\n    df['lower'] = np.minimum(df['Close'],df['Open'])-df['Low']\n    df['range'] = df['High'] - df['Low']\n    df['mean_trade'] = df['Volume']/df['Count']\n    df['log_price_change'] = np.log(df['Close']/df['Open'])\n    return df\ntest=add_features_test(test)\ntest.shape","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"Ys9NTB8MQuWR","executionInfo":{"status":"aborted","timestamp":1641956988396,"user_tz":-480,"elapsed":10,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.044013Z","iopub.execute_input":"2022-01-12T17:36:56.044411Z","iopub.status.idle":"2022-01-12T17:36:56.060131Z","shell.execute_reply.started":"2022-01-12T17:36:56.044374Z","shell.execute_reply":"2022-01-12T17:36:56.059431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = test.columns.drop(['Asset_ID','row_id','group_num'])\ntest[features] = RobustScaler().fit_transform(test[features])","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"VqwHSFGWQuWR","executionInfo":{"status":"aborted","timestamp":1641956988397,"user_tz":-480,"elapsed":11,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.062707Z","iopub.execute_input":"2022-01-12T17:36:56.062943Z","iopub.status.idle":"2022-01-12T17:36:56.078000Z","shell.execute_reply.started":"2022-01-12T17:36:56.062910Z","shell.execute_reply":"2022-01-12T17:36:56.077365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = test.index.unique()\ndef reindex_test(df):\n    df = df.reindex(range(index[0],index[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\ntest=test.groupby('Asset_ID').apply(reindex_test).reset_index(0, drop=True).sort_index()\ntest.shape","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"YlF6tZOqQuWR","executionInfo":{"status":"aborted","timestamp":1641956988397,"user_tz":-480,"elapsed":11,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.079392Z","iopub.execute_input":"2022-01-12T17:36:56.079686Z","iopub.status.idle":"2022-01-12T17:36:56.104762Z","shell.execute_reply.started":"2022-01-12T17:36:56.079611Z","shell.execute_reply":"2022-01-12T17:36:56.104010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  把group_num and row_id 合併起來\ntest['group_num'] = test.index.map(time)\ntest = test.dropna(subset=['group_num'])\ntest['group_num'] = test['group_num'].astype('int')\n\n# 標記產生出來的row為non-real\ntest['id'] = test['group_num'].astype(str) + '_' + test['Asset_ID'].astype(str)\ntest['is_real'] = test.id.isin(ids)*1\ntest = test.drop('id', axis=1)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"RrtxRyh4QuWR","executionInfo":{"status":"aborted","timestamp":1641956988397,"user_tz":-480,"elapsed":11,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.106110Z","iopub.execute_input":"2022-01-12T17:36:56.106372Z","iopub.status.idle":"2022-01-12T17:36:56.435054Z","shell.execute_reply.started":"2022-01-12T17:36:56.106340Z","shell.execute_reply":"2022-01-12T17:36:56.434266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# non_real 的設為0\nfeatures_test = test.columns.drop(['Asset_ID','group_num','is_real'])\ntest.loc[test.is_real==0, features_test]=0","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"rKJ0ZUtsQuWS","executionInfo":{"status":"aborted","timestamp":1641956988398,"user_tz":-480,"elapsed":12,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.436568Z","iopub.execute_input":"2022-01-12T17:36:56.436846Z","iopub.status.idle":"2022-01-12T17:36:56.448569Z","shell.execute_reply.started":"2022-01-12T17:36:56.436809Z","shell.execute_reply":"2022-01-12T17:36:56.447787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['asset_order'] = test.Asset_ID.map(assets_order)\ntest=test.sort_values(by=['asset_order'])\nprint(test.shape)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"zqKY5JMRQuWS","executionInfo":{"status":"aborted","timestamp":1641956988398,"user_tz":-480,"elapsed":12,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.450049Z","iopub.execute_input":"2022-01-12T17:36:56.450339Z","iopub.status.idle":"2022-01-12T17:36:56.459593Z","shell.execute_reply.started":"2022-01-12T17:36:56.450292Z","shell.execute_reply":"2022-01-12T17:36:56.458751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(test.head(20))","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"G67eRI7OQuWS","executionInfo":{"status":"aborted","timestamp":1641956988399,"user_tz":-480,"elapsed":13,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.460953Z","iopub.execute_input":"2022-01-12T17:36:56.461205Z","iopub.status.idle":"2022-01-12T17:36:56.467447Z","shell.execute_reply.started":"2022-01-12T17:36:56.461173Z","shell.execute_reply":"2022-01-12T17:36:56.466671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test['row_id']\nprint(test.shape)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"bVizTy9-QuWS","executionInfo":{"status":"aborted","timestamp":1641956988399,"user_tz":-480,"elapsed":12,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.468888Z","iopub.execute_input":"2022-01-12T17:36:56.469187Z","iopub.status.idle":"2022-01-12T17:36:56.477177Z","shell.execute_reply.started":"2022-01-12T17:36:56.469152Z","shell.execute_reply":"2022-01-12T17:36:56.476431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.to_numpy().reshape(-1,16,1)\ntest.shape","metadata":{"id":"uSXSpQW_QuWS","executionInfo":{"status":"aborted","timestamp":1641956988399,"user_tz":-480,"elapsed":12,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"execution":{"iopub.status.busy":"2022-01-12T17:36:56.478569Z","iopub.execute_input":"2022-01-12T17:36:56.478827Z","iopub.status.idle":"2022-01-12T17:36:56.488562Z","shell.execute_reply.started":"2022-01-12T17:36:56.478793Z","shell.execute_reply":"2022-01-12T17:36:56.487873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_y = np.mean(regressorGRU.predict(test),axis =1)\ntest_y = pd.Series(test_y,name=\"sales\")\ntest_y","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"6c7iy8CEQuWS","executionInfo":{"status":"aborted","timestamp":1641956988400,"user_tz":-480,"elapsed":13,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:56.493711Z","iopub.execute_input":"2022-01-12T17:36:56.493907Z","iopub.status.idle":"2022-01-12T17:36:57.403916Z","shell.execute_reply.started":"2022-01-12T17:36:56.493884Z","shell.execute_reply":"2022-01-12T17:36:57.403220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(Group_Num)\n# print(Row_Id)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"C5jRZpsfQuWT","executionInfo":{"status":"aborted","timestamp":1641956988400,"user_tz":-480,"elapsed":13,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:57.405216Z","iopub.execute_input":"2022-01-12T17:36:57.405477Z","iopub.status.idle":"2022-01-12T17:36:57.408703Z","shell.execute_reply.started":"2022-01-12T17:36:57.405440Z","shell.execute_reply":"2022-01-12T17:36:57.407830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group_Num=Group_Num.reset_index(drop=True)\n# Group_Num","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"ktUQgZTqQuWT","executionInfo":{"status":"aborted","timestamp":1641956988400,"user_tz":-480,"elapsed":12,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:57.409919Z","iopub.execute_input":"2022-01-12T17:36:57.410297Z","iopub.status.idle":"2022-01-12T17:36:57.419454Z","shell.execute_reply.started":"2022-01-12T17:36:57.410263Z","shell.execute_reply":"2022-01-12T17:36:57.418793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Row_Id = Row_Id.reset_index(drop = True)\n# Row_Id","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"uHG0Nj4fQuWT","executionInfo":{"status":"aborted","timestamp":1641956988400,"user_tz":-480,"elapsed":12,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:57.422393Z","iopub.execute_input":"2022-01-12T17:36:57.422646Z","iopub.status.idle":"2022-01-12T17:36:57.428082Z","shell.execute_reply.started":"2022-01-12T17:36:57.422618Z","shell.execute_reply":"2022-01-12T17:36:57.427311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()   # initialize the environment\niter_test = env.iter_test()    # ","metadata":{"execution":{"iopub.status.busy":"2022-01-12T17:36:57.429227Z","iopub.execute_input":"2022-01-12T17:36:57.429632Z","iopub.status.idle":"2022-01-12T17:36:57.462341Z","shell.execute_reply.started":"2022-01-12T17:36:57.429598Z","shell.execute_reply":"2022-01-12T17:36:57.461586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(test_df,sample_prediction_df) = next(iter_test)\nsample_prediction_df['Target']= test_y\nenv.predict(sample_prediction_df)","metadata":{"pycharm":{"name":"#%%\n"},"jupyter":{"outputs_hidden":false},"id":"sLUj7WpBQuWT","executionInfo":{"status":"aborted","timestamp":1641956988401,"user_tz":-480,"elapsed":13,"user":{"displayName":"小柯","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"15784538541517491683"}},"collapsed":false,"execution":{"iopub.status.busy":"2022-01-12T17:36:57.463788Z","iopub.execute_input":"2022-01-12T17:36:57.464050Z","iopub.status.idle":"2022-01-12T17:36:57.480118Z","shell.execute_reply.started":"2022-01-12T17:36:57.464016Z","shell.execute_reply":"2022-01-12T17:36:57.479246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = test_y\n# print(predictions)\nsubmission.Target = predictions\nprint(submission)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-12T17:39:13.954091Z","iopub.execute_input":"2022-01-12T17:39:13.954395Z","iopub.status.idle":"2022-01-12T17:39:13.965673Z","shell.execute_reply.started":"2022-01-12T17:39:13.954356Z","shell.execute_reply":"2022-01-12T17:39:13.964762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}