{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import KFold\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_probability as tfp\nfrom sklearn.preprocessing import RobustScaler\nfrom tensorflow.keras.preprocessing.sequence import TimeseriesGenerator\nimport gresearch_crypto\nimport gc\n\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\n\npd.set_option('display.max_columns', None)\n\nDEBUG = True","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:03:27.232069Z","iopub.execute_input":"2022-08-13T04:03:27.232531Z","iopub.status.idle":"2022-08-13T04:03:36.245506Z","shell.execute_reply.started":"2022-08-13T04:03:27.232436Z","shell.execute_reply":"2022-08-13T04:03:36.244101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv').set_index(\"timestamp\")\nassets = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\n#for assets sorting \nassets_order = pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv').Asset_ID[:14]\nassets_order = dict((t,i) for i,t in enumerate(assets_order))\n\n# TODO : use more data?\nif DEBUG:\n    #train = train[1000000:12000000]\n    train = train[20000000:]\n\ntrain[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP','Target']] = \\\ntrain[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP','Target']].astype(np.float32)\n\ntrain['Target'] = train['Target'].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:03:36.252043Z","iopub.execute_input":"2022-08-13T04:03:36.252804Z","iopub.status.idle":"2022-08-13T04:04:39.355726Z","shell.execute_reply.started":"2022-08-13T04:03:36.252764Z","shell.execute_reply":"2022-08-13T04:04:39.354676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:39.357327Z","iopub.execute_input":"2022-08-13T04:04:39.357720Z","iopub.status.idle":"2022-08-13T04:04:39.369393Z","shell.execute_reply.started":"2022-08-13T04:04:39.357683Z","shell.execute_reply":"2022-08-13T04:04:39.367299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# VWAP column has -inf and inf values. VWAP_max and VWAP_min will be used for replacement\n    \nVWAP_max = np.max(train[np.isfinite(train.VWAP)].VWAP)\nVWAP_min = np.min(train[np.isfinite(train.VWAP)].VWAP)\nprint(VWAP_max, \"\\n\", VWAP_min)\n\ntrain['VWAP'] = np.nan_to_num(train.VWAP, posinf=VWAP_max, neginf=VWAP_min)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:39.372069Z","iopub.execute_input":"2022-08-13T04:04:39.372431Z","iopub.status.idle":"2022-08-13T04:04:39.926419Z","shell.execute_reply.started":"2022-08-13T04:04:39.372395Z","shell.execute_reply":"2022-08-13T04:04:39.925310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the series of the 'real' record ids for futher matching\n    \ndf = train[['Asset_ID', 'Target']].copy()\n\ntimes = dict((t,i) for i,t in enumerate(df.index.unique()))\ndf['id'] = df.index.map(times)\ndf['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\nids = df.id.copy()\n\ndel df","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:39.927869Z","iopub.execute_input":"2022-08-13T04:04:39.928478Z","iopub.status.idle":"2022-08-13T04:04:46.604022Z","shell.execute_reply.started":"2022-08-13T04:04:39.928438Z","shell.execute_reply":"2022-08-13T04:04:46.602989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO : add more features \n# moving average? exponential moving average? rsi?\n# https://technical-analysis-library-in-python.readthedocs.io/en/latest/\n\ndef add_features(df):\n    df['Upper_Shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n    df['Lower_Shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n    \n    df['spread'] = df['High'] - df['Low']\n    df['mean_trade'] = df['Volume']/df['Count']\n    df['log_price_change'] = np.log(df['Close']/df['Open'])\n    return df\n\ntrain=add_features(train)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:46.609092Z","iopub.execute_input":"2022-08-13T04:04:46.611986Z","iopub.status.idle":"2022-08-13T04:04:46.787551Z","shell.execute_reply.started":"2022-08-13T04:04:46.611948Z","shell.execute_reply":"2022-08-13T04:04:46.786554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scale_features = train.columns.drop(['Asset_ID','Target'])\n# TODO : use other scalers?\nRS = RobustScaler()\ntrain[scale_features] = RS.fit_transform(train[scale_features])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:46.789187Z","iopub.execute_input":"2022-08-13T04:04:46.789554Z","iopub.status.idle":"2022-08-13T04:04:48.349807Z","shell.execute_reply.started":"2022-08-13T04:04:46.789518Z","shell.execute_reply":"2022-08-13T04:04:48.348839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filling gaps\n\nind = train.index.unique()\n\ndef reindex(df):\n    df = df.reindex(range(ind[0],ind[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\n\ntrain=train.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:48.351298Z","iopub.execute_input":"2022-08-13T04:04:48.351664Z","iopub.status.idle":"2022-08-13T04:04:50.863816Z","shell.execute_reply.started":"2022-08-13T04:04:48.351628Z","shell.execute_reply":"2022-08-13T04:04:50.861203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Matching records and marking generated rows as 'non-real'\n\ntrain['group_num'] = train.index.map(times)\ntrain = train.dropna(subset=['group_num'])\ntrain['group_num'] = train['group_num'].astype('int')\n\ntrain['id'] = train['group_num'].astype(str) + '_' + train['Asset_ID'].astype(str)\n\ntrain['is_real'] = train.id.isin(ids)*1\ntrain = train.drop('id', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:50.866601Z","iopub.execute_input":"2022-08-13T04:04:50.867975Z","iopub.status.idle":"2022-08-13T04:04:59.660322Z","shell.execute_reply.started":"2022-08-13T04:04:50.867935Z","shell.execute_reply":"2022-08-13T04:04:59.659165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Features values for 'non-real' rows are set to zeros\n\nfeatures = train.columns.drop(['Asset_ID','group_num','is_real'])\ntrain.loc[train.is_real==0, features]=0.","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:59.665039Z","iopub.execute_input":"2022-08-13T04:04:59.665339Z","iopub.status.idle":"2022-08-13T04:04:59.808247Z","shell.execute_reply.started":"2022-08-13T04:04:59.665311Z","shell.execute_reply":"2022-08-13T04:04:59.807247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sorting assets according to their order in the 'supplemental_train.csv'\n\ntrain['asset_order'] = train.Asset_ID.map(assets_order) \ntrain=train.sort_values(by=['group_num', 'asset_order'])\ntrain.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:04:59.809561Z","iopub.execute_input":"2022-08-13T04:04:59.809936Z","iopub.status.idle":"2022-08-13T04:05:01.475575Z","shell.execute_reply.started":"2022-08-13T04:04:59.809902Z","shell.execute_reply":"2022-08-13T04:05:01.474625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Target'].to_numpy().shape ","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:05:01.476997Z","iopub.execute_input":"2022-08-13T04:05:01.477449Z","iopub.status.idle":"2022-08-13T04:05:01.484459Z","shell.execute_reply.started":"2022-08-13T04:05:01.477410Z","shell.execute_reply":"2022-08-13T04:05:01.483489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = train['Target'].to_numpy().reshape(-1, 14)\n#targets = np.expand_dims(targets, axis=1)\n\nfeatures = train.columns.drop(['Asset_ID', 'Target', 'group_num','is_real'])\ntrain = train[features]\n\ntrain=np.array(train)\ntrain = train.reshape(-1,14,train.shape[-1])\n#train = np.expand_dims(train, axis=1)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:05:01.486060Z","iopub.execute_input":"2022-08-13T04:05:01.486688Z","iopub.status.idle":"2022-08-13T04:05:01.951505Z","shell.execute_reply.started":"2022-08-13T04:05:01.486651Z","shell.execute_reply":"2022-08-13T04:05:01.950401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# timeseriesgenerator-like class, except it using target from the last timestep insteed of last+1\nclass sample_generator(keras.utils.Sequence):\n    def __init__(self, x_set, y_set, batch_size, length):\n        self.x, self.y = x_set, y_set\n        self.batch_size = batch_size\n        self.length = length\n        self.size = len(x_set)\n\n    def __len__(self):\n        return int(np.ceil(len(self.x) / float(self.batch_size)))\n\n    def __getitem__(self, idx):\n        batch_x=[]\n        batch_y=[]\n        for i in range(self.batch_size):\n            start_ind = self.batch_size*idx + i\n            end_ind = start_ind + self.length \n            if end_ind <= self.size:\n                batch_x.append(self.x[start_ind : end_ind])\n                batch_y.append(self.y[end_ind -1])\n\n        return np.array(batch_x), np.array(batch_y)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:05:01.952949Z","iopub.execute_input":"2022-08-13T04:05:01.953507Z","iopub.status.idle":"2022-08-13T04:05:01.962545Z","shell.execute_reply.started":"2022-08-13T04:05:01.953468Z","shell.execute_reply":"2022-08-13T04:05:01.961409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#last 10% of the data are used as validation set\nX_train, X_test = train[:-len(train)//10], train[-len(train)//10:]\ny_train, y_test = targets[:-len(train)//10], targets[-len(train)//10:]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:05:01.964172Z","iopub.execute_input":"2022-08-13T04:05:01.964752Z","iopub.status.idle":"2022-08-13T04:05:01.976398Z","shell.execute_reply.started":"2022-08-13T04:05:01.964715Z","shell.execute_reply":"2022-08-13T04:05:01.975396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE=2**10\ntrain_generator = sample_generator(X_train, y_train, length=15, batch_size=BATCH_SIZE)\nval_generator = sample_generator(X_test, y_test, length=15, batch_size=BATCH_SIZE)\n\nprint(f'Sample shape: {train_generator[0][0].shape}')\nprint(f'Target shape: {train_generator[0][1].shape}')","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:05:01.979154Z","iopub.execute_input":"2022-08-13T04:05:01.979832Z","iopub.status.idle":"2022-08-13T04:05:01.999788Z","shell.execute_reply.started":"2022-08-13T04:05:01.979796Z","shell.execute_reply":"2022-08-13T04:05:01.998646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://github.com/tensorflow/tensorflow/issues/37495\ndef MaxCorrelation(y_true,y_pred):\n    \"\"\"Goal is to maximize correlation between y_pred, y_true. Same as minimizing the negative.\"\"\"\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return -tf.math.abs(tfp.stats.correlation(y_true_masked,y_pred_masked, sample_axis=None, event_axis=None))\n\ndef Correlation(y_true,y_pred):\n    return tf.math.abs(tfp.stats.correlation(y_pred,y_true, sample_axis=None, event_axis=None))\n\ndef masked_mse(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.mean_squared_error(y_true = y_true_masked, y_pred = y_pred_masked)\n\ndef masked_mae(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.mean_absolute_error(y_true = y_true_masked, y_pred = y_pred_masked)\n\ndef masked_cosine(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.cosine_similarity(y_true_masked, y_pred_masked)\n\n# TODO : change model? \n\ndef get_model(n_assets=14):  \n    x_input = keras.Input(shape=(train_generator[0][0].shape[1], n_assets, train_generator[0][0].shape[-1]))\n\n    branch_outputs = []\n        \n    for i in range(n_assets):\n            # Slicing the ith asset:\n        a = layers.Lambda(lambda x: x[:,:, i])(x_input)\n        a = layers.Masking(mask_value=0.,)(a)\n        #a = layers.BatchNormalization()(a)\n        a = layers.LSTM(units=32, return_sequences=True)(a)\n        a = layers.LSTM(units=16)(a)\n        branch_outputs.append(a)\n    \n    x = layers.Concatenate()(branch_outputs)\n    x = layers.Dense(units=128)(x)\n    out = layers.Dense(units=n_assets)(x)\n    \n    model = keras.Model(inputs=x_input, outputs=out)\n    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3), \n                  #loss = 'mse',\n                  #loss = 'cosine_similarity',\n                  loss = masked_cosine,\n                  metrics=[Correlation]\n                 )\n    \n    return model \n    \nmodel=get_model()\n#model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:05:02.001502Z","iopub.execute_input":"2022-08-13T04:05:02.001875Z","iopub.status.idle":"2022-08-13T04:05:27.244474Z","shell.execute_reply.started":"2022-08-13T04:05:02.001841Z","shell.execute_reply":"2022-08-13T04:05:27.243387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#example with 3 assets for visibility\ntf.keras.utils.plot_model(get_model(n_assets=3), show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:42.761899Z","iopub.execute_input":"2022-08-13T04:09:42.762270Z","iopub.status.idle":"2022-08-13T04:09:48.371919Z","shell.execute_reply.started":"2022-08-13T04:09:42.762231Z","shell.execute_reply":"2022-08-13T04:09:48.370669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(10)\n\nestop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min',restore_best_weights=True)\nscheduler = keras.optimizers.schedules.ExponentialDecay(1e-3, (0.5*len(X_train)/BATCH_SIZE), 1e-3)\nlr = keras.callbacks.LearningRateScheduler(scheduler, verbose = 1)\n\n# TODO : train with more epochs?\n\nhistory = model.fit(train_generator, validation_data = (val_generator),\n              epochs = 10, callbacks = [lr])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:05:27.255740Z","iopub.execute_input":"2022-08-13T04:05:27.256513Z","iopub.status.idle":"2022-08-13T04:09:05.007374Z","shell.execute_reply.started":"2022-08-13T04:05:27.256476Z","shell.execute_reply":"2022-08-13T04:09:05.006302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, ax = plt.subplots(1,2, figsize=(16,8))\n\n# histories = pd.DataFrame(history.history)\n\n# epochs = list(range(1,len(histories)+1))\n# loss = histories['loss']\n# val_loss = histories['val_loss']\n# Correlation = histories['Correlation']\n# val_Correlation = histories['val_Correlation']\n\n# ax[0].plot(epochs, loss, label = 'Train Loss')\n# ax[0].plot(epochs, val_loss, label = 'Val Loss')\n# ax[0].set_title('Losses')\n# ax[0].set_xlabel('Epoch')\n# ax[0].legend(loc='upper right')\n\n# ax[1].plot(epochs, Correlation, label = 'Train Correlation')\n# ax[1].plot(epochs, val_Correlation, label = 'Val Correlation')\n# ax[1].set_title('Correlations')\n# ax[1].set_xlabel('Epoch')\n# ax[1].legend(loc='upper right')\n\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:05.010247Z","iopub.execute_input":"2022-08-13T04:09:05.010547Z","iopub.status.idle":"2022-08-13T04:09:05.016453Z","shell.execute_reply.started":"2022-08-13T04:09:05.010520Z","shell.execute_reply":"2022-08-13T04:09:05.015503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:05.017944Z","iopub.execute_input":"2022-08-13T04:09:05.018640Z","iopub.status.idle":"2022-08-13T04:09:06.215734Z","shell.execute_reply.started":"2022-08-13T04:09:05.018586Z","shell.execute_reply":"2022-08-13T04:09:06.214690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(val_generator)\npredictions.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:06.217416Z","iopub.execute_input":"2022-08-13T04:09:06.218125Z","iopub.status.idle":"2022-08-13T04:09:36.330887Z","shell.execute_reply.started":"2022-08-13T04:09:06.218085Z","shell.execute_reply":"2022-08-13T04:09:36.329664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Asset:    Corr. coef.')\nprint('---------------------')\nfor i in range(14):\n    #drop first 14 values in the y_test, since they are absent in val_generator labels\n    y_true = np.squeeze(y_test[14:,i])\n    y_pred = np.squeeze(predictions[:,i])\n    \n    #get non-zero targets (assuming that zeros indicates 'non-real' rows)\n    real_target_ind = np.argwhere(y_true!=0)\n    asset_id = list(assets_order.keys())[i]\n    asset_name = assets[assets.Asset_ID == asset_id]['Asset_Name'].item()\n    print(f\"{asset_name}: {np.corrcoef(y_pred[real_target_ind].flatten(), y_true[real_target_ind].flatten())[0,1]:.4f}\") ","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:36.332986Z","iopub.execute_input":"2022-08-13T04:09:36.333425Z","iopub.status.idle":"2022-08-13T04:09:36.367216Z","shell.execute_reply.started":"2022-08-13T04:09:36.333383Z","shell.execute_reply":"2022-08-13T04:09:36.366148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=pd.read_csv('../input/10botics-crypto-prices/test.csv')\nplaceholder=add_features(test)\nplaceholder[scale_features] = RS.transform(np.nan_to_num(placeholder[scale_features]))\nplaceholder['asset_order'] = placeholder.Asset_ID.map(assets_order) ","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:36.369237Z","iopub.execute_input":"2022-08-13T04:09:36.369640Z","iopub.status.idle":"2022-08-13T04:09:37.350131Z","shell.execute_reply.started":"2022-08-13T04:09:36.369578Z","shell.execute_reply":"2022-08-13T04:09:37.348457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=pd.read_csv('../input/10botics-crypto-prices/test.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:37.351993Z","iopub.execute_input":"2022-08-13T04:09:37.352285Z","iopub.status.idle":"2022-08-13T04:09:37.774012Z","shell.execute_reply.started":"2022-08-13T04:09:37.352258Z","shell.execute_reply":"2022-08-13T04:09:37.772893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.drop_duplicates(\"timestamp\")","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:37.776042Z","iopub.execute_input":"2022-08-13T04:09:37.776737Z","iopub.status.idle":"2022-08-13T04:09:37.814083Z","shell.execute_reply.started":"2022-08-13T04:09:37.776671Z","shell.execute_reply":"2022-08-13T04:09:37.812676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df=pd.read_csv('../input/10botics-crypto-prices/test.csv').set_index('timestamp')\ntest_df['VWAP'] =np.nan_to_num(test_df.VWAP, posinf=VWAP_max, neginf=VWAP_min)\ntest_df=add_features(test_df)\ntest_df[scale_features] = RS.transform(np.nan_to_num(test_df[scale_features]))\ntest_df['asset_order'] = test_df.Asset_ID.map(assets_order) \n\n# Filling gaps\n\nind = test_df.index.unique()\n\ndef reindex(df):\n    df = df.reindex(range(ind[0],ind[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\n\ntest_df=test_df.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\n\ntest = np.array(test_df[features].fillna(0))\ntest = test.reshape(-1,14,test.shape[-1])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:37.816455Z","iopub.execute_input":"2022-08-13T04:09:37.817120Z","iopub.status.idle":"2022-08-13T04:09:38.963948Z","shell.execute_reply.started":"2022-08-13T04:09:37.817055Z","shell.execute_reply":"2022-08-13T04:09:38.962776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator = sample_generator(test, np.zeros(len(test_df)).reshape(-1, 14), length=15, batch_size=BATCH_SIZE)\ny_pred = model.predict(test_generator).reshape(-1, 1).squeeze()\ntest_df['pred'] = list(y_pred) + [0] * 14 * 14\ntest_df = test_df.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:39.005555Z","iopub.execute_input":"2022-08-13T04:09:39.006113Z","iopub.status.idle":"2022-08-13T04:09:39.013257Z","shell.execute_reply.started":"2022-08-13T04:09:39.006075Z","shell.execute_reply":"2022-08-13T04:09:39.012013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/10botics-crypto-prices/test.csv\")\nsub = sub.merge(test_df[['timestamp', 'Asset_ID', 'pred']].reset_index(drop=True).drop_duplicates(['timestamp', 'Asset_ID']), how='left', on=['timestamp', 'Asset_ID'])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:40.911453Z","iopub.execute_input":"2022-08-13T04:09:40.911829Z","iopub.status.idle":"2022-08-13T04:09:41.453030Z","shell.execute_reply.started":"2022-08-13T04:09:40.911794Z","shell.execute_reply":"2022-08-13T04:09:41.452028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = sub.reset_index()\nsub = sub[['index', 'pred']]\nsub.columns = ['id', 'target']","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:41.454577Z","iopub.execute_input":"2022-08-13T04:09:41.455188Z","iopub.status.idle":"2022-08-13T04:09:41.495929Z","shell.execute_reply.started":"2022-08-13T04:09:41.455142Z","shell.execute_reply":"2022-08-13T04:09:41.494954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv(\"submission1.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T04:09:41.497478Z","iopub.execute_input":"2022-08-13T04:09:41.497964Z","iopub.status.idle":"2022-08-13T04:09:42.760453Z","shell.execute_reply.started":"2022-08-13T04:09:41.497924Z","shell.execute_reply":"2022-08-13T04:09:42.759464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}