{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-03T19:07:57.702711Z","iopub.execute_input":"2022-01-03T19:07:57.703593Z","iopub.status.idle":"2022-01-03T19:07:57.739479Z","shell.execute_reply.started":"2022-01-03T19:07:57.70347Z","shell.execute_reply":"2022-01-03T19:07:57.73829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport traceback\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport gresearch_crypto\nimport tensorflow as tf\nfrom tensorflow import keras\nimport matplotlib.pyplot as plt\nimport pandas as pd, numpy as np\nfrom tensorflow.keras import layers\nimport tensorflow_probability as tfp\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import RobustScaler\nfrom tensorflow.keras.preprocessing.sequence import TimeseriesGenerator\npd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:07:57.988812Z","iopub.execute_input":"2022-01-03T19:07:57.989063Z","iopub.status.idle":"2022-01-03T19:08:03.790387Z","shell.execute_reply.started":"2022-01-03T19:07:57.989035Z","shell.execute_reply":"2022-01-03T19:08:03.78963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:08:03.792006Z","iopub.execute_input":"2022-01-03T19:08:03.79226Z","iopub.status.idle":"2022-01-03T19:08:53.628069Z","shell.execute_reply.started":"2022-01-03T19:08:03.792223Z","shell.execute_reply":"2022-01-03T19:08:53.627229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impute Missing Time Value\n\"\"\"def c_time_sub(asset_id, data=ctrain):\n    df=data[ctrain[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n    df=df.reindex(range(df.index[0],df.index[-1]+60,60), method=\"pad\")\n    return df\"\"\"\n\ntrain_df.index=train_df['timestamp']\ntrain = train_df.sort_index()\nind = train.index.unique()\ndef reindex(df):\n    df = df.reindex(range(ind[0],ind[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\ntrain = train.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:08:53.629443Z","iopub.execute_input":"2022-01-03T19:08:53.629719Z","iopub.status.idle":"2022-01-03T19:09:11.672234Z","shell.execute_reply.started":"2022-01-03T19:08:53.629666Z","shell.execute_reply":"2022-01-03T19:09:11.671541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.reset_index(drop = True)\ntrain[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Target']] = train[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Target']].astype(np.float32)\ntrain['Target'] = train['Target'].fillna(0)\n\nVWAP_max = np.max(train[np.isfinite(train.VWAP)].VWAP)\nVWAP_min = np.min(train[np.isfinite(train.VWAP)].VWAP)\ntrain['VWAP'] = np.nan_to_num(train.VWAP, posinf=VWAP_max, neginf=VWAP_min)\ndf = train[['Asset_ID', 'Target']].copy()\n\ntimes = dict((t,i) for i,t in enumerate(df.index.unique()))\n\ndf['id'] = df.index.map(times)\ndf['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\nids = df.id.copy()\ndel df","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:09:11.674349Z","iopub.execute_input":"2022-01-03T19:09:11.674634Z","iopub.status.idle":"2022-01-03T19:11:04.562068Z","shell.execute_reply.started":"2022-01-03T19:09:11.674597Z","shell.execute_reply":"2022-01-03T19:11:04.559477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features = [i for i in train.columns if i not in ['Target', 'date', 'timestamp', 'Asset_ID', 'groups']]\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:04.566636Z","iopub.execute_input":"2022-01-03T19:11:04.569872Z","iopub.status.idle":"2022-01-03T19:11:04.602815Z","shell.execute_reply.started":"2022-01-03T19:11:04.569829Z","shell.execute_reply":"2022-01-03T19:11:04.602186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:04.606497Z","iopub.execute_input":"2022-01-03T19:11:04.608394Z","iopub.status.idle":"2022-01-03T19:11:04.617635Z","shell.execute_reply.started":"2022-01-03T19:11:04.608344Z","shell.execute_reply":"2022-01-03T19:11:04.616771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Target'] = train['Target'].fillna(0)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:04.622206Z","iopub.execute_input":"2022-01-03T19:11:04.624117Z","iopub.status.idle":"2022-01-03T19:11:04.776717Z","shell.execute_reply.started":"2022-01-03T19:11:04.624079Z","shell.execute_reply":"2022-01-03T19:11:04.775907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"GPU\" #or \"GPU\"\n\nSEED = 42\n\nEPOCHS = 10\nDEBUG = True\nN_ASSETS = 1\nWINDOW_SIZE = 15\nBATCH_SIZE = 1024\nPCT_VALIDATION = 10 # last 10% of the data are used as validation set","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:04.778229Z","iopub.execute_input":"2022-01-03T19:11:04.778495Z","iopub.status.idle":"2022-01-03T19:11:04.783273Z","shell.execute_reply.started":"2022-01-03T19:11:04.778459Z","shell.execute_reply":"2022-01-03T19:11:04.782284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_df.copy()\n\ntargets = train['Target'].to_numpy().reshape(-1, N_ASSETS)\nfeatures = train.columns.drop(['Asset_ID', 'Target','timestamp'])\ntrain = train[features]\ntrain = train.values\ntrain = train.reshape(-1, N_ASSETS, train.shape[-1])","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:04.785113Z","iopub.execute_input":"2022-01-03T19:11:04.785748Z","iopub.status.idle":"2022-01-03T19:11:05.917116Z","shell.execute_reply.started":"2022-01-03T19:11:04.785708Z","shell.execute_reply":"2022-01-03T19:11:05.916314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class sample_generator(keras.utils.Sequence):\n    def __init__(self, x_set, y_set, batch_size, length):\n        self.x, self.y = x_set, y_set\n        self.batch_size = batch_size\n        self.length = length\n        self.size = len(x_set)\n    def __len__(self): return int(np.ceil(len(self.x) / float(self.batch_size)))\n    def __getitem__(self, idx):\n        batch_x=[]\n        batch_y=[]\n        for i in range(self.batch_size):\n            start_ind = self.batch_size*idx + i\n            end_ind = start_ind + self.length \n            if end_ind <= self.size:\n                batch_x.append(self.x[start_ind : end_ind])\n                batch_y.append(self.y[end_ind -1])\n        return np.array(batch_x), np.array(batch_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:05.92088Z","iopub.execute_input":"2022-01-03T19:11:05.9211Z","iopub.status.idle":"2022-01-03T19:11:05.928836Z","shell.execute_reply.started":"2022-01-03T19:11:05.921073Z","shell.execute_reply":"2022-01-03T19:11:05.927839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test = train[:-len(train)//PCT_VALIDATION], train[-len(train)//PCT_VALIDATION:]\ny_train, y_test = targets[:-len(train)//PCT_VALIDATION], targets[-len(train)//PCT_VALIDATION:]","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:05.930534Z","iopub.execute_input":"2022-01-03T19:11:05.930818Z","iopub.status.idle":"2022-01-03T19:11:05.938879Z","shell.execute_reply.started":"2022-01-03T19:11:05.930783Z","shell.execute_reply":"2022-01-03T19:11:05.938015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = sample_generator(X_train, y_train, length = WINDOW_SIZE, batch_size = BATCH_SIZE)\nval_generator = sample_generator(X_test, y_test, length = WINDOW_SIZE, batch_size = BATCH_SIZE)\nprint(f'Sample shape: {train_generator[0][0].shape}')\nprint(f'Target shape: {train_generator[0][1].shape}')","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:05.940277Z","iopub.execute_input":"2022-01-03T19:11:05.940729Z","iopub.status.idle":"2022-01-03T19:11:05.957731Z","shell.execute_reply.started":"2022-01-03T19:11:05.940675Z","shell.execute_reply":"2022-01-03T19:11:05.956878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Correlations for predicted and real\ndef MaxCorrelation(y_true,y_pred): \n    return -tf.math.abs(tfp.stats.correlation(y_pred,y_true, sample_axis=None, event_axis=None))\ndef Correlation(y_true,y_pred): \n    return tf.math.abs(tfp.stats.correlation(y_pred,y_true, sample_axis=None, event_axis=None))\n\n#Masked losses\ndef masked_mse(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.mean_squared_error(y_true = y_true_masked, y_pred = y_pred_masked)\n\n\ndef masked_mae(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.mean_absolute_error(y_true = y_true_masked, y_pred = y_pred_masked)\n\ndef masked_cosine(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.cosine_similarity(y_true_masked, y_pred_masked)","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:05.958981Z","iopub.execute_input":"2022-01-03T19:11:05.959216Z","iopub.status.idle":"2022-01-03T19:11:05.969768Z","shell.execute_reply.started":"2022-01-03T19:11:05.959184Z","shell.execute_reply":"2022-01-03T19:11:05.967935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Long Short-Term Memory layer\ndef get_squence_model(x):\n    x = layers.LSTM(units=32, return_sequences=True)(x)\n    return x\n\n#Model\ndef get_model(n_assets = N_ASSETS):\n    x_input = keras.Input(shape=(train_generator[0][0].shape[1], n_assets, train_generator[0][0].shape[-1]))\n    branch_outputs = []\n    for i in range(n_assets):\n        a = layers.Lambda(lambda x: x[:,:, i])(x_input) # Slicing the ith asset:\n        a = layers.Masking(mask_value = 0., )(a)\n        a = get_squence_model(a)\n        a = layers.GlobalAvgPool1D()(a)\n        branch_outputs.append(a)\n    x = layers.Concatenate()(branch_outputs)\n    x = layers.Dense(units = 128)(x)\n    out = layers.Dense(units = n_assets)(x)\n    model = keras.Model(inputs=x_input, outputs=out)\n    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3), loss = masked_cosine, metrics=[Correlation])\n    return model\n\nmodel = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:05.971067Z","iopub.execute_input":"2022-01-03T19:11:05.971656Z","iopub.status.idle":"2022-01-03T19:11:09.195404Z","shell.execute_reply.started":"2022-01-03T19:11:05.971617Z","shell.execute_reply":"2022-01-03T19:11:09.194534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(get_model(n_assets=1), show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:09.197138Z","iopub.execute_input":"2022-01-03T19:11:09.19742Z","iopub.status.idle":"2022-01-03T19:11:10.900699Z","shell.execute_reply.started":"2022-01-03T19:11:09.197381Z","shell.execute_reply":"2022-01-03T19:11:10.899772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(0)\nestop = keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 7, verbose = 0, mode = 'min',restore_best_weights = True)\nscheduler = keras.optimizers.schedules.ExponentialDecay(1e-3, (0.5 * len(X_train) / BATCH_SIZE), 1e-3)\nlr = keras.callbacks.LearningRateScheduler(scheduler, verbose = 1)\n\nhistory = model.fit(train_generator, validation_data = (val_generator), epochs = EPOCHS, callbacks = [lr, estop])\n","metadata":{"execution":{"iopub.status.busy":"2022-01-03T19:11:10.902471Z","iopub.execute_input":"2022-01-03T19:11:10.902901Z","iopub.status.idle":"2022-01-03T19:40:52.580357Z","shell.execute_reply.started":"2022-01-03T19:11:10.902853Z","shell.execute_reply":"2022-01-03T19:40:52.579603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history","metadata":{"execution":{"iopub.status.busy":"2022-01-03T20:16:57.401944Z","iopub.execute_input":"2022-01-03T20:16:57.402223Z","iopub.status.idle":"2022-01-03T20:16:57.407937Z","shell.execute_reply.started":"2022-01-03T20:16:57.402171Z","shell.execute_reply":"2022-01-03T20:16:57.407094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport seaborn as sns\nfrom statsmodels.tsa.arima.model import ARIMA\nfrom statsmodels.tsa.statespace.sarimax import SARIMAX","metadata":{"execution":{"iopub.status.busy":"2022-01-03T20:15:10.815619Z","iopub.execute_input":"2022-01-03T20:15:10.815887Z","iopub.status.idle":"2022-01-03T20:15:11.846447Z","shell.execute_reply.started":"2022-01-03T20:15:10.815857Z","shell.execute_reply":"2022-01-03T20:15:11.845733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.","metadata":{"execution":{"iopub.status.busy":"2022-01-03T20:17:17.077755Z","iopub.execute_input":"2022-01-03T20:17:17.078016Z","iopub.status.idle":"2022-01-03T20:17:17.097505Z","shell.execute_reply.started":"2022-01-03T20:17:17.077986Z","shell.execute_reply":"2022-01-03T20:17:17.095843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(16, 8))\nhistories = pd.DataFrame(history.history)\nepochs = list(range(1,len(histories)+1))\nloss = histories['loss']\nval_loss = histories['val_loss']\nCorrelation = histories['Correlation']\nval_Correlation = histories['val_Correlation']\nax[0].plot(epochs, loss, label = 'Train Loss')\nax[0].plot(epochs, val_loss, label = 'Val Loss')\nax[0].set_title('Losses')\nax[0].set_xlabel('Epoch')\nax[0].legend(loc='upper right')\nax[1].plot(epochs, Correlation, label = 'Train Correlation')\nax[1].plot(epochs, val_Correlation, label = 'Val Correlation')\nax[1].set_title('Correlations')\nax[1].set_xlabel('Epoch')\nax[1].legend(loc='upper right')\nfig.show()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-03T20:17:05.491024Z","iopub.execute_input":"2022-01-03T20:17:05.491785Z","iopub.status.idle":"2022-01-03T20:17:05.779982Z","shell.execute_reply.started":"2022-01-03T20:17:05.491732Z","shell.execute_reply":"2022-01-03T20:17:05.779043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-03T20:03:42.410986Z","iopub.status.idle":"2022-01-03T20:03:42.41325Z","shell.execute_reply.started":"2022-01-03T20:03:42.412999Z","shell.execute_reply":"2022-01-03T20:03:42.413025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-03T20:03:42.416705Z","iopub.status.idle":"2022-01-03T20:03:42.418749Z","shell.execute_reply.started":"2022-01-03T20:03:42.418513Z","shell.execute_reply":"2022-01-03T20:03:42.418539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df = get_features(test_df)\n    test_data = test_df.merge(example, how='outer', on='Asset_ID').sort_values('asset_order')\n    test = np.array(test_data[features].fillna(0))\n    test = test.reshape(-1, 1, N_ASSETS, test.shape[-1])\n    test_sample = np.hstack([test_sample, test])[:,-1 * WINDOW_SIZE:]\n    y_pred = model.predict(test_sample).squeeze().reshape(-1, 1).squeeze()\n    test_data['Target'] = y_pred\n    for _, row in test_df.iterrows():\n        try: sample_prediction_df.loc[sample_prediction_df['row_id'] == row['row_id'], 'Target'] = test_data.loc[test_data['row_id'] == row['row_id'], 'Target'].item()\n        except: sample_prediction_df.loc[sample_prediction_df['row_id'] == row['row_id'], 'Target'] = 0\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-03T20:03:42.4214Z","iopub.status.idle":"2022-01-03T20:03:42.422046Z","shell.execute_reply.started":"2022-01-03T20:03:42.421817Z","shell.execute_reply":"2022-01-03T20:03:42.421841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}