{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport seaborn as sns\nfrom sklearn.model_selection import KFold\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport tensorflow_probability as tfp\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nimport gresearch_crypto\nimport gc\n\n\npd.set_option('display.max_columns', None)\n\nDEBUG = False","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-05T12:01:47.509338Z","iopub.execute_input":"2021-11-05T12:01:47.510244Z","iopub.status.idle":"2021-11-05T12:01:57.359759Z","shell.execute_reply.started":"2021-11-05T12:01:47.510193Z","shell.execute_reply":"2021-11-05T12:01:57.359086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv').set_index(\"timestamp\")\nassets = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\nassets_names = dict(zip(assets.Asset_ID, assets.Asset_Name))\n#for assets sorting \nassets_order = pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv').Asset_ID[:14]\nassets_order = dict((t,i) for i,t in enumerate(assets_order))\n\nif DEBUG:\n    train = train[1000000:12000000]","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:01:57.361459Z","iopub.execute_input":"2021-11-05T12:01:57.361907Z","iopub.status.idle":"2021-11-05T12:03:03.906244Z","shell.execute_reply.started":"2021-11-05T12:01:57.361872Z","shell.execute_reply":"2021-11-05T12:03:03.905370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(df):\n    df['Upper_Shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n    df['Lower_Shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:03:03.907508Z","iopub.execute_input":"2021-11-05T12:03:03.908477Z","iopub.status.idle":"2021-11-05T12:03:03.914001Z","shell.execute_reply.started":"2021-11-05T12:03:03.908442Z","shell.execute_reply":"2021-11-05T12:03:03.913126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Assets correlation","metadata":{}},{"cell_type":"code","source":"# data periods where all assets presented\ntrain['assets']=1\ntrain['assets']=train.groupby(by = train.index)['assets'].sum()\ntrain['asset_name'] = train.Asset_ID.map(assets_names)\ntrain['asset_name'].value_counts()\n\nall_same_time = train[train['assets']==14][['Asset_ID', 'Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'asset_name','VWAP','Target']]\nall_same_time.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:03:03.916462Z","iopub.execute_input":"2021-11-05T12:03:03.916790Z","iopub.status.idle":"2021-11-05T12:03:10.546515Z","shell.execute_reply.started":"2021-11-05T12:03:03.916749Z","shell.execute_reply":"2021-11-05T12:03:10.545632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_target = all_same_time.reset_index().pivot(index='asset_name', columns='timestamp')['Target'].transpose().corr()\ncorr_matrix = all_same_time.reset_index().drop(['Target', 'Asset_ID'], axis=1).pivot(index='asset_name', columns='timestamp').transpose().corr()\n\nfig, ax = plt.subplots(1,2,figsize=(20,8))\nsns.heatmap(np.round(corr_target, 2), annot=True, ax=ax[0], square=True)\nsns.heatmap(np.round(corr_matrix, 2), annot=True, ax=ax[1], square=True)\nax[0].title.set_text('Asset Targets correlation')\nax[1].title.set_text('Asset Features correlation')\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-05T12:03:10.547906Z","iopub.execute_input":"2021-11-05T12:03:10.548440Z","iopub.status.idle":"2021-11-05T12:03:52.009970Z","shell.execute_reply.started":"2021-11-05T12:03:10.548389Z","shell.execute_reply":"2021-11-05T12:03:52.008868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = make_subplots(rows=7, cols=2,shared_xaxes=True, vertical_spacing=0.03, subplot_titles=tuple([assets_names[i] for i in range(14)]))\n\ndata = all_same_time[1000:2400]\ndata['time'] = [pd.to_datetime(x, unit='s') for x in data.index]\nfor i in range(14):\n    \n    coin = data[data.Asset_ID == i]\n    name = assets_names[i]\n\n    fig.add_trace(go.Scatter(x=coin['time'], y=coin['VWAP'], name = name + ', VWAP'),row=i//2+1, col= i%2 +1)\n\nfig.update_layout(height=1000, title_text=' Weighted average prices')\nfig.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-11-05T12:03:52.011104Z","iopub.execute_input":"2021-11-05T12:03:52.011323Z","iopub.status.idle":"2021-11-05T12:03:52.701613Z","shell.execute_reply.started":"2021-11-05T12:03:52.011295Z","shell.execute_reply":"2021-11-05T12:03:52.700792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data preparation","metadata":{}},{"cell_type":"code","source":"train=train.dropna()\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:03:52.702903Z","iopub.execute_input":"2021-11-05T12:03:52.703610Z","iopub.status.idle":"2021-11-05T12:03:57.623000Z","shell.execute_reply.started":"2021-11-05T12:03:52.703576Z","shell.execute_reply":"2021-11-05T12:03:57.622182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VWAP_max = np.max(train[np.isfinite(train.VWAP)].VWAP)\nVWAP_min = np.min(train[np.isfinite(train.VWAP)].VWAP)\nprint(VWAP_max, \"\\n\", VWAP_min)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:03:57.624196Z","iopub.execute_input":"2021-11-05T12:03:57.624424Z","iopub.status.idle":"2021-11-05T12:04:00.369646Z","shell.execute_reply.started":"2021-11-05T12:03:57.624394Z","shell.execute_reply":"2021-11-05T12:04:00.368693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assets=train.Asset_ID.to_numpy()\ntargets = train['Target'].to_numpy() \n\nfeatures = ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Upper_Shadow','Upper_Shadow']\ntrain = add_features(train)[features]","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:04:00.371138Z","iopub.execute_input":"2021-11-05T12:04:00.371458Z","iopub.status.idle":"2021-11-05T12:04:03.233817Z","shell.execute_reply.started":"2021-11-05T12:04:00.371417Z","shell.execute_reply":"2021-11-05T12:04:03.233066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#scaler = QuantileTransformer(n_quantiles=10000, output_distribution='normal', random_state=0)\nscaler = RobustScaler()\n\ntrain.VWAP = np.nan_to_num(train.VWAP, posinf=VWAP_max, neginf=VWAP_min)\ntrain = scaler.fit_transform(train)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:04:03.235841Z","iopub.execute_input":"2021-11-05T12:04:03.236065Z","iopub.status.idle":"2021-11-05T12:04:11.475919Z","shell.execute_reply.started":"2021-11-05T12:04:03.236039Z","shell.execute_reply":"2021-11-05T12:04:11.475010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model","metadata":{}},{"cell_type":"code","source":"##https://github.com/tensorflow/tensorflow/issues/37495\ndef MaxCorrelation(y_true,y_pred):\n    \"\"\"\n    Goal is to maximize correlation between y_pred, y_true. Same as minimizing the negative.\n    \"\"\"\n    return -tf.math.abs(tfp.stats.correlation(y_pred,y_true, sample_axis=None, event_axis=None))\n\ndef Correlation(y_true,y_pred):\n\n    return tf.math.abs(tfp.stats.correlation(y_pred,y_true, sample_axis=None, event_axis=None))\n\ndef get_model():  \n    asset_input = keras.Input(shape=(1,))\n    feat_input = keras.Input(shape=(train.shape[-1:]))\n    \n    x = layers.Embedding(15, 16, input_length=1)(asset_input)\n    \n    x = keras.layers.Flatten()(x)\n    combined = keras.layers.Concatenate()([x, feat_input])\n     \n    x = layers.Dense(units=512)(combined)\n    x = layers.BatchNormalization()(x)\n    x = layers.Dense(units=192)(x)\n    #x = layers.Dense(units=128)(x)\n    x = layers.Dense(units=96)(x)\n    #out = layers.Dense(units=1, activation='tanh')(x)\n    out = layers.Dense(units=1)(x)\n    \n    model = keras.Model(inputs=[asset_input, feat_input], outputs=out)\n    \n    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5), \n                  #loss = 'mse', \n                loss = 'cosine_similarity',\n                  #loss = MaxCorrelation,\n                  metrics=[Correlation]\n                    )\n    \n    return model  \n\nmodel=get_model()\nmodel.summary()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-05T12:20:04.007280Z","iopub.execute_input":"2021-11-05T12:20:04.008190Z","iopub.status.idle":"2021-11-05T12:20:04.094414Z","shell.execute_reply.started":"2021-11-05T12:20:04.008144Z","shell.execute_reply":"2021-11-05T12:20:04.093548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"size = list(range(len(train)))\n\ntrain_ind,test_ind = train_test_split(np.array(size), shuffle=True,random_state=42, test_size=0.15)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:37:12.480148Z","iopub.execute_input":"2021-11-05T12:37:12.481015Z","iopub.status.idle":"2021-11-05T12:37:21.068369Z","shell.execute_reply.started":"2021-11-05T12:37:12.480977Z","shell.execute_reply":"2021-11-05T12:37:21.067487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, y_train, assets_train = train[train_ind], targets[train_ind], assets[train_ind]\nX_valid, y_valid, assets_valid = train[test_ind], targets[test_ind], assets[test_ind]","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:04:23.782645Z","iopub.execute_input":"2021-11-05T12:04:23.782906Z","iopub.status.idle":"2021-11-05T12:04:28.463611Z","shell.execute_reply.started":"2021-11-05T12:04:23.782869Z","shell.execute_reply":"2021-11-05T12:04:28.462831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(0)\nBATCH_SIZE=2**15\n\n#estop = keras.callbacks.EarlyStopping(monitor='val_Correlation', patience=7, verbose=0, mode='max',restore_best_weights=True)\nestop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, verbose=0, mode='min',restore_best_weights=True)\nscheduler = keras.optimizers.schedules.ExponentialDecay(1e-4, (5e-2*(len(X_train))/BATCH_SIZE), 1e-3)\n#scheduler = keras.optimizers.schedules.ExponentialDecay(1e-3, (1e-4*(len(X_train))/BATCH_SIZE), 1e-3)\nlr = keras.callbacks.LearningRateScheduler(scheduler, verbose = 1)\n        \nmodel.fit([assets_train, X_train], y_train, \n          validation_data = ([assets_valid, X_valid], y_valid), \n          epochs = 20, batch_size = BATCH_SIZE, \n          shuffle=True, callbacks = [lr, estop])\n","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:20:09.870858Z","iopub.execute_input":"2021-11-05T12:20:09.871172Z","iopub.status.idle":"2021-11-05T12:35:31.834633Z","shell.execute_reply.started":"2021-11-05T12:20:09.871138Z","shell.execute_reply":"2021-11-05T12:35:31.833552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    asset = test_df.Asset_ID.to_numpy()\n    test = add_features(test_df)[features]\n    test.VWAP = np.nan_to_num(test.VWAP, posinf=VWAP_max, neginf=VWAP_min)\n    test = scaler.transform(test)\n    y_pred = model.predict([asset, test]).squeeze()\n    \n    sample_prediction_df['Target'] = y_pred\n    #display(test_df)\n    #display(sample_prediction_df)    \n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T12:35:36.594542Z","iopub.execute_input":"2021-11-05T12:35:36.594854Z","iopub.status.idle":"2021-11-05T12:35:37.115680Z","shell.execute_reply.started":"2021-11-05T12:35:36.594807Z","shell.execute_reply":"2021-11-05T12:35:37.115047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}