{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport seaborn as sbn\nimport sklearn as sk\nimport tensorflow_probability as tfp\nimport gresearch_crypto\nimport warnings\nimport time\nimport gc\n\n\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import TimeSeriesSplit\nfrom keras.layers import Dense, Conv1D, BatchNormalization, LSTM, Bidirectional, LeakyReLU, Activation, MaxPooling1D, Concatenate, Dropout, Conv2D, Masking, Lambda, AveragePooling2D, AveragePooling1D\nfrom keras.preprocessing.sequence import pad_sequences\nfrom tensorflow import keras\nfrom keras import backend as K\nfrom keras.utils.vis_utils import plot_model\n\nwarnings.filterwarnings(\"ignore\")\ntf.autograph.set_verbosity(0)\n\ntrain = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\nasset_details = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\nasset_ids = list(train[\"Asset_ID\"].unique())\nasset_ids.sort()\nmodels = {}","metadata":{"execution":{"iopub.status.busy":"2022-02-09T17:10:07.542182Z","iopub.execute_input":"2022-02-09T17:10:07.542569Z","iopub.status.idle":"2022-02-09T17:11:21.094011Z","shell.execute_reply.started":"2022-02-09T17:10:07.542471Z","shell.execute_reply":"2022-02-09T17:11:21.093281Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Misc","metadata":{}},{"cell_type":"code","source":"def Plot_Results(history, y_test, y_pred):\n    fig ,axs = plt.subplots(1, 2, figsize=(10, 5))\n        \n    axs[0].scatter(y_test, y_pred)\n    axs[0].legend()\n    \n    axs[1].plot(history.history[\"loss\"])\n    axs[1].plot(history.history[\"val_loss\"])\n    axs[1].legend()\n\n    plt.show()\n    \ndef Reduce_Memory_Usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n         \ndef Compute(df): \n    R=list()\n    c=list(df['Close'])\n    for i in range(df.shape[0]):\n        future=c[min([i+16,df.shape[0]-1])]\n        past=c[min([i+1,df.shape[0]-1])]\n        R.append(future/past)\n        \n    R=np.array(R)\n    df['pred']=R-1\n    return df\n\ndef Calculate_Target(df):\n    crops=list()\n    for a in range(14):\n        crops.append(Compute(df[df['Asset_ID']==a]))\n        \n    conc=pd.concat(crops)\n    conc=conc.reset_index()\n    j=np.array(list(conc['Target'].isnull()))\n    new_targets=np.where(j,conc['pred'],conc['Target'])\n    conc['Target']=new_targets\n    conc=conc.drop(columns=['pred', 'index'],axis=1)\n    return conc\n\ndef Common_Timestamps(df):\n    timestamps = train['timestamp'].value_counts()==14\n    timestamps = timestamps[timestamps]\n    df = df.loc[df['timestamp'].isin(timestamps.index)]\n    \n    return df\n\ntrain = Reduce_Memory_Usage(train)\ntrain = Common_Timestamps(train)\ntrain = Calculate_Target(train)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T17:11:21.095703Z","iopub.execute_input":"2022-02-09T17:11:21.09614Z","iopub.status.idle":"2022-02-09T17:13:05.468166Z","shell.execute_reply.started":"2022-02-09T17:11:21.096087Z","shell.execute_reply":"2022-02-09T17:13:05.467067Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"def EDA(df):              \n    fig ,axs = plt.subplots(len(asset_ids), figsize=(15, 20))\n    for asset_id in asset_ids: \n        sbn.lineplot(ax=axs[asset_id], data=train[train['Asset_ID']==asset_id], x=\"timestamp\", y=\"Close\")\n#EDA(train)","metadata":{"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters","metadata":{}},{"cell_type":"code","source":"n_samples = 100000\nn_assets = 14\nn_columns = 20\nn_timesteps = 1","metadata":{"execution":{"iopub.status.busy":"2022-02-09T17:13:05.472566Z","iopub.execute_input":"2022-02-09T17:13:05.472846Z","iopub.status.idle":"2022-02-09T17:13:05.476763Z","shell.execute_reply.started":"2022-02-09T17:13:05.472813Z","shell.execute_reply":"2022-02-09T17:13:05.475914Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss Function","metadata":{}},{"cell_type":"code","source":"def correlation_coefficient_loss(y_true, y_pred):\n    x = y_true\n    y = y_pred\n    \n    mx = tf.reduce_mean(x)\n    my = tf.reduce_mean(y)\n\n    xm, ym = x-mx, y-my\n\n    r_num = K.sum(tf.multiply(xm,ym))\n    r_den = K.sqrt(tf.multiply(K.sum(K.square(xm)), K.sum(K.square(ym))))\n\n    r = r_num / r_den\n \n    r = K.maximum(K.minimum(r, 1.0), -1.0)\n\n    return 1 - K.square(r)\n\ndef root_mean_squared_error(y_true, y_pred):\n        return K.sqrt(K.mean(K.square(y_pred - y_true)))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T17:13:05.49779Z","iopub.execute_input":"2022-02-09T17:13:05.49894Z","iopub.status.idle":"2022-02-09T17:13:05.510154Z","shell.execute_reply.started":"2022-02-09T17:13:05.498891Z","shell.execute_reply":"2022-02-09T17:13:05.50952Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def Model(n_assets, n_timesteps, n_columns):\n    outs = []\n    input = keras.layers.Input(shape=(n_assets, n_timesteps, n_columns))\n\n    for i in range(n_assets):\n        inp = Lambda(lambda x: x[:,i,:,:])(input)\n        x = Masking(mask_value=0)(inp)   \n        x = Conv1D(1024, 1)(x)\n        x = LSTM(256, return_sequences=True)(x)\n        x = BatchNormalization()(x)\n        x = Activation(keras.activations.relu)(x)\n        \n        x = LSTM(128, return_sequences=True)(x)\n        x = BatchNormalization()(x)\n        x = Activation(keras.activations.relu)(x)\n        outs.append(x)\n\n    out = Concatenate(axis=1)(outs)\n    out = Conv1D(2048, 14)(out)\n    \n    out = Dense(512)(out)\n    out = Activation(keras.activations.elu)(out)\n    \n    out = Dense(256)(out)\n    out = Activation(keras.activations.elu)(out)\n    \n    out = Dense(128)(out)\n    out = Activation(keras.activations.elu)(out)\n    \n    out = Dense(64)(out)\n    out = Activation(keras.activations.elu)(out)\n    \n    out = Dense(32)(out)\n    out = Activation(keras.activations.elu)(out)\n    \n    out = Dense(14)(out)\n    out = Activation(keras.activations.linear)(out)\n    \n    model = tf.keras.Model(inputs=input, outputs=out)  \n\n    optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)\n    model.compile(loss=correlation_coefficient_loss, optimizer=optimizer)\n              \n    return model\n\nplot_model(Model(n_assets, n_timesteps, n_columns))","metadata":{"execution":{"iopub.status.busy":"2022-02-09T17:13:05.47906Z","iopub.execute_input":"2022-02-09T17:13:05.479579Z","iopub.status.idle":"2022-02-09T17:13:05.495603Z","shell.execute_reply.started":"2022-02-09T17:13:05.47954Z","shell.execute_reply":"2022-02-09T17:13:05.494874Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PreProcessing","metadata":{}},{"cell_type":"code","source":"def Pre_Processing(df): \n    df.replace([np.inf, -np.inf], np.nan, inplace=True)\n    df.dropna(inplace=True)\n    \n    Add_Features(df)\n       \ndef Add_Features(df):\n    #Quarter/One Hot Encode\n    x = df['timestamp'].astype('datetime64[s]')\n    ohe = keras.utils.to_categorical(x.dt.quarter, num_classes=5, dtype=np.int8)\n    df['Q1'] = ohe[:, 1]\n    df['Q2'] = ohe[:, 2]\n    df['Q3'] = ohe[:, 3]\n    df['Q4'] = ohe[:, 4]       \n    #Market Cap\n    df['market_cap'] = df['Asset_ID'].map(asset_details.set_index('Asset_ID')['Weight'])\n    #Close-Open\n    df['price_change'] = df['Close'] - df['Open'].astype(np.float16)\n    #High-Low\n    df['spread'] = df['High'] - df['Low'].astype(np.float16)\n    #Upper Shadow\n    df['upper_shadow'] = df['High'] - np.maximum(df['Open'], df['Close'])\n    #Lower Shadow\n    df['lower_shadow'] = np.minimum(df['Open'], df['Close']) - df['Low']\n    #Mean Trade\n    df['mean_trade'] = df['Volume'] / df['Count']\n    #Log Price Change\n    df['log_price_change'] = np.log(df['Close']/df['Open'])\n    \nPre_Processing(train)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T17:13:05.511413Z","iopub.execute_input":"2022-02-09T17:13:05.511673Z","iopub.status.idle":"2022-02-09T17:13:19.68688Z","shell.execute_reply.started":"2022-02-09T17:13:05.511638Z","shell.execute_reply":"2022-02-09T17:13:19.68591Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def Train_Model(df): \n    scaler = MinMaxScaler()\n    X = np.empty((n_samples, n_assets, n_timesteps, n_columns))\n    Y = np.empty((n_samples, n_assets, n_timesteps, 1))\n\n    for asset_id in asset_ids:\n        asset = df.loc[df['Asset_ID']==asset_id]\n        x = asset.loc[:, df.columns != 'Target'].to_numpy()\n        x = scaler.fit_transform(x)\n        y = asset['Target'].to_numpy()\n        for i in range(n_samples):\n            temp = x[i,:]\n            temp = temp.reshape(1, temp.shape[0])\n            X[i, asset_id,:,:] = temp\n            \n            temp = y[i]\n            temp = temp.reshape(1, 1)\n            Y[i, asset_id,:,:] = temp\n  \n    model = Model(n_assets, n_timesteps, n_columns)\n              \n    es = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=10, verbose=0, min_delta=0.0001, mode=\"min\", restore_best_weights=True)\n    cp = tf.keras.callbacks.ModelCheckpoint(r'./model.h5', save_best_only=True, monitor='val_loss', mode='min')\n    rlr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, mode='min')\n\n    tscv = TimeSeriesSplit(5)\n    i=0\n    for train_index, test_index in tscv.split(X):\n        print(\"Fold \" + str(i) + \" ---------------------------------------------------\")\n        i += 1\n        X_train, X_val = X[train_index,:,:,:], X[test_index,:,:,:]\n        y_train, y_val = Y[train_index,:,:,:], Y[test_index,:,:,:]\n            \n        \n        history = model.fit(X_train, y_train,\n                            validation_data=(X_val, y_val),\n                            epochs=500,\n                            batch_size=32,\n                            callbacks =[es, cp])\n        \n        \n        y_pred = model.predict(X_val)\n        Plot_Results(history, y_val, y_pred)\n    \n    return model\n\nmodel = Train_Model(train) ","metadata":{"execution":{"iopub.status.busy":"2022-02-09T17:13:19.688128Z","iopub.execute_input":"2022-02-09T17:13:19.688356Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission ","metadata":{}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nt0 = time.time()\nfor i, (df_test, df_pred) in enumerate(iter_test):    \n    df_test = Reduce_Memory_Usage(df_test)\n    Pre_Processing(df_test)\n  \n    df_test = df_test.drop('row_id', axis=1)\n    df_test = df_test.to_numpy().reshape(-1, n_assets, n_timesteps, n_columns)\n\n    df_pred['Target'] = model.predict(df_test).squeeze()\n    env.predict(df_pred)\nt1 = time.time() - t0\nprint(t1)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]}]}