{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"A simple NN starter using asset Embedding. Hope it will be useful for you.\n\nHeavily inspired from this notebook: https://www.kaggle.com/lucasmorin/tf-keras-nn-with-stock-embedding.\n        \nThis code is only trained with 100,000 data for time consuming. It can be change to all data with the debug pamameter. \n","metadata":{}},{"cell_type":"code","source":"import io\nimport json\nimport requests\nimport functools\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\nfrom sklearn.model_selection import train_test_split\nfrom scipy.stats import pearsonr\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import RobustScaler\nimport tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nfrom keras import backend as K\n\ndebug = False","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:58:16.656459Z","iopub.execute_input":"2021-11-05T08:58:16.657017Z","iopub.status.idle":"2021-11-05T08:58:16.667387Z","shell.execute_reply.started":"2021-11-05T08:58:16.656977Z","shell.execute_reply":"2021-11-05T08:58:16.666366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"if debug:\n    train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv', nrows=100000)\nelse:\n    train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\n\n\ntrain = train[~train.isin([np.nan, np.inf, -np.inf]).any(1)].reset_index(drop=True)\n\n\nprint('train shape:',train.shape)\ntrain.head() ","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:58:16.66928Z","iopub.execute_input":"2021-11-05T08:58:16.669581Z","iopub.status.idle":"2021-11-05T08:58:54.088749Z","shell.execute_reply.started":"2021-11-05T08:58:16.669518Z","shell.execute_reply":"2021-11-05T08:58:54.088055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/g-research-crypto-forecasting/example_test.csv')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:58:54.09003Z","iopub.execute_input":"2021-11-05T08:58:54.090389Z","iopub.status.idle":"2021-11-05T08:58:54.109759Z","shell.execute_reply.started":"2021-11-05T08:58:54.090336Z","shell.execute_reply":"2021-11-05T08:58:54.109036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The columns in the train data is just the same as the real market, so we can make very practical models. Besides, the test date is part of the train data, so the LB is not trustable, we should rely on the CV score.","metadata":{}},{"cell_type":"markdown","source":"## Feature Engineering\n\nOnly use np.log1p to price data now. Some features will be add in the future.","metadata":{}},{"cell_type":"code","source":"def feature_engineer(df):\n    for col in ['Open', 'High', 'Low', 'Close', 'VWAP']:\n        df[col] = np.log1p(df[col])\n    df = df.fillna(0)\n    return df\n\ntrain = feature_engineer(train)\ntest = feature_engineer(test)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:58:54.111635Z","iopub.execute_input":"2021-11-05T08:58:54.112156Z","iopub.status.idle":"2021-11-05T08:58:57.468389Z","shell.execute_reply.started":"2021-11-05T08:58:54.112115Z","shell.execute_reply":"2021-11-05T08:58:57.467647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\ntrain_date = pd.DataFrame(train.timestamp.unique())\ntrain_date.columns = ['timestamp']\ntrain_date['date'] = [datetime.fromtimestamp(u) for u in train_date['timestamp']]\n\nprint('train data begin date:',train_date.head(1)['date'].values[0])\nprint('train data end date:',train_date.tail(1)['date'].values[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:58:57.469679Z","iopub.execute_input":"2021-11-05T08:58:57.470288Z","iopub.status.idle":"2021-11-05T08:59:00.450479Z","shell.execute_reply.started":"2021-11-05T08:58:57.470248Z","shell.execute_reply":"2021-11-05T08:59:00.449674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_date = pd.DataFrame(test.timestamp.unique())\ntest_date.columns = ['timestamp']\ntest_date['date'] = [datetime.fromtimestamp(u) for u in test_date['timestamp']]\nprint('example test data begin date:',test_date.head(1)['date'].values[0])\nprint('example test data end date:',test_date.tail(1)['date'].values[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:00.451665Z","iopub.execute_input":"2021-11-05T08:59:00.452112Z","iopub.status.idle":"2021-11-05T08:59:00.463333Z","shell.execute_reply.started":"2021-11-05T08:59:00.452072Z","shell.execute_reply":"2021-11-05T08:59:00.461912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reduce train memory\nReduce the memory of train data in case of OOM problem.","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    end_mem_ori = df.memory_usage().sum() / 1024**2\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased from {:5.2f} to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem_ori, end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\ntrain = reduce_mem_usage(train)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:00.464138Z","iopub.execute_input":"2021-11-05T08:59:00.464385Z","iopub.status.idle":"2021-11-05T08:59:04.129502Z","shell.execute_reply.started":"2021-11-05T08:59:00.464336Z","shell.execute_reply":"2021-11-05T08:59:04.128621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_columns =  ['Count', 'Open', 'High', 'Low', 'Close',\n       'Volume', 'VWAP']\ncategory_columns = ['Asset_ID']\ntarget_columns = ['Target']","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:04.130842Z","iopub.execute_input":"2021-11-05T08:59:04.131172Z","iopub.status.idle":"2021-11-05T08:59:04.137448Z","shell.execute_reply.started":"2021-11-05T08:59:04.131133Z","shell.execute_reply":"2021-11-05T08:59:04.135167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_nunique = train['Asset_ID'].nunique()\nprint('asset_nunique:',asset_nunique)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:04.138694Z","iopub.execute_input":"2021-11-05T08:59:04.139482Z","iopub.status.idle":"2021-11-05T08:59:04.248699Z","shell.execute_reply.started":"2021-11-05T08:59:04.139444Z","shell.execute_reply":"2021-11-05T08:59:04.248014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = RobustScaler()\ntrain[numerical_columns] = scaler.fit_transform(train[numerical_columns])\ntest[numerical_columns] = scaler.transform(test[numerical_columns])\n","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:04.250985Z","iopub.execute_input":"2021-11-05T08:59:04.251663Z","iopub.status.idle":"2021-11-05T08:59:11.693132Z","shell.execute_reply.started":"2021-11-05T08:59:04.251627Z","shell.execute_reply":"2021-11-05T08:59:11.692396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## NN model\n\nNN model with asset embedding.","metadata":{}},{"cell_type":"code","source":"hidden_units = (32,16,8,4,2)\n\ncat_data = train['Asset_ID']\n\ndef base_model():\n    \n    # Each instance will consist of two inputs: a single user id, and a single movie id\n    stock_id_input = keras.Input(shape=(1,), name='stock_id')\n\n    \n    num_input = keras.Input(shape=(len(numerical_columns),), name='num_data')\n\n\n    #embedding, flatenning and concatenating\n    stock_embedded = keras.layers.Embedding(16, 8, \n                                           input_length=1, name='stock_embedding')(stock_id_input)\n    \n    \n    stock_flattened = keras.layers.Flatten()(stock_embedded)\n\n\n    out = keras.layers.Concatenate()([stock_flattened,num_input])\n    hidden_units = (32,16,8,4,2)\n\n\n\n    # Add one or more hidden layers\n    for n_hidden in hidden_units:\n\n        out = keras.layers.Dense(n_hidden, activation='selu')(out)        \n\n    # A single output: our predicted rating\n    out = keras.layers.Dense(1, activation='linear', name='prediction')(out)\n    \n    model = keras.Model(\n    inputs = [stock_id_input, num_input],\n    outputs = out,\n    )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:11.69461Z","iopub.execute_input":"2021-11-05T08:59:11.694871Z","iopub.status.idle":"2021-11-05T08:59:11.704082Z","shell.execute_reply.started":"2021-11-05T08:59:11.694836Z","shell.execute_reply":"2021-11-05T08:59:11.703294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = 'NN'\npred_name = 'pred_{}'.format(model_name)\nn_folds = 5\n\nfeatures_to_consider = numerical_columns + category_columns\n\ntrain[pred_name] = 0\nmodellist = []\n\nfrom sklearn.model_selection import KFold,GroupKFold\n# kfold = KFold(n_splits = 5, random_state = 2021, shuffle = True)\nkfoldgroup = GroupKFold(n_splits = 5)\noof_predictions = np.zeros(train.shape[0])\ntest_predictions = np.zeros(test.shape[0])\n        \n# for fold, (trn_ind, val_ind) in enumerate(kfold.split(train)):\nfor fold, (trn_ind, val_ind) in enumerate(kfoldgroup.split(range(len(train)),train.Target,train.timestamp)):\n    X_train = train.loc[trn_ind, features_to_consider]\n    y_train = train.loc[trn_ind, target_columns].values\n    X_test = train.loc[val_ind, features_to_consider]\n    y_test = train.loc[val_ind, target_columns].values\n    \n    \n    model = base_model()\n    \n    model.compile(\n        keras.optimizers.Adam(learning_rate=0.001),\n        loss=tf.keras.metrics.mean_squared_error,\n        metrics=['MSE'],\n    )\n\n    num_data = X_train[numerical_columns]\n    stock_data = X_train['Asset_ID']\n    \n\n    \n    num_data_test = X_test[numerical_columns]\n    stock_data_test = X_test['Asset_ID']\n    \n    es = tf.keras.callbacks.EarlyStopping(\n        monitor='val_loss', min_delta=1e-05, patience=15, verbose=0,\n        mode='min', baseline=0.25)\n\n    plateau = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_loss', factor=0.5, patience=7, verbose=0,\n        mode='min')\n\n    model.fit([stock_data,num_data], \n              y_train, \n              batch_size=4096,\n              epochs=100,\n              validation_data=([stock_data_test, num_data_test], y_test),\n              callbacks=[es, plateau],\n              shuffle=True,\n             verbose = 0)\n\n    preds = model.predict([stock_data_test, num_data_test]).reshape(1,-1)[0]\n    oof_predictions[val_ind] = preds\n    score_fold = round(pearsonr(y_test.reshape(1,-1)[0], preds)[0],5)\n    print(f'fold {fold} oof score:',score_fold)\n\n    test_predictions += model.predict([test['Asset_ID'], test[numerical_columns]]).reshape(1,-1)[0]/5\n    modellist.append(model)\n    \n    \n","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:11.705649Z","iopub.execute_input":"2021-11-05T08:59:11.705962Z","iopub.status.idle":"2021-11-05T08:59:57.093549Z","shell.execute_reply.started":"2021-11-05T08:59:11.705927Z","shell.execute_reply":"2021-11-05T08:59:57.092492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['Target'] = test_predictions\nscore = round(pearsonr(train[target_columns].values.reshape(1,-1)[0], oof_predictions)[0],5)\nprint('oof score all:',score)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:57.09473Z","iopub.status.idle":"2021-11-05T08:59:57.095327Z","shell.execute_reply.started":"2021-11-05T08:59:57.095076Z","shell.execute_reply":"2021-11-05T08:59:57.095102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{"papermill":{"duration":0.04604,"end_time":"2021-11-04T06:02:56.393096","exception":false,"start_time":"2021-11-04T06:02:56.347056","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:57.096552Z","iopub.status.idle":"2021-11-05T08:59:57.09711Z","shell.execute_reply.started":"2021-11-05T08:59:57.096875Z","shell.execute_reply":"2021-11-05T08:59:57.096899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    test_df = feature_engineer(test_df)\n    test_predictions = 0\n    test_df[numerical_columns] = scaler.transform(test_df[numerical_columns])\n    for model in modellist:\n        test_predictions += model.predict([test_df['Asset_ID'], test_df[numerical_columns]]).reshape(1,-1)[0]/len(modellist)\n    sample_prediction_df['Target'] = test_predictions\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T08:59:57.098227Z","iopub.status.idle":"2021-11-05T08:59:57.098812Z","shell.execute_reply.started":"2021-11-05T08:59:57.098576Z","shell.execute_reply":"2021-11-05T08:59:57.098601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}