{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-13T13:00:13.376555Z","iopub.execute_input":"2022-01-13T13:00:13.376934Z","iopub.status.idle":"2022-01-13T13:00:13.432870Z","shell.execute_reply.started":"2022-01-13T13:00:13.376875Z","shell.execute_reply":"2022-01-13T13:00:13.432148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Install external pandas_ta package\n\nOriginally, this worked and I calculated technical indicators using the pandas_ta package. However, over night it did not work anymore, due to an error I can not follow and seems to be kaggle-related. This is why I manually calculate indicators now.\n\n\n1. Download tar.gz package file from pypi\n2. Replace tar.gz by .xyz to avoid Kaggle is unpacking automatically\n3. Upload .xyz to your kaggle datasets and upload it here\n4. Run the code below","metadata":{}},{"cell_type":"code","source":"#! mkdir -p /tmp/pip/cache/\n#! cp ../input/pandas-ta/pandas-ta-0.3.14.xyz /tmp/pip/cache/pandas-ta-0.3.14.tar.gz\n#! pip install --no-index --find-links /tmp/pip/cache/ pandas-ta","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:00:13.438442Z","iopub.execute_input":"2022-01-13T13:00:13.438768Z","iopub.status.idle":"2022-01-13T13:00:13.444967Z","shell.execute_reply.started":"2022-01-13T13:00:13.438729Z","shell.execute_reply":"2022-01-13T13:00:13.444193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import data","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\npd.options.mode.chained_assignment = None\nimport gc\n\ndirectory = '/kaggle/input/g-research-crypto-forecasting/'\nfile_path = os.path.join(directory, 'train.csv')\ndtypes = {\n    'timestamp': np.int64,\n    'Asset_ID': np.int8,\n     'Count': np.int32,\n     'Open': np.float64,\n     'High': np.float64,\n     'Low': np.float64,\n    'Close': np.float64,\n     'Volume': np.float64,\n     'VWAP': np.float64,\n    'Target': np.float64,\n}\n\ndata = pd.read_csv(file_path, dtype=dtypes, usecols=list(dtypes.keys()))\n\nfile_path = os.path.join(directory, 'asset_details.csv')\ndetails = pd.read_csv(file_path)\n\ndata.set_index('timestamp', inplace = True)\ndata.sort_index(inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:00:13.446636Z","iopub.execute_input":"2022-01-13T13:00:13.446940Z","iopub.status.idle":"2022-01-13T13:00:50.210567Z","shell.execute_reply.started":"2022-01-13T13:00:13.446905Z","shell.execute_reply":"2022-01-13T13:00:50.209795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define functions","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pylab as plt\nfrom sklearn.preprocessing import MinMaxScaler\nfrom datetime import datetime\n\n# a function to generate technical indicators and return featues, targets, and an index\ndef prepare_features(df_in, length = 15, with_target = True):\n\n    df_tmp = df_in.copy()\n    # calculate sma\n    df_tmp['sma'] = df_tmp.Close.rolling(window = length).mean()\n\n    # calculate rsi with ewma\n    close_delta = df_tmp['Close'].diff()\n\n    # Make two series: one for lower closes and one for higher closes\n    up = close_delta.clip(lower=0)\n    down = -1 * close_delta.clip(upper=0)\n\n    # Use exponential moving average\n    ma_up = up.ewm(com = length - 1, adjust=True, min_periods = length).mean()\n    ma_down = down.ewm(com = length - 1, adjust=True, min_periods = length).mean()\n\n    rsi = ma_up / ma_down\n    df_tmp['rsi'] = 100 - (100/(1 + rsi))\n\n    # calculate atr\n    high_low = df_tmp['High'] - df_tmp['Low']\n    high_close = np.abs(df_tmp['High'] - df_tmp['Close'].shift())\n    low_close = np.abs(df_tmp['Low'] - df_tmp['Close'].shift())\n\n    ranges = pd.concat([high_low, high_close, low_close], axis=1)\n    true_range = np.max(ranges, axis=1)\n\n    df_tmp['atr'] = true_range.rolling(length).sum() / length\n    \n    \n    df_tmp.dropna(subset = ['sma', 'rsi', 'atr'], inplace = True)\n    X = df_tmp[['sma', 'rsi', 'atr']]\n    \n    if with_target:\n        y = df_tmp.Target\n    else:\n        y = None\n        \n    return X, y, df_tmp.index\n\n# a function to define the recurrent network\ndef make_rnn_model(inpt_shape):\n    model = tf.keras.Sequential([\n            tf.keras.layers.InputLayer(input_shape = inpt_shape),\n            tf.keras.layers.GRU(10),\n            tf.keras.layers.Dropout(0.25),\n            tf.keras.layers.Dense(1)\n    ])\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:00:50.212773Z","iopub.execute_input":"2022-01-13T13:00:50.213031Z","iopub.status.idle":"2022-01-13T13:00:52.947256Z","shell.execute_reply.started":"2022-01-13T13:00:50.212996Z","shell.execute_reply":"2022-01-13T13:00:52.946517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The models are generated on my local machine using this code\n\n#import os\n#import numpy as np\n#import pandas as pd\n#pd.options.mode.chained_assignment = None\n#import gc\n#from datetime import datetime\n#from sklearn.preprocessing import MinMaxScaler\n#import tensorflow as tf\n#from pickle import dump\n\n#directory = '/Users/ralfkellner/Library/Mobile Documents/com~apple~CloudDocs/Python/Module/DLTA/datasets/cryptoPrediction/'\n#file_path = os.path.join(directory, 'train.csv')\n#dtypes = {\n#    'timestamp': np.int64,\n#    'Asset_ID': np.int8,\n#     'Count': np.int32,\n#     'Open': np.float64,\n#     'High': np.float64,\n#     'Low': np.float64,\n#    'Close': np.float64,\n#     'Volume': np.float64,\n#     'VWAP': np.float64,\n#    'Target': np.float64,\n#}\n\n#data = pd.read_csv(file_path, dtype=dtypes, usecols=list(dtypes.keys()))\n\n#file_path = os.path.join(directory, 'asset_details.csv')\n#details = pd.read_csv(file_path)\n\n#data.set_index('timestamp', inplace = True)\n#data.sort_index(inplace = True)\n\n#train_start = datetime.timestamp(datetime(2021,1,1))\n#train_end = datetime.timestamp(datetime(2021,5,1))\n#test_end = datetime.timestamp(datetime(2021,6,1))\n\n#train_data = data.loc[train_start:train_end]\n#test_data = data.loc[train_end:test_end]\n\n#def prepare_features(df_in, length = 15, with_target = True):\n\n#    df_tmp = df_in.copy()\n    # calculate sma\n#    df_tmp['sma'] = df_tmp.Close.rolling(window = length).mean()\n\n    # calculate rsi with ewma\n#    close_delta = df_tmp['Close'].diff()\n\n    # Make two series: one for lower closes and one for higher closes\n#    up = close_delta.clip(lower=0)\n#    down = -1 * close_delta.clip(upper=0)\n\n    # Use exponential moving average\n#    ma_up = up.ewm(com = length - 1, adjust=True, min_periods = length).mean()\n#    ma_down = down.ewm(com = length - 1, adjust=True, min_periods = length).mean()\n\n#    rsi = ma_up / ma_down\n#    df_tmp['rsi'] = 100 - (100/(1 + rsi))\n\n    # calculate atr\n#    high_low = df_tmp['High'] - df_tmp['Low']\n#    high_close = np.abs(df_tmp['High'] - df_tmp['Close'].shift())\n#    low_close = np.abs(df_tmp['Low'] - df_tmp['Close'].shift())\n\n#    ranges = pd.concat([high_low, high_close, low_close], axis=1)\n#    true_range = np.max(ranges, axis=1)\n\n#    df_tmp['atr'] = true_range.rolling(length).sum() / length\n    \n    \n#    df_tmp.dropna(subset = ['sma', 'rsi', 'atr'], inplace = True)\n#    X = df_tmp[['sma', 'rsi', 'atr']]\n    \n#    if with_target:\n#        y = df_tmp.Target\n#    else:\n#        y = None\n        \n#    return X, y, df_tmp.index\n\n\n#for id_ in train_data.Asset_ID.unique():\n#    train_asset = train_data[train_data.Asset_ID == id_]\n#    test_asset = test_data[test_data.Asset_ID == id_]\n\n#    X_train, y_train, idx = prepare_features(train_asset)\n#    X_test, y_test, idx = prepare_features(test_asset)\n\n#    X_scaler = MinMaxScaler()\n#    X_scaler.fit(X_train)\n\n#    dump(X_scaler, open(f'network_scalers/{id_}.pkl', 'wb'))\n#    print(f'Scaler for asset {id_} has been exported.')\n\n#    X_train_, X_test_ = X_scaler.transform(X_train), X_scaler.transform(X_test)\n\n    # build a recurrent network\n#    lookback = 30\n\n#    X_train_rnn = []\n#    y_train_rnn = []\n\n#    for t in range(len(X_train_) - lookback):\n#        X_train_rnn.append(X_train_[t:(t + lookback)])\n#        y_train_rnn.append(y_train.values[(t + lookback)])\n\n#    X_test_rnn = []\n#    y_test_rnn = []\n\n#    for t in range(len(X_test_) - lookback):\n#        X_test_rnn.append(X_test_[t:(t + lookback)])\n#        y_test_rnn.append(y_test.values[(t + lookback)])\n\n#    X_train_rnn = np.array(X_train_rnn)\n#    X_test_rnn = np.array(X_test_rnn)\n\n#    y_train_rnn = np.array(y_train_rnn)\n#    y_test_rnn = np.array(y_test_rnn)\n\n#    model = tf.keras.Sequential([\n#        tf.keras.layers.InputLayer(input_shape = (X_train_rnn.shape[1], X_train_rnn.shape[2])),\n#        tf.keras.layers.GRU(10),\n#        tf.keras.layers.Dropout(0.25),\n#        tf.keras.layers.Dense(1)\n#    ])\n\n#    model.compile(loss = 'mean_absolute_error', optimizer = 'adam')\n#    model.fit(X_train_rnn, y_train_rnn, epochs = 1, validation_data = (X_test_rnn, y_test_rnn))\n\n#    print(np.corrcoef(model.predict(X_train_rnn).flatten(), y_train_rnn)[0, 1])\n#    print(np.corrcoef(model.predict(X_test_rnn).flatten(), y_test_rnn)[0, 1])\n\n#    model.save_weights(f'network_weights/rnn/{id_}.h5', save_format = 'h5')\n#    print(f'Model for asset {id_} has been exported.')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load scalers, models and recent data\n\nScalers and models are build using this notebook: https://www.kaggle.com/rkellner/easy-gru-network-with-technical-indicators","metadata":{}},{"cell_type":"code","source":"from pickle import load\nfrom datetime import datetime\n\ndata_recent = data.drop(['Target'], axis = 1).loc[datetime.timestamp(datetime(2021,6,1)):1623542340]\ndata_recent.loc[:, 'row_id'] = np.nan\n\nscalers = {}\nmodels = {}\ndata_memory = {}\n\nlength = 15\nlookback = 30\n\nfor id_ in details.Asset_ID:\n    scalers[id_] = load(open(f'/kaggle/input/scalers/{id_}.pkl', 'rb'))\n    models[id_] = make_rnn_model((lookback, scalers[id_].n_features_in_))\n    models[id_].load_weights(f'/kaggle/input/rnn-weights/{id_}.h5')\n    data_memory[id_] = data_recent[data_recent.Asset_ID == id_].iloc[-50:, :]","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:00:52.948614Z","iopub.execute_input":"2022-01-13T13:00:52.948878Z","iopub.status.idle":"2022-01-13T13:00:57.694620Z","shell.execute_reply.started":"2022-01-13T13:00:52.948842Z","shell.execute_reply":"2022-01-13T13:00:57.693746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make predictions","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nstart_time = datetime.now()\n\nfor (test_df, sample_predictions) in iter_test:\n    \n    # set the timestamp as index for new data\n    test_df.set_index('timestamp', inplace = True)\n\n    # current predictions are generated at every iteration, past current_predictions should be deleted upfront\n    try:\n        del current_predictions\n    except:\n        print('No current predictions')\n\n    for id_ in details.Asset_ID:\n\n        # extract data for a specific id\n        test_df_tmp = test_df[test_df.Asset_ID == id_]\n        # append new data information to the id data memory\n        data_memory[id_] = pd.concat([data_memory[id_], test_df_tmp]) \n        # get features for old and new observations\n        X_data, holder_, idx = prepare_features(data_memory[id_], with_target = False)\n        # scale features\n        X_data_ = scalers[id_].transform(X_data)\n\n        # prepare for rnn model\n        X_data_rnn = []\n\n        for t in range(len(X_data_) - lookback):\n            X_data_rnn.append(X_data_[t:(t + lookback)])\n\n        X_data_rnn = np.array(X_data_rnn)\n\n        # make prediction and add this to the id specific data sheet\n        data_memory[id_].loc[:, 'Target_hat'] = np.concatenate([np.array([np.nan] * (length + lookback)), models[id_](X_data_rnn).numpy().flatten()])\n\n        # build a current predictions dataframe for all ids at this iteration\n        try:\n            current_predictions = pd.concat((current_predictions, data_memory[id_].dropna(subset = ['row_id'])))\n        except:\n            current_predictions = data_memory[id_].dropna(subset = ['row_id']).copy()\n\n        # delete past observations to avoid the data memory getting to large\n        data_memory[id_] = data_memory[id_].iloc[len(test_df_tmp):, :]\n        data_memory[id_].loc[: , 'row_id'] = np.nan\n        data_memory[id_].loc[: , 'Target_hat'] = np.nan\n\n    # make sample prediction\n    sample_predictions = sample_predictions.merge(current_predictions, on = 'row_id', how = 'inner')\n    sample_predictions = sample_predictions[['row_id', 'Target_hat']]\n    sample_predictions.rename(columns={\"row_id\": \"row_id\", \"Target_hat\": \"Target\"}, inplace = True)\n\n    env.predict(sample_predictions)\n    \ntime_elapsed = datetime.now() - start_time\nprint('Time elapsed (hh:mm:ss.ms) {}'.format(time_elapsed))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:00:57.695982Z","iopub.execute_input":"2022-01-13T13:00:57.696240Z","iopub.status.idle":"2022-01-13T13:00:59.270926Z","shell.execute_reply.started":"2022-01-13T13:00:57.696192Z","shell.execute_reply":"2022-01-13T13:00:59.270035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Notebook timeout: Unfortunately, every approach which lasts longer than approximately 0.8 seconds for the test iteration will fail when submitting and resulting in a runtime error. While the code above surely can be optimized in terms of run-time, I find this limitation quite significant. The model above is super easy and far from realistic, still, together with data handling takes too long. Quite frustrating!","metadata":{}}]}