{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Crypto Prediction, XGB Regressor using polars dataframes library\n\nPolars is an alternative for Pandas that is significantly faster. See H2O's benchmark [here](http://https://h2oai.github.io/db-benchmark/).\n\nThe predictions for this competition take a long time and if there are too many features the submissions time out. Using the polars library speeds up the dataframe generation in the submission loop and therefore more features can be used.\n\nI also created a Kaggle dataset with the python wheel to install polars offline: https://www.kaggle.com/rluethy/polars-fast-dataframe-library\n\nAcknowledgements:\n\nhttps://github.com/pola-rs/polars\n\nhttps://www.pola.rs/","metadata":{}},{"cell_type":"markdown","source":"### Install polars library","metadata":{}},{"cell_type":"code","source":"!pip install ../input/polars-fast-dataframe-library/typing_extensions-4.0.1-py3-none-any.whl\n!pip install ../input/polars-fast-dataframe-library/polars-0.12.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:18:06.853253Z","iopub.execute_input":"2022-01-01T04:18:06.853568Z","iopub.status.idle":"2022-01-01T04:19:13.919358Z","shell.execute_reply.started":"2022-01-01T04:18:06.85347Z","shell.execute_reply":"2022-01-01T04:19:13.918507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport datetime\nimport time\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nfrom IPython.core.display import display\nimport traceback\nimport xgboost as xgb\n\nimport gresearch_crypto","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-01T04:19:13.921287Z","iopub.execute_input":"2022-01-01T04:19:13.921518Z","iopub.status.idle":"2022-01-01T04:19:15.065291Z","shell.execute_reply.started":"2022-01-01T04:19:13.921493Z","shell.execute_reply":"2022-01-01T04:19:15.064699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read training data","metadata":{}},{"cell_type":"code","source":"train_csv = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nasset_csv = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\ndf = pl.read_csv(train_csv)\ndf = df.sort(\"timestamp\")\nprint(df.shape)\n\ntotimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n\ndf_test = df[df[\"timestamp\"]>totimestamp(\"12/06/2021\")]\ndf_train = df[df[\"timestamp\"]<totimestamp(\"12/06/2021\")]\nprint(df_train.shape)\ndisplay(df_train.head())\ndisplay(df_train.tail())\n\ndf_asset_details = pd.read_csv(asset_csv)\ndisplay(df_asset_details)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:19:15.06659Z","iopub.execute_input":"2022-01-01T04:19:15.067391Z","iopub.status.idle":"2022-01-01T04:19:31.510598Z","shell.execute_reply.started":"2022-01-01T04:19:15.067346Z","shell.execute_reply":"2022-01-01T04:19:31.509949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get (lagged) features\nhttps://www.kaggle.com/tomforbes/gresearch-submitting-lagged-features-via-api","metadata":{}},{"cell_type":"code","source":"# Two new features from the competition tutorial\n\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef hlco_ratio(df): \n    return (df['High'] - df['Low'])/np.abs(df['Close']-df['Open'])\n\ndef get_features(df, asset_id, train=True):\n    '''\n   \n    This function takes a dataframe with all asset data and return the lagged features for a single asset.\n    \n    df - Full dataframe with all assets included\n    asset_id - integer from 0-13 inclusive to represent a cryptocurrency asset\n    train - True - you are training your model\n          - False - you are submitting your model via api\n    '''\n    \n    df = df[df['Asset_ID']==asset_id]\n    if train == True:\n        df_feat = df[['timestamp','Asset_ID','Open','High','Low','Close','Volume','VWAP','Target']]\n    else:\n        df_feat = df[['timestamp','Asset_ID','Open','High','Low','Close','Volume','VWAP']]\n    \n    # Create your features here, they can be lagged or not\n    df_feat['sma15'] = df_feat['Close'].rolling_mean(15)/df_feat['Close']\n    df_feat['sma30'] = df_feat['Close'].rolling_mean(30)/df_feat['Close'] \n    df_feat['sma60'] = df_feat['Close'].rolling_mean(60)/df_feat['Close'] \n    df_feat['std30'] = df_feat['Close'].rolling_std(30) \n    \n    df_feat['return15'] = df_feat['Close']/df_feat['Close'].shift(15)\n    df_feat['return30'] = df_feat['Close']/df_feat['Close'].shift(30)\n    df_feat['return60'] = df_feat['Close']/df_feat['Close'].shift(60)\n \n    df_feat['vwap15'] = df_feat['VWAP']/df_feat['VWAP'].rolling_mean(15)\n    df_feat['vw1'] = df_feat['VWAP']/df_feat['VWAP'].shift(1)\n    df_feat['vw2'] = df_feat['VWAP']/df_feat['VWAP'].shift(2)\n\n    df_feat['volume15'] = df_feat['Volume']/df_feat['Volume'].rolling_mean(15)\n    df_feat['v1'] = df_feat['Volume']/df_feat['Volume'].shift(1)\n    df_feat['v2'] = df_feat['Volume']/df_feat['Volume'].shift(2)\n\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n\n    df_feat[\"high_div_low\"] = np.log(df_feat[\"High\"] / df_feat[\"Low\"])\n    df_feat['trade'] = df_feat['Close'] - df_feat['Open']\n    df_feat['shadow1'] = df_feat['trade'] / df_feat['Volume']\n    df_feat['shadow3'] = df_feat['Upper_Shadow'] / df_feat['Volume']\n    df_feat['shadow5'] = df_feat['Lower_Shadow'] / df_feat['Volume']\n    df_feat['mean1'] = (df_feat['shadow5'] + df_feat['shadow3']) / 2\n    df_feat['mean2'] = (df_feat['shadow1'] + df_feat['Volume']) / 2\n    df_feat['hlco_ratio'] = hlco_ratio(df_feat)\n\n    df_feat = df_feat.drop(['Open','High','Low','Close','Volume','VWAP'])\n    features = [f for f in df_feat.columns if f not in ['Target','timestamp', 'Asset_ID']]\n    for f in features:\n        df_feat[f] = df_feat[f].set(df_feat[f].is_infinite(), None)\n        df_feat[f] = df_feat[f].set(df_feat[f].is_nan(), None)\n    \n    df_feat = df_feat.fill_null(\"mean\")\n\n    return df_feat, features\n\ndef get_features_pandas(df, asset_id, train=True):\n    '''\n   \n    This function takes a dataframe with all asset data and return the lagged features for a single asset.\n    \n    df - Full dataframe with all assets included\n    asset_id - integer from 0-13 inclusive to represent a cryptocurrency asset\n    train - True - you are training your model\n          - False - you are submitting your model via api\n    '''\n    \n    df = df[df['Asset_ID']==asset_id]\n    if train == True:\n        df_feat = df[['timestamp','Asset_ID','Open','High','Low','Close','Volume','VWAP','Target']]\n    else:\n        df_feat = df[['timestamp','Asset_ID','Open','High','Low','Close','Volume','VWAP']]\n    \n    # Create your features here, they can be lagged or not\n    df_feat['sma15'] = df_feat['Close'].rolling(15).mean()/df_feat['Close']\n    df_feat['sma30'] = df_feat['Close'].rolling(30).mean()/df_feat['Close'] \n    df_feat['sma60'] = df_feat['Close'].rolling(60).mean()/df_feat['Close'] \n    df_feat['std30'] = df_feat['Close'].rolling(30).std() \n    \n    df_feat['return15'] = df_feat['Close']/df_feat['Close'].shift(15)\n    df_feat['return30'] = df_feat['Close']/df_feat['Close'].shift(30)\n    df_feat['return60'] = df_feat['Close']/df_feat['Close'].shift(60)\n \n    df_feat['vwap15'] = df_feat['VWAP']/df_feat['VWAP'].rolling(15).mean()\n    df_feat['vw1'] = df_feat['VWAP']/df_feat['VWAP'].shift(1)\n    df_feat['vw2'] = df_feat['VWAP']/df_feat['VWAP'].shift(2)\n\n    df_feat['volume15'] = df_feat['Volume']/df_feat['Volume'].rolling(15).mean()\n    df_feat['v1'] = df_feat['Volume']/df_feat['Volume'].shift(1)\n    df_feat['v2'] = df_feat['Volume']/df_feat['Volume'].shift(2)\n\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n\n    df_feat[\"high_div_low\"] = np.log(df_feat[\"High\"] / df_feat[\"Low\"])\n    df_feat['trade'] = df_feat['Close'] - df_feat['Open']\n    df_feat['shadow1'] = df_feat['trade'] / df_feat['Volume']\n    df_feat['shadow3'] = df_feat['Upper_Shadow'] / df_feat['Volume']\n    df_feat['shadow5'] = df_feat['Lower_Shadow'] / df_feat['Volume']\n    df_feat['mean1'] = (df_feat['shadow5'] + df_feat['shadow3']) / 2\n    df_feat['mean2'] = (df_feat['shadow1'] + df_feat['Volume']) / 2\n    df_feat['hlco_ratio'] = hlco_ratio(df_feat)\n    \n    df_feat = df_feat.drop(['Open','High','Low','Close','Volume','VWAP'], axis=1)\n    features = [f for f in df_feat.columns if f not in ['Target','timestamp', 'Asset_ID']]\n    df_feat = df_feat.replace([np.inf, -np.inf], np.nan)\n    df_feat = df_feat.fillna(df_feat.mean())\n    \n    return df_feat, features\n","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:19:31.512377Z","iopub.execute_input":"2022-01-01T04:19:31.512816Z","iopub.status.idle":"2022-01-01T04:19:31.538709Z","shell.execute_reply.started":"2022-01-01T04:19:31.512785Z","shell.execute_reply":"2022-01-01T04:19:31.537845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Measure time creating features using polars","metadata":{}},{"cell_type":"code","source":"%%timeit -n 1 -r 10\nx, features = get_features(df_train[:14*250], 2, False)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:19:31.539878Z","iopub.execute_input":"2022-01-01T04:19:31.540091Z","iopub.status.idle":"2022-01-01T04:19:31.578969Z","shell.execute_reply.started":"2022-01-01T04:19:31.540067Z","shell.execute_reply":"2022-01-01T04:19:31.577985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, features = get_features(df_train, 2, False)\nprint(features)\nprint(x.shape)\nx.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:19:31.580156Z","iopub.execute_input":"2022-01-01T04:19:31.580538Z","iopub.status.idle":"2022-01-01T04:19:34.04923Z","shell.execute_reply.started":"2022-01-01T04:19:31.580507Z","shell.execute_reply":"2022-01-01T04:19:34.048407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Measure time creating features using pandas","metadata":{}},{"cell_type":"code","source":"pd_df = df_train.to_pandas()","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:19:34.050481Z","iopub.execute_input":"2022-01-01T04:19:34.051347Z","iopub.status.idle":"2022-01-01T04:19:35.329153Z","shell.execute_reply.started":"2022-01-01T04:19:34.051301Z","shell.execute_reply":"2022-01-01T04:19:35.328466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit -n 1 -r 10\nx, features = get_features_pandas(pd_df.loc[:14*250], 2, False)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:19:35.329984Z","iopub.execute_input":"2022-01-01T04:19:35.330185Z","iopub.status.idle":"2022-01-01T04:19:35.571063Z","shell.execute_reply.started":"2022-01-01T04:19:35.33016Z","shell.execute_reply":"2022-01-01T04:19:35.57015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, features = get_features_pandas(pd_df, 2, False)\nprint(features)\nprint(x.shape)\nx.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:19:35.572209Z","iopub.execute_input":"2022-01-01T04:19:35.572503Z","iopub.status.idle":"2022-01-01T04:19:38.569076Z","shell.execute_reply.started":"2022-01-01T04:19:35.572465Z","shell.execute_reply":"2022-01-01T04:19:38.568079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The polars version is about 10 times faster in this example: ~2.5 ms for polar vs ~24 ms for pandas**","metadata":{}},{"cell_type":"markdown","source":"### Train xgboost models for each asset","metadata":{}},{"cell_type":"code","source":"def get_xgb_regr_for_asset(df_train, df_test, asset_id, asset_name, params, plot_imp=True):\n    df, features = get_features(df_train,asset_id,train=True)\n   \n    model = xgb.XGBRegressor(**params)\n    model.fit(df[features].to_numpy(), df[\"Target\"].to_numpy())\n    model.get_booster().feature_names = features\n\n    if plot_imp:\n        fig=plt.gcf()\n        xgb.plot_importance(model)\n        plt.title(\"Feature Importance for \"+asset_name)\n        plt.show()\n\n    tst, _ =  get_features(df_test,asset_id,train=True)\n    pred = model.predict(tst[features].to_numpy())\n    p = np.corrcoef(tst[\"Target\"].to_numpy(),pred)[0][1]\n    print(p)\n    if np.isnan(p):\n        p=0\n\n    return model, p\n\nxgb_params = {\"n_estimators\": 100,\n            \"max_depth\": 3,\n            \"learning_rate\": 0.06,\n            \"subsample\": 0.7,\n            \"colsample_bytree\": 0.6,\n            \"random_state\": 2020,\n            \"tree_method\": \"hist\",\n            \"objective\": \"reg:pseudohubererror\",\n            }\nmodels = {}\n\nt0 = time.time()\nt1 = time.time()\n\nfor idx, row in df_asset_details.iterrows():\n    print(f\"Training model for {row['Asset_Name']:<16} (ID={row['Asset_ID']:<2})\", end=\" \")\n    models[row['Asset_ID']], tst_corr = get_xgb_regr_for_asset(df_train, df_test, \n                                                                           row['Asset_ID'], \n                                                                           row['Asset_Name'], xgb_params) \n    \n    df_asset_details.loc[idx,\"test_corr\"] = tst_corr\n    print(f\" time spent {time.time()-t1} \")\n    t1= time.time()\n    \nprint(f\"Training time for all models {time.time()-t0:.0f} \")\ndisplay(df_asset_details)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:19:38.571874Z","iopub.execute_input":"2022-01-01T04:19:38.572273Z","iopub.status.idle":"2022-01-01T04:22:29.723667Z","shell.execute_reply.started":"2022-01-01T04:19:38.572228Z","shell.execute_reply":"2022-01-01T04:22:29.722737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"start = time.time()\n\n# define max_lookback - an integer > (greater than) the furthest look back in your lagged features\nmax_lookback = 60\n\n# create dataframe to store data from the api to create lagged features\nl = df_train.shape[0]\nhistory = df_train[l-(max_lookback*14+100):]\nhistory = history.drop(\"Target\")\nhistory.insert_at_idx(history.shape[1],\n                      pl.Series(values=[-1 for i in range(history.shape[0])], \n                                dtype=pl.datatypes.Int32, name=\"row_id\"))\nprint(history.shape)\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    # concatenate new api data to history dataframe\n    df_test[\"Asset_ID\"] = df_test[\"Asset_ID\"].astype(np.int64)\n    df_test[\"Count\"] = df_test[\"Count\"].astype(np.float64)\n    history = pl.concat([history, pl.DataFrame(df_test)])\n\n    for j , row in df_test.iterrows():\n        if models[row['Asset_ID']] is not None:\n            try:\n                model = models[row['Asset_ID']]\n                row_features, features = get_features(history, row['Asset_ID'], train=False)\n                x_test = row_features[-1][features].to_numpy()\n                y_pred = model.predict(x_test)[0]\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n            except:\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n                traceback.print_exc()\n        else: \n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n    history = history.sort('timestamp')\n    l = history.shape[0]\n    history = history[l-(max_lookback*14+100):]\n        \n    env.predict(df_pred)\n    \nt = time.time() - start\nprint(f\"Test time: {t:.3f}, {t/i:.6f} per iteration\")","metadata":{"execution":{"iopub.status.busy":"2022-01-01T04:22:29.725201Z","iopub.execute_input":"2022-01-01T04:22:29.725474Z","iopub.status.idle":"2022-01-01T04:22:30.204419Z","shell.execute_reply.started":"2022-01-01T04:22:29.725422Z","shell.execute_reply":"2022-01-01T04:22:30.20332Z"},"trusted":true},"execution_count":null,"outputs":[]}]}