{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gresearch_crypto\nimport xgboost as xgb\nimport traceback\nimport datetime\n\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\ndf = pd.read_csv(TRAIN_CSV)\ndf_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n\nasset_to_weight = df_asset_details.Weight.values\ndf[\"Weight\"] = df[\"Asset_ID\"].apply(lambda x: asset_to_weight[x])","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:54:33.483828Z","iopub.execute_input":"2021-11-21T11:54:33.484328Z","iopub.status.idle":"2021-11-21T11:55:43.947404Z","shell.execute_reply.started":"2021-11-21T11:54:33.484249Z","shell.execute_reply":"2021-11-21T11:55:43.946610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean(df):\n    df.replace([np.inf, -np.inf], np.nan, inplace=True)\n    df.dropna(how=\"any\", inplace=True)\n\ndef test_train_split(df):\n    X_train = df[df['timestamp'] <= 1623542400].drop('Target', axis=1)\n    y_train = df[df['timestamp'] <= 1623542400].Target\n    X_test = df[df['timestamp'] > 1623542400].iloc[:-1].drop('Target', axis=1)\n    y_test = df[df['timestamp'] > 1623542400].iloc[:-1].Target\n    return X_train, y_train, X_test, y_test\n\nclean(df)\nX_train, y_train, X_test, y_test = test_train_split(df)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:55:43.949330Z","iopub.execute_input":"2021-11-21T11:55:43.949586Z","iopub.status.idle":"2021-11-21T11:55:51.263902Z","shell.execute_reply.started":"2021-11-21T11:55:43.949552Z","shell.execute_reply":"2021-11-21T11:55:51.263159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NOT USED, I'll use it later though <3\nfrom pandas import DataFrame\nfrom pandas import concat\n \ndef time_lag(data, n_in=1, n_out=1, dropnan=True, interpolate = False):\n    \"\"\"\n    Frame a time series as a supervised learning dataset.\n    Arguments:\n        data: Sequence of observations as a list or NumPy array.\n        n_in: Number of lag observations as input (X).\n        n_out: Number of observations as output (y).\n        dropnan: Boolean whether or not to drop rows with NaN values.\n    Returns:\n        Pandas DataFrame of series framed for supervised learning.\n    \"\"\"\n    n_vars = 1 if type(data) is list else data.shape[1]\n    df = DataFrame(data)\n    cols, names = list(), list()\n    # input sequence (t-n, ... t-1)\n    for i in range(n_in, 0, -1):\n        cols.append(df.shift(i))\n        names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]\n    # forecast sequence (t, t+1, ... t+n)\n    for i in range(0, n_out):\n        cols.append(df.shift(-i))\n        if i == 0:\n            names += [('var%d(t)' % (j+1)) for j in range(n_vars)]\n        else:\n            names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]\n    # put it all together\n    agg = concat(cols, axis=1)\n    agg.columns = names\n    # drop rows with NaN values\n    if dropnan:\n        agg.dropna(inplace=True)\n    if interpolate:\n        agg.fillna(method='bfill', inplace=True)\n    return agg\n ","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:48:50.672679Z","iopub.execute_input":"2021-11-21T11:48:50.672927Z","iopub.status.idle":"2021-11-21T11:48:50.685383Z","shell.execute_reply.started":"2021-11-21T11:48:50.672893Z","shell.execute_reply":"2021-11-21T11:48:50.684609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(df):\n    df.loc[df[\"VWAP\"] <= 0, \"VWAP\"] = df.loc[df[\"VWAP\"] <= 0, \"Close\"]\n    df[\"Unity\"] = 1\n    df[\"Open\"] = np.log(df[\"Open\"])\n    df[\"High\"] = np.log(df[\"High\"])\n    df[\"Low\"] = np.log(df[\"Low\"])\n    df[\"Close\"] = np.log(df[\"Close\"])\n    df[\"VWAP\"] = np.log(df[\"VWAP\"])\n    #df['log_ret'] = np.log(df.Close/df.Open).fillna(0)\n    \n    #df['weird_feature'] = -(df['log_ret'] - (df['Weight'] * df['log_ret']).sum() / df['Weight'].sum())\n    \n    #norm_cols = ['Open','VWAP']\n    #ref = \"Close\"\n    #for col in norm_cols:\n    #    df[\"norm_\" + col] = df[col] / df[ref]\n    \n    #return df\n    #return pd.concat([df, time_lag(df[[\"VWAP\", \"Volume\", \"Open\", \"Close\"]], n_in=1, n_out=0, dropnan=False, interpolate=True)], axis=1)\n\n#X_train = \nprocess(X_train)\n#X_test = \nprocess(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:58:35.593515Z","iopub.execute_input":"2021-11-21T11:58:35.594216Z","iopub.status.idle":"2021-11-21T11:58:38.532424Z","shell.execute_reply.started":"2021-11-21T11:58:35.594151Z","shell.execute_reply":"2021-11-21T11:58:38.531661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:49:10.826276Z","iopub.execute_input":"2021-11-21T12:49:10.826850Z","iopub.status.idle":"2021-11-21T12:49:10.842978Z","shell.execute_reply.started":"2021-11-21T12:49:10.826810Z","shell.execute_reply":"2021-11-21T12:49:10.842156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f1(rows):\n    return (rows[\"Weight\"]*rows[\"Close\"]).sum()/rows[\"Weight\"].sum()\ndef f2(rows):\n    return (rows[\"Weight\"]*rows[\"Open\"]).sum()/rows[\"Weight\"].sum()\n\nX_test[\"mClose\"] = X_test.timestamp.map(X_test.groupby(\"timestamp\").apply(f1))\nX_test[\"mOpen\"] = X_test.timestamp.map(X_test.groupby(\"timestamp\").apply(f2))","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:25:55.780687Z","iopub.execute_input":"2021-11-21T12:25:55.781361Z","iopub.status.idle":"2021-11-21T12:27:22.715519Z","shell.execute_reply.started":"2021-11-21T12:25:55.781323Z","shell.execute_reply":"2021-11-21T12:27:22.714734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[\"mClose\"] = X_train.timestamp.map(X_train.groupby(\"timestamp\").apply(f1))\nX_train[\"mOpen\"] = X_train.timestamp.map(X_train.groupby(\"timestamp\").apply(f2))","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:28:18.399992Z","iopub.execute_input":"2021-11-21T12:28:18.400687Z","iopub.status.idle":"2021-11-21T12:46:09.993389Z","shell.execute_reply.started":"2021-11-21T12:28:18.400644Z","shell.execute_reply":"2021-11-21T12:46:09.992610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#(X_test[\"mClose\"] - X_test[\"mOpen\"]).head(30)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:50:18.426155Z","iopub.execute_input":"2021-11-21T12:50:18.427032Z","iopub.status.idle":"2021-11-21T12:50:18.440659Z","shell.execute_reply.started":"2021-11-21T12:50:18.426983Z","shell.execute_reply":"2021-11-21T12:50:18.439853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NOT USED\n# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef generate_features(df, lag = 1, shuffle = False):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    \n    #df_feat.fillna(-999,inplace=True)\n    if lag > 0:\n        df_feat = series_to_supervised(df_feat, n_in = lag)\n    df_feat['Upper_Shadow'] = upper_shadow(df)\n    df_feat['Lower_Shadow'] = lower_shadow(df)\n    if shuffle is True:\n        df_feat = df_feat.sample(frac=1)\n    return df_feat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.columns","metadata":{"execution":{"iopub.status.busy":"2021-11-21T11:41:16.810032Z","iopub.execute_input":"2021-11-21T11:41:16.810765Z","iopub.status.idle":"2021-11-21T11:41:16.817986Z","shell.execute_reply.started":"2021-11-21T11:41:16.810728Z","shell.execute_reply":"2021-11-21T11:41:16.817100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n\nclass BestModel:\n    def __init__(self):\n        self.beta = None\n        self.scaler = StandardScaler()\n    \n    def fit(self, X_train, y_train):\n        #self.scaler.fit(X_train)\n        #X = self.scaler.transform(X_train)\n        X = X_train.values\n        k = 0.5\n        mat = X.T@X\n        mat += k*np.identity(len(mat))\n        self.beta = np.linalg.inv(mat)@X.T@y_train.values\n        \n    def predict(self, X_test):\n        #X = self.scaler.transform(X_test)\n        X = X_test.values\n        return X@self.beta","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:47:13.596512Z","iopub.execute_input":"2021-11-21T12:47:13.596771Z","iopub.status.idle":"2021-11-21T12:47:13.603427Z","shell.execute_reply.started":"2021-11-21T12:47:13.596742Z","shell.execute_reply":"2021-11-21T12:47:13.602088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\"Count\", \"Open\", \"Close\", \"High\", \"Low\", \"Volume\", \"VWAP\", \"Unity\", \"mClose\", \"mOpen\"]#, 'weird_feature']\nprint(X_train.columns)","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:48:19.241850Z","iopub.execute_input":"2021-11-21T12:48:19.242393Z","iopub.status.idle":"2021-11-21T12:48:19.247465Z","shell.execute_reply.started":"2021-11-21T12:48:19.242353Z","shell.execute_reply":"2021-11-21T12:48:19.246543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [BestModel() for _ in range(len(df_asset_details))]\n\ny_pred = pd.Series(data=np.full_like(y_test.values, np.nan), index=y_test.index)\nfor asset_ID, model in enumerate(models):\n    X_asset_train = X_train[X_train.Asset_ID == asset_ID]\n    y_asset_train = y_train[X_train.Asset_ID == asset_ID]\n    X_asset_test = X_test[X_test.Asset_ID == asset_ID]\n    \n    model.fit(X_asset_train[features], y_asset_train)\n    y_pred[X_test.Asset_ID == asset_ID] = model.predict(X_asset_test[features])\n    print(f\"Trained model for asset {asset_ID}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:48:22.690010Z","iopub.execute_input":"2021-11-21T12:48:22.690431Z","iopub.status.idle":"2021-11-21T12:48:30.680890Z","shell.execute_reply.started":"2021-11-21T12:48:22.690387Z","shell.execute_reply":"2021-11-21T12:48:30.680109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr(a, b, w):\n    cov = lambda x, y: np.sum(w * (x - np.average(x, weights=w)) * (y - np.average(y, weights=w))) / np.sum(w)\n    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))\n\nalt_weight = np.ones_like(y_pred)\nR = corr(y_pred, y_test.values, X_test.Weight)\nprint(f\"{R:.5f}\")\n# 0.01641 shenanigans\n# 0.01523 remove two\n#0.01464 all\n# 0.01519 - log\n#0.01784587806310679\n\n#0.01992 - 0.01\n#0.02151 - 0.5\n# 0.02205 - 1.0\n# 0.02235 - 2.0\n#0.02081 - 0.2","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:48:30.685434Z","iopub.execute_input":"2021-11-21T12:48:30.685861Z","iopub.status.idle":"2021-11-21T12:48:30.826411Z","shell.execute_reply.started":"2021-11-21T12:48:30.685819Z","shell.execute_reply":"2021-11-21T12:48:30.825608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import pearsonr\nprint(pearsonr(y_pred, y_test))","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:48:34.371024Z","iopub.execute_input":"2021-11-21T12:48:34.371289Z","iopub.status.idle":"2021-11-21T12:48:34.409077Z","shell.execute_reply.started":"2021-11-21T12:48:34.371257Z","shell.execute_reply":"2021-11-21T12:48:34.408274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(models[0].beta)):\n    print(f\"{features[i]}: {models[0].beta[i]/np.sum(np.abs(models[0].beta)):.8f}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-21T12:47:50.203309Z","iopub.execute_input":"2021-11-21T12:47:50.203965Z","iopub.status.idle":"2021-11-21T12:47:50.211708Z","shell.execute_reply.started":"2021-11-21T12:47:50.203926Z","shell.execute_reply":"2021-11-21T12:47:50.210503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(np.abs(models[0].beta))\nplt.ylim(bottom=0)","metadata":{"execution":{"iopub.status.busy":"2021-11-18T11:17:21.426627Z","iopub.status.idle":"2021-11-18T11:17:21.42733Z","shell.execute_reply.started":"2021-11-18T11:17:21.427063Z","shell.execute_reply":"2021-11-18T11:17:21.427102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict & submit\n\nReferences: [Detailed API Introduction](https://www.kaggle.com/sohier/detailed-api-introduction)\n\nSomething that helped me understand this iterator was adding a pdb checkpoint inside of the for loop:\n\n```python\nimport pdb; pdb.set_trace()\n```\n\nSee [Python Debugging With Pdb](https://realpython.com/python-debugging-pdb/) if you want to use it and you don't know how to.\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T20:57:49.349459Z","iopub.status.idle":"2021-11-02T20:57:49.349757Z","shell.execute_reply.started":"2021-11-02T20:57:49.349596Z","shell.execute_reply":"2021-11-02T20:57:49.349613Z"}}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    df_pred['Target'] = np.nan\n    \n    df_test[\"Weight\"] = df_test[\"Asset_ID\"].apply(lambda x: asset_to_weight[x])\n    process(df_test)\n    df_test[\"mClose\"] = (df_test['Weight'] * df_test['Close']).sum() / df_test['Weight'].sum()\n    df_test[\"mOpen\"] = (df_test['Weight'] * df_test['Open']).sum() / df_test['Weight'].sum()\n    \n    for asset_ID, model in enumerate(models):\n        X_asset_test = df_test[df_test.Asset_ID == asset_ID]\n        df_pred.loc[df_test.Asset_ID == asset_ID, 'Target'] = model.predict(X_asset_test[features])\n    df_pred['Target'] = df_pred['Target'].interpolate('nearest')\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-18T11:17:42.937635Z","iopub.execute_input":"2021-11-18T11:17:42.938083Z","iopub.status.idle":"2021-11-18T11:17:43.062328Z","shell.execute_reply.started":"2021-11-18T11:17:42.938037Z","shell.execute_reply":"2021-11-18T11:17:43.061461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"oh yes!\")","metadata":{"execution":{"iopub.status.busy":"2021-11-18T11:17:39.62895Z","iopub.execute_input":"2021-11-18T11:17:39.629498Z","iopub.status.idle":"2021-11-18T11:17:39.634017Z","shell.execute_reply.started":"2021-11-18T11:17:39.629452Z","shell.execute_reply":"2021-11-18T11:17:39.633322Z"},"trusted":true},"execution_count":null,"outputs":[]}]}