{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype.name\n        \n        if col_type not in ['object', 'category', 'datetime64[ns, UTC]']:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-11-19T16:38:51.320307Z","iopub.execute_input":"2021-11-19T16:38:51.320911Z","iopub.status.idle":"2021-11-19T16:38:51.351635Z","shell.execute_reply.started":"2021-11-19T16:38:51.320810Z","shell.execute_reply":"2021-11-19T16:38:51.351019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gresearch_crypto\nimport traceback\nimport datetime\nimport matplotlib.pyplot as plt\n\nREDUCE_MEMORY = False\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\ndf = pd.read_csv(TRAIN_CSV)\nif REDUCE_MEMORY:\n    df = reduce_mem_usage(df)\ndf_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n\nasset_to_weight = df_asset_details.Weight.values\ndf[\"Weight\"] = df[\"Asset_ID\"].apply(lambda x: asset_to_weight[x])","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:07:29.409133Z","iopub.execute_input":"2021-11-19T17:07:29.409852Z","iopub.status.idle":"2021-11-19T17:08:39.560342Z","shell.execute_reply.started":"2021-11-19T17:07:29.409702Z","shell.execute_reply":"2021-11-19T17:08:39.559227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean(df):\n    df.replace([np.inf, -np.inf], np.nan, inplace=True)\n    df.dropna(how=\"any\", inplace=True)\n\ndef test_train_split(df):\n    X_train = df[df['timestamp'] <= 1623542400].drop('Target', axis=1)\n    y_train = df[df['timestamp'] <= 1623542400].Target\n    X_test = df[df['timestamp'] > 1623542400].iloc[:-1].drop('Target', axis=1)\n    y_test = df[df['timestamp'] > 1623542400].iloc[:-1].Target\n    return X_train, y_train, X_test, y_test\n\nclean(df)\nX_train, y_train, X_test, y_test = test_train_split(df)","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:08:39.562396Z","iopub.execute_input":"2021-11-19T17:08:39.562683Z","iopub.status.idle":"2021-11-19T17:08:51.589340Z","shell.execute_reply.started":"2021-11-19T17:08:39.562650Z","shell.execute_reply":"2021-11-19T17:08:51.588125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.utils.validation import _deprecate_positional_args\nfrom sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\n\n# modified code for group gaps; source\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass PurgedGroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n    Allows for a gap in groups to avoid potentially leaking info from\n    train into test if the model has windowed or lag features.\n    Provides train/test indices to split time series data samples\n    that are observed at fixed time intervals according to a\n    third-party provided group.\n    In each split, test indices must be higher than before, and thus shuffling\n    in cross validator is inappropriate.\n    This cross-validation object is a variation of :class:`KFold`.\n    In the kth split, it returns first k folds as train set and the\n    (k+1)th fold as test set.\n    The same group will not appear in two different folds (the number of\n    distinct groups has to be at least equal to the number of folds).\n    Note that unlike standard cross-validation methods, successive\n    training sets are supersets of those that come before them.\n    Read more in the :ref:`User Guide <cross_validation>`.\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_group_size : int, default=Inf\n        Maximum group size for a single training set.\n    group_gap : int, default=None\n        Gap between train and test\n    max_test_group_size : int, default=Inf\n        We discard this number of groups from the end of each train split\n    \"\"\"\n\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_group_size=np.inf,\n                 max_test_group_size=np.inf,\n                 group_gap=None,\n                 verbose=False,\n                 train_gap = 0 # in days\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_group_size = max_train_group_size\n        self.group_gap = group_gap\n        self.max_test_group_size = max_test_group_size\n        self.verbose = verbose\n        self.train_gap = train_gap\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        train_gap = self.train_gap\n        n_splits = self.n_splits\n        group_gap = self.group_gap\n        max_test_group_size = self.max_test_group_size\n        max_train_group_size = self.max_train_group_size\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n\n        group_test_size = min(n_groups // n_folds, max_test_group_size)\n        group_test_starts = range(n_groups - n_splits * group_test_size - train_gap *n_splits,\n                                  n_groups, group_test_size + train_gap) # altered\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n\n            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n                train_array_tmp = group_dict[train_group_idx]\n                \n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n\n            train_end = train_array.size\n \n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n\n            test_array  = test_array[group_gap:]\n            \n            \n            if self.verbose > 0:\n                    pass\n                    \n            yield [int(i) for i in train_array], [int(i) for i in test_array]\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\n\n# this is code slightly modified from the sklearn docs here:\n# https://scikit-learn.org/stable/auto_examples/model_selection/plot_cv_indices.html#sphx-glr-auto-examples-model-selection-plot-cv-indices-py\ndef plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10):\n    \"\"\"Create a sample plot for indices of a cross-validation object.\"\"\"\n    \n    cmap_cv = plt.cm.coolwarm\n\n    jet = plt.cm.get_cmap('jet', 256)\n    seq = np.linspace(0, 1, 256)\n    _ = np.random.shuffle(seq)   # inplace\n    cmap_data = ListedColormap(jet(seq))\n\n    # Generate the training/testing visualizations for each CV split\n    for ii, (tr, tt) in enumerate(cv.split(X=X, y=y, groups=group)):\n        # Fill in indices with the training/test groups\n        indices = np.array([np.nan] * len(X))\n        indices[tt] = 1\n        indices[tr] = 0\n\n        # Visualize the results\n        ax.scatter(range(len(indices)), [ii + .5] * len(indices),\n                   c=indices, marker='_', lw=lw, cmap=cmap_cv,\n                   vmin=-.2, vmax=1.2)\n\n    # Plot the data classes and groups at the end\n    ax.scatter(range(len(X)), [ii + 1.5] * len(X),\n               c=y, marker='_', lw=lw, cmap=plt.cm.Set3)\n\n    ax.scatter(range(len(X)), [ii + 2.5] * len(X),\n               c=group, marker='_', lw=lw, cmap=cmap_data)\n\n    # Formatting\n    yticklabels = list(range(n_splits)) + ['target', 'day']\n    ax.set(yticks=np.arange(n_splits+2) + .5, yticklabels=yticklabels,\n           xlabel='Sample index', ylabel=\"CV iteration\",\n           ylim=[n_splits+2.2, -.2], xlim=[0, len(y)])\n    ax.set_title('{}'.format(type(cv).__name__), fontsize=15)\n    return ax","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2021-11-19T17:08:51.591248Z","iopub.execute_input":"2021-11-19T17:08:51.591566Z","iopub.status.idle":"2021-11-19T17:08:52.635415Z","shell.execute_reply.started":"2021-11-19T17:08:51.591520Z","shell.execute_reply":"2021-11-19T17:08:52.634480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NOT USED, I'll use it later though <3\nfrom pandas import DataFrame\nfrom pandas import concat\n \ndef time_lag(data, n_in=1, n_out=1, dropnan=True, interpolate = False):\n    \"\"\"\n    Frame a time series as a supervised learning dataset.\n    Arguments:\n        data: Sequence of observations as a list or NumPy array.\n        n_in: Number of lag observations as input (X).\n        n_out: Number of observations as output (y).\n        dropnan: Boolean whether or not to drop rows with NaN values.\n    Returns:\n        Pandas DataFrame of series framed for supervised learning.\n    \"\"\"\n    n_vars = 1 if type(data) is list else data.shape[1]\n    df = DataFrame(data)\n    cols, names = list(), list()\n    # input sequence (t-n, ... t-1)\n    for i in range(n_in, 0, -1):\n        cols.append(df.shift(i))\n        names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]\n    # forecast sequence (t, t+1, ... t+n)\n    for i in range(0, n_out):\n        cols.append(df.shift(-i))\n        if i == 0:\n            names += [('var%d(t)' % (j+1)) for j in range(n_vars)]\n        else:\n            names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]\n    # put it all together\n    agg = concat(cols, axis=1)\n    agg.columns = names\n    # drop rows with NaN values\n    if dropnan:\n        agg.dropna(inplace=True)\n    if interpolate:\n        agg.fillna(method='bfill', inplace=True)\n    return agg\n ","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:08:52.637802Z","iopub.execute_input":"2021-11-19T17:08:52.638056Z","iopub.status.idle":"2021-11-19T17:08:52.647403Z","shell.execute_reply.started":"2021-11-19T17:08:52.638023Z","shell.execute_reply":"2021-11-19T17:08:52.646298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(df):\n    df[\"Unity\"] = 1\n    \n    norm_cols = ['Open','VWAP']\n    ref = \"Close\"\n    for col in norm_cols:\n        df[\"norm_\" + col] = df[col] / df[ref]\n    \n    return pd.concat([df, time_lag(df[[\"VWAP\", \"Volume\", \"Open\", \"Close\"]], n_in=1, n_out=0, dropnan=False, interpolate=True)], axis=1)\n\nX_train = process(X_train)\nif REDUCE_MEMORY:\n    X_train = reduce_mem_usage(X_train)\nX_test = process(X_test)\nif REDUCE_MEMORY:\n    X_train = reduce_mem_usage(X_train)","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:08:52.649258Z","iopub.execute_input":"2021-11-19T17:08:52.649998Z","iopub.status.idle":"2021-11-19T17:09:02.064085Z","shell.execute_reply.started":"2021-11-19T17:08:52.649929Z","shell.execute_reply":"2021-11-19T17:09:02.063060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_test[\"Unity\"] = 1","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:02.065434Z","iopub.execute_input":"2021-11-19T17:09:02.066738Z","iopub.status.idle":"2021-11-19T17:09:02.071917Z","shell.execute_reply.started":"2021-11-19T17:09:02.066696Z","shell.execute_reply":"2021-11-19T17:09:02.070714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NOT USED\n# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef generate_features(df, lag = 1, shuffle = False):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    \n    #df_feat.fillna(-999,inplace=True)\n    if lag > 0:\n        df_feat = series_to_supervised(df_feat, n_in = lag)\n    df_feat['Upper_Shadow'] = upper_shadow(df)\n    df_feat['Lower_Shadow'] = lower_shadow(df)\n    if shuffle is True:\n        df_feat = df_feat.sample(frac=1)\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:02.073350Z","iopub.execute_input":"2021-11-19T17:09:02.074270Z","iopub.status.idle":"2021-11-19T17:09:02.086307Z","shell.execute_reply.started":"2021-11-19T17:09:02.074225Z","shell.execute_reply":"2021-11-19T17:09:02.085425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n\nclass BestModel:\n    def __init__(self):\n        self.beta = None\n        self.scaler = StandardScaler()\n    \n    def fit(self, X_train, y_train):\n        #self.scaler.fit(X_train)\n        #X = self.scaler.transform(X_train)\n        X = X_train.values\n        mat = X.T@X\n        self.beta = np.linalg.inv(mat)@X.T@y_train.values\n        \n    def predict(self, X_test):\n        #X = self.scaler.transform(X_test)\n        X = X_test.values\n        return X@self.beta","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:02.087636Z","iopub.execute_input":"2021-11-19T17:09:02.088436Z","iopub.status.idle":"2021-11-19T17:09:02.104578Z","shell.execute_reply.started":"2021-11-19T17:09:02.088321Z","shell.execute_reply":"2021-11-19T17:09:02.103771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\"Count\", \"norm_Open\", \"Open\", \"Close\", \"Volume\", \"norm_VWAP\", \"VWAP\", \"Unity\", 'var1(t-1)', 'var2(t-1)', 'var3(t-1)',\n       'var4(t-1)']\nprint(X_train.columns)","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:02.106161Z","iopub.execute_input":"2021-11-19T17:09:02.106461Z","iopub.status.idle":"2021-11-19T17:09:02.118295Z","shell.execute_reply.started":"2021-11-19T17:09:02.106421Z","shell.execute_reply":"2021-11-19T17:09:02.117474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:02.123963Z","iopub.execute_input":"2021-11-19T17:09:02.124301Z","iopub.status.idle":"2021-11-19T17:09:02.163444Z","shell.execute_reply.started":"2021-11-19T17:09:02.124260Z","shell.execute_reply":"2021-11-19T17:09:02.162575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:02.165162Z","iopub.execute_input":"2021-11-19T17:09:02.165719Z","iopub.status.idle":"2021-11-19T17:09:02.172339Z","shell.execute_reply.started":"2021-11-19T17:09:02.165667Z","shell.execute_reply":"2021-11-19T17:09:02.171510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [BestModel() for _ in range(len(df_asset_details))]\n \nLESS_SAMPLES = False\n\nif LESS_SAMPLES: \n\n    X_train_ = X_train.copy().reset_index(drop = True).tail(1000000)\n    y_train_ = y_train.copy().reset_index(drop = True).tail(1000000) \n\n    y_pred = pd.Series(data=np.full_like(y_test.values, np.nan), index=y_test.index)\n    for asset_ID, model in enumerate(models):\n        X_asset_train = X_train_[X_train_.Asset_ID == asset_ID]\n        y_asset_train = y_train_[X_train_.Asset_ID == asset_ID]\n        X_asset_test = X_test[X_test.Asset_ID == asset_ID]\n\n        model.fit(X_asset_train[features], y_asset_train)\n        y_pred[X_test.Asset_ID == asset_ID] = model.predict(X_asset_test[features])\n    #     y_pred = y_pred.fillna(0)\n        print(f\"Trained model for asset {asset_ID}\")\nelse:\n\n    y_pred = pd.Series(data=np.full_like(y_test.values, np.nan), index=y_test.index)\n    for asset_ID, model in enumerate(models):\n        X_asset_train = X_train[X_train.Asset_ID == asset_ID]\n        y_asset_train = y_train[X_train.Asset_ID == asset_ID]\n        X_asset_test = X_test[X_test.Asset_ID == asset_ID]\n \n        model.fit(X_asset_train[features], y_asset_train)\n        y_pred[X_test.Asset_ID == asset_ID] = model.predict(X_asset_test[features])\n    #     y_pred = y_pred.fillna(0)\n        print(f\"Trained model for asset {asset_ID}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:02.173747Z","iopub.execute_input":"2021-11-19T17:09:02.174015Z","iopub.status.idle":"2021-11-19T17:09:17.438751Z","shell.execute_reply.started":"2021-11-19T17:09:02.173983Z","shell.execute_reply":"2021-11-19T17:09:17.434463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def percent_same_direction(a,b):\n    assert len(a) == len(b)\n    count = 0 \n    for i in range(len(a)):\n        if a[i]/abs(a[i]) == b[i]/abs(b[i]):\n            count +=1\n    return count/len(a)","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:17.441140Z","iopub.execute_input":"2021-11-19T17:09:17.441556Z","iopub.status.idle":"2021-11-19T17:09:17.450296Z","shell.execute_reply.started":"2021-11-19T17:09:17.441491Z","shell.execute_reply":"2021-11-19T17:09:17.449258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr(a, b, w):\n    cov = lambda x, y: np.sum(w * (x - np.average(x, weights=w)) * (y - np.average(y, weights=w))) / np.sum(w)\n    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))\n","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:17.452163Z","iopub.execute_input":"2021-11-19T17:09:17.452734Z","iopub.status.idle":"2021-11-19T17:09:17.464366Z","shell.execute_reply.started":"2021-11-19T17:09:17.452680Z","shell.execute_reply":"2021-11-19T17:09:17.463616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alt_weight = np.ones_like(y_pred)\nR = corr(y_pred, y_test.values, X_test.Weight)\nprint(f\"{R:.5f}\")\n# 0.01641 shenanigans\n# 0.01523 remove two\n#0.01464 all","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:17.466283Z","iopub.execute_input":"2021-11-19T17:09:17.467037Z","iopub.status.idle":"2021-11-19T17:09:17.660581Z","shell.execute_reply.started":"2021-11-19T17:09:17.466965Z","shell.execute_reply":"2021-11-19T17:09:17.659533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import pearsonr\nprint(pearsonr(y_pred, y_test))\nprint(percent_same_direction(y_pred.values, y_test.values))","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:09:17.662108Z","iopub.execute_input":"2021-11-19T17:09:17.662771Z","iopub.status.idle":"2021-11-19T17:09:19.987721Z","shell.execute_reply.started":"2021-11-19T17:09:17.662716Z","shell.execute_reply":"2021-11-19T17:09:19.986712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (15,5))\nplt.plot(y_pred[:1000])\nplt.plot(y_test[:1000])\n","metadata":{"execution":{"iopub.status.busy":"2021-11-19T16:49:35.875563Z","iopub.execute_input":"2021-11-19T16:49:35.875918Z","iopub.status.idle":"2021-11-19T16:49:36.286676Z","shell.execute_reply.started":"2021-11-19T16:49:35.875870Z","shell.execute_reply":"2021-11-19T16:49:36.286034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(models[0].beta)):\n    print(f\"{features[i]}: {models[0].beta[i]/np.sum(np.abs(models[0].beta)):.8f}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-19T16:40:46.964113Z","iopub.execute_input":"2021-11-19T16:40:46.964553Z","iopub.status.idle":"2021-11-19T16:40:46.971407Z","shell.execute_reply.started":"2021-11-19T16:40:46.964520Z","shell.execute_reply":"2021-11-19T16:40:46.970549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(np.abs(models[0].beta))\nplt.ylim(bottom=0)","metadata":{"execution":{"iopub.status.busy":"2021-11-19T16:40:46.974629Z","iopub.execute_input":"2021-11-19T16:40:46.974948Z","iopub.status.idle":"2021-11-19T16:40:47.192000Z","shell.execute_reply.started":"2021-11-19T16:40:46.974917Z","shell.execute_reply":"2021-11-19T16:40:47.191226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CV","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, mean_absolute_error\n\n# get_features_params = {'lag':1,'less_features': True, 'shuffle':False, 'diff':True}\nfeatures = [\"Count\", \"norm_Open\", \"Open\", \"Close\", \"Volume\", \"norm_VWAP\", \"VWAP\", \"Unity\", 'var1(t-1)', 'var2(t-1)', 'var3(t-1)',\n       'var4(t-1)']\n\nX_train['date'] = pd.to_datetime(X_train['timestamp'], unit = 's')\nX_train = X_train.sort_values('date')\ngroups = pd.factorize(X_train['date'].dt.day.astype(str) + '_' + X_train['date'].dt.month.astype(str) + '_' + X_train['date'].dt.year.astype(str))[0]\nX_train['groups'] = groups\nX_train = X_train.reset_index(drop = True)\ny_labels = y_train.reset_index(drop = True)\n\ncv = PurgedGroupTimeSeriesSplit(\n    n_splits=5,\n    max_train_group_size=500,\n    group_gap=20,\n    max_test_group_size=80,\n    train_gap = 120\n)\n\nmaes = []\ncorrelations = []\npercents_same_direction = []\n","metadata":{"execution":{"iopub.status.busy":"2021-11-19T17:13:31.020165Z","iopub.execute_input":"2021-11-19T17:13:31.021368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (train_idx, valid_idx) in enumerate(cv.split(\n    X_train,\n    y_labels,\n    groups=groups)):\n\n    train_data = X_train.loc[train_idx, :], y_labels.loc[train_idx]\n    valid_data = X_train.loc[valid_idx, :], y_labels.loc[valid_idx]\n\n#         display(X_train[valid_idx, :])\n    models = [BestModel() for _ in range(len(df_asset_details))]\n\n    y_pred = pd.Series(data=np.full_like(valid_data[1].values, np.nan), index=valid_data[1].index)\n    for asset_ID, model in enumerate(models):\n        X_asset_train = train_data[0][train_data[0].Asset_ID == asset_ID]\n        y_asset_train = train_data[1][train_data[0].Asset_ID == asset_ID]\n        X_asset_val = valid_data[0][valid_data[0].Asset_ID == asset_ID]\n        try:\n            model.fit(X_asset_train[features], y_asset_train)\n            y_pred[valid_data[0].Asset_ID == asset_ID] = model.predict(X_asset_val[features])\n        except:\n            traceback.print_exc()\n            continue\n        print(f\"Trained model for asset {asset_ID}\")\n#     pipe.fit(X_train[train_idx, :], y_labels[train_idx])\n#     preds = pipe.predict(X_train[valid_idx, :])\n    mae = mean_absolute_error(valid_data[1], y_pred)\n    correlation = corr(valid_data[1], y_pred, valid_data[0][\"Weight\"])\n    same_direction = percent_same_direction(y_pred.values, valid_data[1].values)\n#         print(preds)\n    maes.append(mae)\n    correlations.append(correlation)\n    percents_same_direction.append(same_direction)\n\nprint(f'Trial done: mae values on folds: {maes}, correlation: {correlations}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict & submit\n\nReferences: [Detailed API Introduction](https://www.kaggle.com/sohier/detailed-api-introduction)\n\nSomething that helped me understand this iterator was adding a pdb checkpoint inside of the for loop:\n\n```python\nimport pdb; pdb.set_trace()\n```\n\nSee [Python Debugging With Pdb](https://realpython.com/python-debugging-pdb/) if you want to use it and you don't know how to.\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T20:57:49.349459Z","iopub.status.idle":"2021-11-02T20:57:49.349757Z","shell.execute_reply.started":"2021-11-02T20:57:49.349596Z","shell.execute_reply":"2021-11-02T20:57:49.349613Z"}}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    df_pred['Target'] = np.nan\n    \n    df_test = process(df_test)\n    for asset_ID, model in enumerate(models):\n        X_asset_test = df_test[df_test.Asset_ID == asset_ID]\n        df_pred.loc[df_test.Asset_ID == asset_ID, 'Target'] = model.predict(X_asset_test[features])\n    df_pred['Target'] = df_pred['Target'].interpolate('nearest')\n    env.predict(df_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"oh yes!\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}