{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Crypto currency prediction\nMembers: ...\n\n> Acknowledgement: we used some code from [Proposal for a meaningful LB + Strict LGBM](https://www.kaggle.com/julian3833/proposal-for-a-meaningful-lb-strict-lgbm). We follow the definition of \"Strict\" from this author.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datetime import datetime\nimport tensorflow as tf\nimport gresearch_crypto\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-24T12:37:45.079475Z","iopub.execute_input":"2022-01-24T12:37:45.080685Z","iopub.status.idle":"2022-01-24T12:37:50.153608Z","shell.execute_reply.started":"2022-01-24T12:37:45.080610Z","shell.execute_reply":"2022-01-24T12:37:50.152862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data and preprocessing\nWe follow the \"strict\" criteria from [here](https://www.kaggle.com/julian3833/proposal-for-a-meaningful-lb-strict-lgbm). Therefore our score is valid.","metadata":{}},{"cell_type":"code","source":"# Loading traning data strictly\ndef read_csv_strict(file_name='../input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    df = df[df['datetime'] < '2021-06-13 00:00:00']\n    return df\n\ndata_df = read_csv_strict()\ndata_folder = '../input/g-research-crypto-forecasting/'\nasset_details_df = pd.read_csv(data_folder + 'asset_details.csv').set_index('Asset_ID')","metadata":{"execution":{"iopub.status.busy":"2022-01-23T13:01:33.336108Z","iopub.execute_input":"2022-01-23T13:01:33.336432Z","iopub.status.idle":"2022-01-23T13:02:29.351115Z","shell.execute_reply.started":"2022-01-23T13:01:33.336392Z","shell.execute_reply":"2022-01-23T13:02:29.349836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (Maybe) useful functions","metadata":{}},{"cell_type":"code","source":"def train_test_split(df, datetime):\n    return df[df['datetime'] < datetime], df[df['datetime'] >= datetime]\n\nDEFAULT_FEATURES = ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']\ndef get_Xy(df, asset_id, features=DEFAULT_FEATURES, na_treatment='drop'):\n    \"\"\"\n    Make sure don't pass the original dataframe into this function, otherwise it would be altered.\n    Available N/A treatments: drop, zero. Default to drop.\n    \"\"\"\n    df = df[df[\"Asset_ID\"] == asset_id]\n    df = df.replace([np.inf, -np.inf], np.nan)\n    if na_treatment == 'zero':\n        df = df.replace(np.nan, 0.)\n    else:\n        df = df.dropna(how='any')\n    \n    X = df[features]\n    y = df['Target']\n    return X, y\n\ndef get_corr(pred, y):\n    return np.correlate(pred, y)\n\ndef get_score(preds, ys):\n    corrs, weights = [], []\n    for asset in preds.keys():\n        corrs.append(np.correlate(preds[asset], ys[asset]))\n        weights.append(asset_details_df.loc[asset, 'Weight'])\n    corrs = np.array(corrs)\n    weights = np.array(weights)\n    return (corrs * weights).sum() / weights.sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:37:38.329904Z","iopub.execute_input":"2022-01-24T12:37:38.330172Z","iopub.status.idle":"2022-01-24T12:37:38.342183Z","shell.execute_reply.started":"2022-01-24T12:37:38.330144Z","shell.execute_reply":"2022-01-24T12:37:38.341530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Base class of all models\nAll models shall derive from this class, and rewrite function `train()` and `predict()`. The input of these functions should be pandas DataFrame.","metadata":{}},{"cell_type":"code","source":"class CryptoModel:\n    def train(self, df):\n        pass\n    \n    def predict(self, df_test, df_pred):\n        pass","metadata":{"execution":{"iopub.status.busy":"2022-01-23T13:02:29.3748Z","iopub.execute_input":"2022-01-23T13:02:29.375055Z","iopub.status.idle":"2022-01-23T13:02:29.393075Z","shell.execute_reply.started":"2022-01-23T13:02:29.375026Z","shell.execute_reply":"2022-01-23T13:02:29.3923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Baseline \\#1: End-to-end Fully Connected Neural Network\nTurns out NN is prone to overfitting. Results are all negative.","metadata":{"_kg_hide-input":false}},{"cell_type":"code","source":"# Baseline fully-connected NN on local data only\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import TensorDataset, DataLoader\nfrom tqdm import tqdm\n\nclass FCNetwork(nn.Module):\n    def __init__(self, sizes, activation=nn.ReLU, activation_out=False):\n        super().__init__()\n        n_layers = len(sizes) - 1\n        self.layers = []\n        for i in range(n_layers):\n            self.layers.append(nn.Linear(sizes[i], sizes[i+1]))\n            if i != n_layers - 1 or activation_out:\n                self.layers.append(activation())\n        if len(self.layers) == 0:\n            self.layers.append(nn.Identity())\n        self.layers = nn.ModuleList(self.layers)\n    \n    def forward(self, x):\n        for layer in self.layers:\n            x = layer(x)\n        return x\n\nclass FCModel(nn.Module, CryptoModel):\n    def __init__(self, sizes_public, sizes_private, lr=1e-4, activation=nn.ReLU, dtype=torch.float, device='cpu'):\n        super().__init__()\n        n_assets = 14\n        self.device = device\n        self.dtype = dtype\n        self.public_network = FCNetwork(sizes_public, activation=activation, activation_out=True).to(device)\n        self.n_public_layers = len(sizes_public) - 1\n        self.private_networks = nn.ModuleList([FCNetwork(sizes_private, activation=activation) for _ in range(n_assets)]).to(device)\n        self.n_private_layers = len(sizes_private) - 1\n        params = list(self.public_network.parameters()) + list(self.private_networks.parameters())\n        self.optimizer = optim.Adam(params=params, lr=lr)\n        self.features = ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']\n        print('Initialized.')\n  \n    def forward(self, x, asset):\n        h = self.public_network(x)\n        pn = self.private_networks[asset]\n        return pn(h).squeeze()\n  \n    def train(self, df, loss_fn=nn.MSELoss(), epoch=10, batch_size=1024):\n        df = df.copy()\n        assets = df['Asset_ID'].drop_duplicates().to_list()\n        \n        dataloaders = {}\n        print('Begin moving.')\n        for asset in assets:\n            X, y = get_Xy(df, asset)\n            X = torch.tensor(X.to_numpy(), device=self.device, dtype=self.dtype)\n            y = torch.tensor(y.to_numpy(), device=self.device, dtype=self.dtype)\n            dataset = TensorDataset(X, y)\n            dataloaders[asset] = DataLoader(dataset, batch_size=batch_size)\n        print('End moving.')\n            \n        for i in range(epoch):\n            losses = []\n            \n            for asset in dataloaders.keys():\n                dataloader = dataloaders[asset]\n                loop = tqdm(enumerate(dataloader), total=len(dataloader))\n                for index, (x, target) in loop:\n                    pred = self.forward(x, asset)\n                    loss = loss_fn(pred, target)\n                    self.optimizer.zero_grad()\n                    loss.backward()\n                    self.optimizer.step()\n                    losses.append(loss.item())\n                    loop.set_description('Epoch [{}/{}]'.format(i+1, epoch))\n                    loop.set_postfix(loss = loss.item())\n    \n    def test(self, X, y):\n        X = torch.tensor(X, device=self.device, dtype=self.dtype)\n        pred = self.forward(X).numpy()\n        return np.corrcoef(X, y)[0, 1]\n    \n    def predict(self, df_test, df_pred):\n        for j , row in df_test.iterrows():\n            idx = row['row_id']\n            x_test = row[self.features]\n            x_test = torch.tensor(x_test, device=self.device, dtype=self.dtype)\n            y_pred = self.forward(x_test, row['Asset_ID']).item()\n            df_pred.loc[df_pred['row_id'] == idx, 'Target'] = y_pred\n        return df_pred\n\n    def finalize(self):\n        self.to('cpu')\n        self.device = 'cpu'","metadata":{"execution":{"iopub.status.busy":"2022-01-23T13:02:29.396531Z","iopub.execute_input":"2022-01-23T13:02:29.397223Z","iopub.status.idle":"2022-01-23T13:02:29.433168Z","shell.execute_reply.started":"2022-01-23T13:02:29.397189Z","shell.execute_reply":"2022-01-23T13:02:29.432065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nfc_model = FCModel([7, 16], [16, 1], device=device)\nfc_model.train(data_df[data_df['Asset_ID'] == 1], epoch=3)\nfc_model.finalize()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-01-23T13:02:29.434405Z","iopub.execute_input":"2022-01-23T13:02:29.434885Z","iopub.status.idle":"2022-01-23T13:02:29.45245Z","shell.execute_reply.started":"2022-01-23T13:02:29.434855Z","shell.execute_reply":"2022-01-23T13:02:29.451265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Baseline \\#2: Light GBM with Shadow-Features\nUsing 2 features from [G-Research - Starter [0.361 LB]](https://www.kaggle.com/danofer/g-research-starter-0-361-lb).\n\n`LB=0.018`.","metadata":{}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom tqdm import tqdm\nclass BasicLGBM(CryptoModel):\n    def __init__(self):\n        self.models = [None] * 14\n        self.features = ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Lower_Shadow', 'Upper_Shadow']\n        \n    def train(self, df):\n        df = df.copy()\n        df = self._extend(df)\n        assets = df['Asset_ID'].drop_duplicates().to_list()\n        \n        for asset in assets:\n            X, y = get_Xy(df, asset, features=self.features)\n            model = LGBMRegressor(n_estimators=10)\n            model.fit(X, y)\n            self.models[asset] = model\n            \n    def predict(self, df_test, df_pred):\n        df_test = df_test.copy()\n        df_test = self._extend(df_test)\n        total = len(df_test.index)\n        \n        row_generator = df_test.iterrows()\n        row_generator = tqdm(generator, total=total) # Enable progress bar, comment this line to disable it.\n        for j , row in row_generator:\n            asset = row['Asset_ID']\n            x_test = row[self.features]\n            model = self.models[int(asset)]\n            y_pred = model.predict([x_test])[0]\n            mask = df_pred['row_id'] == j\n            df_pred.loc[mask, 'Target'] = y_pred\n        return df_pred\n            \n    def _extend(self, df):\n        df = df.copy()\n        df['Upper_Shadow'] = self._upper_shadow(df)\n        df['Lower_Shadow'] = self._lower_shadow(df)\n        return df\n    \n    def _upper_shadow(self, df):\n        return df['High'] - np.maximum(df['Close'], df['Open'])\n    \n    def _lower_shadow(self, df):\n        return np.minimum(df['Close'], df['Open']) - df['Low']","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:45:35.150975Z","iopub.execute_input":"2022-01-24T12:45:35.151247Z","iopub.status.idle":"2022-01-24T12:45:36.362725Z","shell.execute_reply.started":"2022-01-24T12:45:35.151219Z","shell.execute_reply":"2022-01-24T12:45:36.361846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LGBM Model v1.0\nThis model includes features: `Upper_Shadow`, `Lower_Shadow`, `Liquidity`, `Avg_Vol`, `Rel_Upper`, `Upper_VWAP`, `Upper_Volume`. By now it does not consider correlation between groups.\n\n`LB=0.0402`","metadata":{}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom tqdm import tqdm\nclass LGBMv1(CryptoModel):\n    def __init__(self):\n        self.models = [None] * 14\n        self.features = ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Lower_Shadow', 'Upper_Shadow',\n                         'Avg_Vol', 'Rel_Upper', 'Upper_VWAP', 'Upper_Vol', 'R_quantile', 'Rel_Upper_quantile']\n        \n    def train(self, df):\n        df = df.copy()\n        df = self._extend(df)\n        assets = df['Asset_ID'].drop_duplicates().to_list()\n        \n        for asset in assets:\n            X, y = get_Xy(df, asset, features=self.features)\n            model = LGBMRegressor(n_estimators=10)\n            model.fit(X, y)\n            self.models[asset] = model\n            \n    def predict(self, df_test, df_pred):\n        df_test = df_test.copy()\n        df_test = self._extend(df_test)\n        total = len(df_test.index)\n        \n        row_generator = df_test.iterrows()\n        row_generator = tqdm(row_generator, total=total) # Enable progress bar, comment this line to disable it.\n        for j , row in row_generator:\n            asset = row['Asset_ID']\n            x_test = row[self.features]\n            model = self.models[int(asset)]\n            y_pred = model.predict([x_test])[0]\n            mask = df_pred['row_id'] == j\n            df_pred.loc[mask, 'Target'] = y_pred\n        return df_pred\n            \n    def _extend(self, df):\n        df['Upper_Shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n        df['Lower_Shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n        #df['Liquidity'] = ((2 * (df['High'] - df['Low']) - np.absolute(df['Open'] - df['Close']))/ df['Volume']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Avg_Vol'] = (df['Volume'] / df['Count']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Rel_Upper'] = ((df['High'] - df['VWAP']) / (df['High'] - df['Low'])).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Upper_VWAP'] = ((df['High'] - df['VWAP']) / df['VWAP']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Upper_Vol'] = ((df['High'] - df['VWAP']) / df['Volume']).replace([np.inf, -np.inf, np.nan], 0.)\n        return df","metadata":{"execution":{"iopub.status.busy":"2022-01-23T13:02:29.473568Z","iopub.execute_input":"2022-01-23T13:02:29.474166Z","iopub.status.idle":"2022-01-23T13:02:29.494163Z","shell.execute_reply.started":"2022-01-23T13:02:29.474116Z","shell.execute_reply":"2022-01-23T13:02:29.493447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel_lgbm = LGBMv1()\nmodel_lgbm.train(data_df)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-23T13:02:29.495285Z","iopub.execute_input":"2022-01-23T13:02:29.495802Z","iopub.status.idle":"2022-01-23T13:03:22.478265Z","shell.execute_reply.started":"2022-01-23T13:02:29.495757Z","shell.execute_reply":"2022-01-23T13:03:22.477249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"benchmark_debug = Benchmark()\nbenchmark_debug.benchmark(model_lgbm)","metadata":{"execution":{"iopub.status.busy":"2022-01-23T13:03:22.480316Z","iopub.execute_input":"2022-01-23T13:03:22.480668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nfor m in model_lgbm.models:\n    z = list(zip(list(m.feature_name_), list(m.feature_importances_)))\n    z.sort(key=lambda x:x[1], reverse=True)\n    print(z[:5])\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LGBM Model v1.1\nAdds group correlation.\n\n`LB=0.0447`","metadata":{}},{"cell_type":"code","source":"class LGBMv1_1(LGBMv1):\n    def __init__(self):\n        super().__init__()\n        self.features += ['Upper_Rel', 'Lower_Rel', 'Upper_Rel_GA', 'Lower_Rel_GA', 'Avg_Vol_GA', 'Rel_Upper_GA']\n        \n    def train(self, df):\n        df = df.copy()\n        df = self._extend(df)\n        df = self._extend_correlation(df)\n        assets = df['Asset_ID'].drop_duplicates().to_list()\n        \n        for asset in assets:\n            X, y = get_Xy(df, asset, features=self.features)\n            model = LGBMRegressor(n_estimators=10)\n            model.fit(X, y)\n            self.models[asset] = model\n            \n    def predict(self, df_test, df_pred):\n        df_test = df_test.copy()\n        df_test = self._extend(df_test)\n        df_test = self._extend_correlation(df_test)\n        total = len(df_test.index)\n        \n        row_generator = df_test.iterrows()\n        row_generator = tqdm(row_generator, total=total) # Enable progress bar, comment this line to disable it.\n        for j , row in row_generator:\n            asset = row['Asset_ID']\n            x_test = row[self.features]\n            model = self.models[int(asset)]\n            y_pred = model.predict([x_test])[0]\n            mask = df_pred['row_id'] == j\n            df_pred.loc[mask, 'Target'] = y_pred\n        return df_pred\n        \n    def _extend_correlation(self, df):\n        df['Upper_Rel'] = (df['Upper_Shadow'] / df['Open']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Lower_Rel'] = (df['Lower_Shadow'] / df['Open']).replace([np.inf, -np.inf, np.nan], 0.)\n        #df['Liquidity'] = ((2 * (df['High'] - df['Low']) - np.absolute(df['Open'] - df['Close']))/ df['Volume']).replace([np.inf, -np.inf, np.nan], 0.)\n        #df['Liquidity_GA'] = df.groupby(['timestamp'])['Liquidity'].transform('mean')\n        df['Upper_Rel_GA'] = df.groupby(['timestamp'])['Upper_Rel'].transform('mean')\n        df['Lower_Rel_GA'] = df.groupby(['timestamp'])['Lower_Rel'].transform('mean')\n        df['Avg_Vol_GA'] = df.groupby(['timestamp'])['Avg_Vol'].transform('mean')\n        df['Rel_Upper_GA'] = df.groupby(['timestamp'])['Rel_Upper'].transform('mean')\n        return df\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#model_lgbm = LGBMv1_1()\n#model_lgbm.train(data_df)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM v1.2","metadata":{}},{"cell_type":"markdown","source":"added data of quantile.","metadata":{}},{"cell_type":"code","source":"from lightgbm import LGBMRegressor\nfrom tqdm import tqdm\nclass LGBMv1_2(CryptoModel):\n    def __init__(self):\n        self.models = [None] * 14\n        '''\n        self.features = ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Lower_Shadow', 'Upper_Shadow',\n                         'Avg_Vol', 'Rel_Upper', 'Upper_VWAP', 'Upper_Vol', 'R_quantile', 'Rel_Upper_quantile']\n        '''\n        self.features = ['Count', 'Volume', 'Lower_Shadow', 'Upper_Shadow','Avg_Vol', 'Rel_Upper', 'Upper_VWAP', 'Upper_Vol', 'R_quantile', 'Rel_Upper_quantile']\n    def train(self, df):\n        df = df.copy()\n        df = self._extend(df)\n        assets = df['Asset_ID'].drop_duplicates().to_list()\n        \n        for asset in assets:\n            X, y = get_Xy(df, asset, features=self.features)\n            model = LGBMRegressor(n_estimators=10)\n            model.fit(X, y)\n            self.models[asset] = model\n            \n    def predict(self, df_test, df_pred):\n        df_test = df_test.copy()\n        df_test = self._extend(df_test)\n        total = len(df_test.index)\n        \n        row_generator = df_test.iterrows()\n        row_generator = tqdm(row_generator, total=total) # Enable progress bar, comment this line to disable it.\n        for j , row in row_generator:\n            asset = row['Asset_ID']\n            x_test = row[self.features]\n            model = self.models[int(asset)]\n            y_pred = model.predict([x_test])[0]\n            mask = df_pred['row_id'] == j\n            df_pred.loc[mask, 'Target'] = y_pred\n        return df_pred\n            \n    def _extend(self, df):\n        df['Upper_Shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n        df['Lower_Shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n        #df['Liquidity'] = ((2 * (df['High'] - df['Low']) - np.absolute(df['Open'] - df['Close']))/ df['Volume']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Avg_Vol'] = (df['Volume'] / df['Count']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Rel_Upper'] = ((df['High'] - df['VWAP']) / (df['High'] - df['Low'])).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Upper_VWAP'] = ((df['High'] - df['VWAP']) / df['VWAP']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Upper_Vol'] = ((df['High'] - df['VWAP']) / df['Volume']).replace([np.inf, -np.inf, np.nan], 0.)\n        \n        \n        df['Return_1min'] = (df['Close']/df['Open'] - 1.0).replace([np.inf, -np.inf, np.nan], 0.)\n        df['R_max'] = df.groupby(['timestamp'])['Return_1min'].transform('max')\n        df['R_min'] = df.groupby(['timestamp'])['Return_1min'].transform('min')\n        df['R_quantile'] = (df['Return_1min'] - df['R_min'])/(df['R_max'] - df['R_min'])\n        \n        df['Rel_Upper_max'] = df.groupby(['timestamp'])['Rel_Upper'].transform('max')\n        df['Rel_Upper_min'] = df.groupby(['timestamp'])['Rel_Upper'].transform('min')\n        df['Rel_Upper_quantile'] = (df['Rel_Upper'] - df['Rel_Upper_min'])/(df['Rel_Upper_max'] - df['Rel_Upper_min']).replace([np.inf, -np.inf, np.nan], 0.5)\n        \n    \n        return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lgbm = LGBMv1_2()\nmodel_lgbm.train(data_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Substitute Evaluation\nThe evaluation process of submission is slow and vague, therefore this section is implemented to benchmark the model.","metadata":{}},{"cell_type":"code","source":"class Benchmark:\n    def __init__(self, debug=False):\n        data_folder = '../input/g-research-crypto-forecasting/'\n        self.test_features = ['timestamp', 'Asset_ID', 'Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'group_num']\n        self.asset_details_df = pd.read_csv(data_folder + 'asset_details.csv')\n        if debug:\n            self.df_test = pd.read_csv(data_folder + 'example_test.csv')\n            self.df_pred = pd.read_csv(data_folder + 'example_sample_submission.csv')\n            self.target = self.df_test.copy()\n            self.target['Target'] = self.df_pred['Target']\n        else:\n            df = pd.read_csv(data_folder + 'supplemental_train.csv')\n            df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n            df = df[df['datetime'] >= '2021-06-13 00:00:00'].head(100000)\n            df = df.replace([np.inf, -np.inf], np.nan).dropna(how='any').reset_index()\n            df.loc[:, 'group_num'] = 0\n            df.loc[:, 'row_id'] = df.index\n            self.df_test = df[self.test_features].copy()\n            self.df_pred = df[['group_num', 'row_id', 'Target']].copy()\n            self.df_pred['Target'] = 0.\n            self.target = self.df_test.copy()\n            self.target['Target'] = df['Target']\n    \n    def benchmark(self, model):\n        df_test, df_pred, target, detail = self.df_test, self.df_pred, self.target, self.asset_details_df\n        group_ids = df_test['group_num'].drop_duplicates().to_list()\n        for idx in group_ids:\n            y = model.predict(df_test[df_test['group_num'] == idx], df_pred[df_pred['group_num'] == idx])\n            mask = df_pred['group_num'] == idx\n            df_pred.loc[mask, 'Target'] = y['Target']\n        \n        asset_ids = df_test['Asset_ID'].drop_duplicates().to_list()\n        corrs, weights = [], []\n        for idx in asset_ids:\n            asset_target_df = target.loc[target['Asset_ID'] == idx, 'Target']\n            row_ids = asset_target_df.index\n            asset_target = asset_target_df.to_numpy()\n            asset_pred = df_pred.iloc[row_ids]['Target'].to_numpy()\n            corr =  np.corrcoef(asset_target, asset_pred)[0, 1]\n            if np.isnan(corr):\n                corr = 0.\n            corrs.append(corr)\n            weights.append(detail[detail['Asset_ID'] == idx]['Weight'])\n        corrs = np.array(corrs).squeeze()\n        weights = np.array(weights).squeeze()\n        print(corrs)\n        print(weights)\n        return (corrs * weights).sum() / weights.sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:44:24.291248Z","iopub.execute_input":"2022-01-24T12:44:24.292472Z","iopub.status.idle":"2022-01-24T12:44:24.310646Z","shell.execute_reply.started":"2022-01-24T12:44:24.292429Z","shell.execute_reply":"2022-01-24T12:44:24.309600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbenchmark_debug = Benchmark()\nbenchmark_debug.benchmark(model_lgbm)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# kernel ridge regression","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datetime import datetime\nfrom sklearn import linear_model\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:38:03.764591Z","iopub.execute_input":"2022-01-24T12:38:03.765740Z","iopub.status.idle":"2022-01-24T12:38:04.714136Z","shell.execute_reply.started":"2022-01-24T12:38:03.765681Z","shell.execute_reply":"2022-01-24T12:38:04.713310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loading traning data strictly\ndef read_csv_strict(file_name='../input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    df = df[df['datetime'] < '2021-06-13 00:00:00']\n    return df\n\ndata_df = read_csv_strict()\ndata_folder = '../input/g-research-crypto-forecasting/'\nasset_details_df = pd.read_csv(data_folder + 'asset_details.csv').set_index('Asset_ID')","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:38:06.687306Z","iopub.execute_input":"2022-01-24T12:38:06.687630Z","iopub.status.idle":"2022-01-24T12:39:14.458425Z","shell.execute_reply.started":"2022-01-24T12:38:06.687598Z","shell.execute_reply":"2022-01-24T12:39:14.457415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RidgeRegression():\n    def __init__(self):\n        self.models = [None] * 14\n        self.features = ['Count', 'Volume', 'Lower_Shadow', 'Upper_Shadow','Avg_Vol', 'Rel_Upper', 'Upper_VWAP', 'Upper_Vol', 'R_quantile', 'Rel_Upper_quantile']\n        \n    def _extend(self,df):\n        df = df.copy()\n        \n        df['Upper_Shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n        df['Lower_Shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n        #df['Liquidity'] = ((2 * (df['High'] - df['Low']) - np.absolute(df['Open'] - df['Close']))/ df['Volume']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Avg_Vol'] = (df['Volume'] / df['Count']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Rel_Upper'] = ((df['High'] - df['VWAP']) / (df['High'] - df['Low'])).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Upper_VWAP'] = ((df['High'] - df['VWAP']) / df['VWAP']).replace([np.inf, -np.inf, np.nan], 0.)\n        df['Upper_Vol'] = ((df['High'] - df['VWAP']) / df['Volume']).replace([np.inf, -np.inf, np.nan], 0.)\n        \n        \n        df['Return_1min'] = (df['Close']/df['Open'] - 1.0).replace([np.inf, -np.inf, np.nan], 0.)\n        df['R_max'] = df.groupby(['timestamp'])['Return_1min'].transform('max')\n        df['R_min'] = df.groupby(['timestamp'])['Return_1min'].transform('min')\n        df['R_quantile'] = (df['Return_1min'] - df['R_min'])/(df['R_max'] - df['R_min'])\n        \n        df['Rel_Upper_max'] = df.groupby(['timestamp'])['Rel_Upper'].transform('max')\n        df['Rel_Upper_min'] = df.groupby(['timestamp'])['Rel_Upper'].transform('min')\n        df['Rel_Upper_quantile'] = (df['Rel_Upper'] - df['Rel_Upper_min'])/(df['Rel_Upper_max'] - df['Rel_Upper_min']).replace([np.inf, -np.inf, np.nan], 0.5)\n        return df\n    \n    def train(self, df):\n        df = df.copy()\n        df = self._extend(df)\n        assets = df['Asset_ID'].drop_duplicates().to_list()\n        \n        for asset in assets:\n            X, y = get_Xy(df, asset, features=self.features)\n            alphas = np.linspace(0.0001,0.1)\n            model = linear_model.RidgeCV(alphas = alphas,store_cv_values=True)\n            model.fit(X, y)\n            self.models[asset] = model\n            print(f'{asset} finished.')\n            \n    def predict(self, df_test, df_pred):\n        df_test = df_test.copy()\n        df_test = self._extend(df_test)\n        total = len(df_test.index)\n        \n        row_generator = df_test.iterrows()\n        row_generator = tqdm(row_generator, total=total) # Enable progress bar, comment this line to disable it.\n        for j , row in row_generator:\n            asset = row['Asset_ID']\n            x_test = row[self.features]\n            model = self.models[int(asset)]\n            y_pred = model.predict([x_test])[0]\n            mask = df_pred['row_id'] == j\n            df_pred.loc[mask, 'Target'] = y_pred\n        return df_pred","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:40:16.158096Z","iopub.execute_input":"2022-01-24T12:40:16.158970Z","iopub.status.idle":"2022-01-24T12:40:16.180132Z","shell.execute_reply.started":"2022-01-24T12:40:16.158923Z","shell.execute_reply":"2022-01-24T12:40:16.179491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_RR = RidgeRegression()\nmodel_RR.train(data_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:40:19.445012Z","iopub.execute_input":"2022-01-24T12:40:19.445464Z","iopub.status.idle":"2022-01-24T12:43:38.493395Z","shell.execute_reply.started":"2022-01-24T12:40:19.445413Z","shell.execute_reply":"2022-01-24T12:43:38.490667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"benchmark_debug = Benchmark()\nbenchmark_debug.benchmark(model_RR)","metadata":{"execution":{"iopub.status.busy":"2022-01-24T12:45:44.932813Z","iopub.execute_input":"2022-01-24T12:45:44.933961Z","iopub.status.idle":"2022-01-24T12:48:25.169993Z","shell.execute_reply.started":"2022-01-24T12:45:44.933908Z","shell.execute_reply":"2022-01-24T12:48:25.169089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df = data[data['Asset_ID'] == 1].dropna()","metadata":{"execution":{"iopub.status.busy":"2022-01-24T06:58:58.739145Z","iopub.execute_input":"2022-01-24T06:58:58.740146Z","iopub.status.idle":"2022-01-24T06:58:58.767009Z","shell.execute_reply.started":"2022-01-24T06:58:58.740097Z","shell.execute_reply":"2022-01-24T06:58:58.766308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_data = np.array(data_df['Close']/data_df['Open']).reshape(-1,1)\ny_data = data_df['Target']\n\nalphas_to_test = np.linspace(0.0001,0.1)\nmodel = linear_model.RidgeCV(alphas = [0.08],store_cv_values=True)\n\nmodel.fit(x_data,y_data)\nprint(model.cv_values_[0])","metadata":{"execution":{"iopub.status.busy":"2022-01-24T07:18:05.841343Z","iopub.execute_input":"2022-01-24T07:18:05.841845Z","iopub.status.idle":"2022-01-24T07:18:05.857465Z","shell.execute_reply.started":"2022-01-24T07:18:05.841805Z","shell.execute_reply":"2022-01-24T07:18:05.856789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(alphas_to_test,model.cv_values_.mean(axis = 0))","metadata":{"execution":{"iopub.status.busy":"2022-01-24T07:09:22.078362Z","iopub.execute_input":"2022-01-24T07:09:22.078988Z","iopub.status.idle":"2022-01-24T07:09:22.231732Z","shell.execute_reply.started":"2022-01-24T07:09:22.078955Z","shell.execute_reply":"2022-01-24T07:09:22.231021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.predict(x_data[:5])\n","metadata":{"execution":{"iopub.status.busy":"2022-01-24T07:18:08.717636Z","iopub.execute_input":"2022-01-24T07:18:08.717861Z","iopub.status.idle":"2022-01-24T07:18:08.725038Z","shell.execute_reply.started":"2022-01-24T07:18:08.717837Z","shell.execute_reply":"2022-01-24T07:18:08.72442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission\n**Important: the last cell could be run only once, due to the API requirement.**\n\nHowever, it's important that this section should be able to run without any error to ensure a successful submission","metadata":{}},{"cell_type":"code","source":"# Register the model here\nmodel_submission = model_lgbm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (df_test, df_pred) in enumerate(iter_test):\n    df_pred = model_submission.predict(df_test, df_pred)\n    env.predict(df_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# debugging cell","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}