{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":30894,"databundleVersionId":3147400,"sourceType":"competition"}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport torch\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport datetime as dt\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom torch.utils.data import Dataset, DataLoader\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:46:46.600502Z","iopub.execute_input":"2023-11-26T16:46:46.601360Z","iopub.status.idle":"2023-11-26T16:46:51.610547Z","shell.execute_reply.started":"2023-11-26T16:46:46.601318Z","shell.execute_reply":"2023-11-26T16:46:51.609643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('..')","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:46:51.612437Z","iopub.execute_input":"2023-11-26T16:46:51.612887Z","iopub.status.idle":"2023-11-26T16:46:51.617615Z","shell.execute_reply.started":"2023-11-26T16:46:51.612855Z","shell.execute_reply":"2023-11-26T16:46:51.616719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## load and preprocess data","metadata":{}},{"cell_type":"code","source":"path = './input/g-research-crypto-forecasting/'\nasset_list = pd.read_csv(path+\"asset_details.csv\")\ntrain_dataset = pd.read_csv(path+\"train.csv\")\nvalidation_dataset = pd.read_csv(path+\"supplemental_train.csv\")\ntest_dataset = pd.read_csv(path+\"example_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:47:25.825286Z","iopub.execute_input":"2023-11-26T16:47:25.826047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# purging to avoid leakage of information\ntrain_dataset = train_dataset.iloc[:round(len(train_dataset)*0.95)]\n\ntrain_dataset.dropna(axis=0, subset=\"Target\", inplace=True)\nvalidation_dataset.dropna(axis=0, subset=\"Target\", inplace=True)\n\ntrain_dataset.reset_index(inplace=True)\nvalidation_dataset.reset_index(inplace=True)\ntest_dataset.reset_index(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:47:11.811286Z","iopub.status.idle":"2023-11-26T16:47:11.812026Z","shell.execute_reply.started":"2023-11-26T16:47:11.811748Z","shell.execute_reply":"2023-11-26T16:47:11.811774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_names = [asset_list[asset_list['Asset_ID']==aid]['Asset_Name'].item() for aid in range(14)]\nasset_names","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:47:11.813214Z","iopub.status.idle":"2023-11-26T16:47:11.813731Z","shell.execute_reply.started":"2023-11-26T16:47:11.813450Z","shell.execute_reply":"2023-11-26T16:47:11.813473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_one_asset(asset):\n    asset.sort_values(by='timestamp', inplace=True)\n    scaler = StandardScaler()\n    \n    asset.loc[np.isinf(asset['VWAP']), 'VWAP'] = asset[~np.isinf(asset['VWAP'])]['VWAP'].mean()\n    asset.loc[np.isnan(asset['VWAP']), 'VWAP'] = asset[~np.isnan(asset['VWAP'])]['VWAP'].mean()\n\n    asset['Vol'] = (asset['Close'].diff()/asset['Close'])**2\n    # add some time series feature\n    primary_features = ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']\n    asset[primary_features] = scaler.fit_transform(asset[primary_features])\n\n    asset['upper_shadow'] = asset['High'] - asset[['Open', 'Close']].max(axis=1)\n    asset['lower_shadow'] = asset[['Open', 'Close']].min(axis=1) - asset['Low']\n    asset['upper_ratio'] = asset['upper_shadow']/(asset['High'] - asset['Low'])\n    asset['lower_ratio'] = asset['lower_shadow']/(asset['High'] - asset['Low'])\n    asset['Average_volume'] = asset['Volume']/asset['Count']\n    asset['Regret'] = asset['VWAP']/asset['Close']\n\n    asset['MA5'] = asset['Close'].rolling(5).mean()\n    asset['MA15'] = asset['Close'].rolling(15).mean()\n    asset['MA30'] = asset['Close'].rolling(30).mean()\n    asset['MA60'] = asset['Close'].rolling(60).mean()\n\n    asset['VMA5'] = asset['Volume'].rolling(5).mean()\n    asset['VMA15'] = asset['Volume'].rolling(15).mean()\n    asset['VMA30'] = asset['Volume'].rolling(30).mean()\n    asset['VMA60'] = asset['Volume'].rolling(60).mean()\n\n    asset['RV5'] = asset['Vol'].rolling(5).sum()\n    asset['RV15'] = asset['Vol'].rolling(15).sum()\n    asset['RV30'] = asset['Vol'].rolling(30).sum()\n    asset['RV60'] = asset['Vol'].rolling(60).sum()\n    \n    asset.fillna(0, inplace=True)\n    \n    secondary_feature = ['Vol', 'upper_shadow', 'lower_shadow', 'upper_ratio', 'lower_ratio', \n                         'Average_volume', 'Regret', 'MA5', 'MA15', 'MA30', 'MA60', 'VMA5', \n                         'VMA15', 'VMA30', 'VMA60', 'RV5', 'RV15', 'RV30', 'RV60']\n    \n    asset[secondary_feature] = scaler.fit_transform(asset[secondary_feature])\n    \n    return asset","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:47:11.815460Z","iopub.status.idle":"2023-11-26T16:47:11.815884Z","shell.execute_reply.started":"2023-11-26T16:47:11.815683Z","shell.execute_reply":"2023-11-26T16:47:11.815707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_time_series_feature(data, verbose=False):\n    processed_assets = []\n    for aid in range(14):\n        if verbose:\n            print(f\"Processing: {asset_names[aid]}\")\n        asset = data[data['Asset_ID'] == aid].copy(deep=True)\n        asset = process_one_asset(asset)\n        processed_assets.append(asset)\n    \n    if verbose:\n        print(\"Concat processed single asset data\")\n    processed_assets = pd.concat(processed_assets)\n    processed_assets.sort_values(by='index', inplace=True)\n    processed_assets.drop(['index'], axis=1, inplace=True)\n\n    if verbose:\n        print(\"One-Hot encoding\")\n\n    onehot = pd.get_dummies(processed_assets['Asset_ID'], dtype=int)\n    for i in range(14):\n        if i not in onehot.columns:\n            onehot[i] = 0\n    onehot = onehot.reindex(columns=list(range(14)))\n    onehot.columns = [asset_names[aid] for aid in onehot.columns]\n    processed_assets = pd.concat([processed_assets, onehot], axis=1)\n    \n    return processed_assets","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:47:11.817146Z","iopub.status.idle":"2023-11-26T16:47:11.817496Z","shell.execute_reply.started":"2023-11-26T16:47:11.817329Z","shell.execute_reply":"2023-11-26T16:47:11.817345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"-------------------------------\\nprocess training data\")\nprocessed_train = process_time_series_feature(train_dataset, verbose=True)\ndel train_dataset\nprint(\"-------------------------------\\nprocess validation data\")\nprocessed_val = process_time_series_feature(validation_dataset, verbose=True)\ndel validation_dataset\nprint(\"-------------------------------\\nprocess test data\")\nprocessed_test = process_time_series_feature(test_dataset, verbose=True)\ndel test_dataset","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:47:11.818386Z","iopub.status.idle":"2023-11-26T16:47:11.818735Z","shell.execute_reply.started":"2023-11-26T16:47:11.818540Z","shell.execute_reply":"2023-11-26T16:47:11.818554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Part","metadata":{}},{"cell_type":"code","source":"# hyperparameters\nBATCH_SIZE = 32\nNUM_FEATURES = 40\nMODEL_DIM = 64\nFFN_DIM = 128\nDROPOUT = 0.2\nNUM_HEADS = 2\nNUM_LAYERS = 2\nMAX_EPOCH = 20\nEARLY_STOP_EPOCH = 5\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:24:21.822579Z","iopub.execute_input":"2023-11-26T16:24:21.823772Z","iopub.status.idle":"2023-11-26T16:24:21.829284Z","shell.execute_reply.started":"2023-11-26T16:24:21.823730Z","shell.execute_reply":"2023-11-26T16:24:21.828309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n        self.timestamps = df['timestamp'].unique()\n        self.feature_cols = ['Count', 'Open', 'High', 'Low', 'Close','Volume', 'VWAP', 'Vol', \n                             'upper_shadow', 'lower_shadow', 'upper_ratio', 'lower_ratio', \n                             'Average_volume', 'Regret', 'MA5', 'MA15', 'MA30', 'MA60', 'VMA5', \n                             'VMA15', 'VMA30', 'VMA60', 'RV5', 'RV15', 'RV30', 'RV60', 'Binance Coin', \n                             'Bitcoin', 'Bitcoin Cash', 'Cardano', 'Dogecoin', 'EOS.IO', 'Ethereum', \n                             'Ethereum Classic', 'IOTA', 'Litecoin', 'Maker', 'Monero', 'Stellar', 'TRON']\n    \n    def __len__(self):\n        return len(self.timestamps)\n    \n    def __getitem__(self, idx):\n        ts = self.timestamps[idx]\n        data = self.df[self.df['timestamp'] == ts]\n        x = data.loc[:, self.feature_cols].values\n        num_assets = x.shape[0]\n        if num_assets < 14:\n            padding = np.zeros((14-num_assets, x.shape[1]))\n            x = np.concatenate((x, padding), axis=0)\n        x = torch.tensor(x, dtype=torch.float32)\n        if 'Target' in self.df.columns:\n            y = self.df.loc[self.df['timestamp'] == ts, 'Target'].values\n            y = torch.tensor(y, dtype=torch.float32)\n            y = F.pad(y, (0, 14-num_assets))\n            return x, y, num_assets\n        else:\n            return x, num_assets","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:00:16.054643Z","iopub.execute_input":"2023-11-26T16:00:16.055055Z","iopub.status.idle":"2023-11-26T16:00:16.065443Z","shell.execute_reply.started":"2023-11-26T16:00:16.055023Z","shell.execute_reply":"2023-11-26T16:00:16.064260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CryptoTransformer(nn.Module):\n    def __init__(self, model_dim=64, num_heads=2, num_layers=2, dropout=0.2, ffn_dim=128):\n        super().__init__()\n        \n        self.input_up = nn.Linear(NUM_FEATURES, model_dim)\n        self.encoder_layer = nn.TransformerEncoderLayer(d_model = model_dim, \n                                                        nhead = num_heads,\n                                                        dim_feedforward = ffn_dim,\n                                                        dropout = dropout,\n                                                        batch_first = True)\n        self.encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=num_layers)\n        self.output = nn.Linear(model_dim, 1)\n        \n    def forward(self, x):\n        emb_x = self.input_up(x)\n        enc_x = self.encoder(emb_x)\n        output = self.output(enc_x)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:02:50.168249Z","iopub.execute_input":"2023-11-26T16:02:50.168671Z","iopub.status.idle":"2023-11-26T16:02:50.176626Z","shell.execute_reply.started":"2023-11-26T16:02:50.168637Z","shell.execute_reply":"2023-11-26T16:02:50.175490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_weights(m):\n    if isinstance(m, nn.Linear):\n        torch.nn.init.xavier_uniform_(m.weight)\n        m.bias.data.fill_(0.)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:07:02.774581Z","iopub.execute_input":"2023-11-26T16:07:02.775248Z","iopub.status.idle":"2023-11-26T16:07:02.780941Z","shell.execute_reply.started":"2023-11-26T16:07:02.775213Z","shell.execute_reply":"2023-11-26T16:07:02.779603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" def train_loop(dataloader, net, loss_fn, optimizer, device):\n    running_loss = 0\n    current = 0\n    net.train()\n\n    with tqdm(dataloader) as t:\n        for batch, (X, y, num_assets) in enumerate(t):\n            batch_size = X.shape[0]\n            X = X.to(device)\n            y = y.to(device)\n            y_pred = net(X)\n\n            y_pred = [y_pred[i, :num_assets[i]] for i in range(batch_size)]\n            y = [y[i, :num_assets[i]] for i in range(batch_size)]\n            y_pred = torch.cat(y_pred).view(-1).to(device)\n            y = torch.cat(y).view(-1).to(device)\n\n            loss = loss_fn(y_pred, y)\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            running_loss = (len(X) * loss.item() + running_loss * current) / (len(X) + current)\n            current += batch_size\n            t.set_postfix({'running_loss':running_loss})\n        \n    return running_loss","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:00:24.662222Z","iopub.execute_input":"2023-11-26T16:00:24.662601Z","iopub.status.idle":"2023-11-26T16:00:24.670843Z","shell.execute_reply.started":"2023-11-26T16:00:24.662573Z","shell.execute_reply":"2023-11-26T16:00:24.669983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def val_loop(dataloader, net, loss_fn, device):\n    running_loss = 0.0\n    current = 0\n    net.eval()\n    \n    with torch.no_grad():\n        with tqdm(dataloader) as t:\n            for batch, (X, y, num_assets) in enumerate(t):\n                batch_size = X.shape[0]\n                X = X.to(device)\n                y = y.to(device)\n                y_pred = net(X)\n\n                y_pred = [y_pred[i, :num_assets[i]] for i in range(batch_size)]\n                y = [y[i, :num_assets[i]] for i in range(batch_size)]\n                y_pred = torch.cat(y_pred).view(-1).to(device)\n                y = torch.cat(y).view(-1).to(device)\n\n                loss = loss_fn(y_pred, y)\n                running_loss = (len(X) * loss.item() + running_loss * current) / (len(X) + current)\n                t.set_postfix({'running_loss':running_loss})\n                \n    return running_loss","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:00:24.999808Z","iopub.execute_input":"2023-11-26T16:00:25.000218Z","iopub.status.idle":"2023-11-26T16:00:25.008475Z","shell.execute_reply.started":"2023-11-26T16:00:25.000186Z","shell.execute_reply":"2023-11-26T16:00:25.007218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = MyDataset(processed_train)\ntrain_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n\nval_dataset = MyDataset(processed_val)\nval_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:02:40.718076Z","iopub.execute_input":"2023-11-26T16:02:40.718482Z","iopub.status.idle":"2023-11-26T16:02:41.205057Z","shell.execute_reply.started":"2023-11-26T16:02:40.718451Z","shell.execute_reply":"2023-11-26T16:02:41.204156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = CryptoTransformer(model_dim=MODEL_DIM, num_heads=NUM_HEADS, num_layers=NUM_LAYERS, dropout=DROPOUT, ffn_dim=FFN_DIM).to(device)\nnet.apply(init_weights)\nloss_fn = nn.MSELoss(reduction='mean')\noptimizer = torch.optim.AdamW(net.parameters())\ntotal_num = sum(p.numel() for p in net.parameters())\ntrainable_num = sum(p.numel() for p in net.parameters() if p.requires_grad)\nprint(f\"Total number of parameters: {total_num/1e3:.0f}K, number of trainable parameters: {trainable_num/1e3:.0f}K\")","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:07:31.615283Z","iopub.execute_input":"2023-11-26T16:07:31.615706Z","iopub.status.idle":"2023-11-26T16:07:31.630391Z","shell.execute_reply.started":"2023-11-26T16:07:31.615671Z","shell.execute_reply":"2023-11-26T16:07:31.629127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('./checkpoint'):\n    os.mkdir('./checkpoint')\nif not os.path.exists('./logs'):\n    os.mkdir('./logs')","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:26:01.530609Z","iopub.execute_input":"2023-11-26T16:26:01.531050Z","iopub.status.idle":"2023-11-26T16:26:01.537802Z","shell.execute_reply.started":"2023-11-26T16:26:01.531014Z","shell.execute_reply":"2023-11-26T16:26:01.536613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_val_loss = float(\"inf\")\nbest_epoch = 0\ntrain_losses = []\nval_losses = []\n\nfor t in range(1, MAX_EPOCH+1):\n    print(f\"Epoch {t}\\n-------------------------------\")\n    train_loss = train_loop(train_dataloader, net, loss_fn, optimizer, device)\n    val_loss = val_loop(val_dataloader, net, loss_fn, device)\n    train_losses.append(train_loss)\n    val_losses.append(val_loss)\n    \n    torch.save(net, f\"./checkpoint/epoch_{t}.pt\")\n    if val_loss < min_val_loss:\n        min_val_loss = val_loss\n        best_epoch = t\n    if t - best_epoch >= EARLY_STOP_EPOCH:\n        break\nprint('Best epoch: {}, val_loss: {}'.format(last_min_ind, min_val_loss))","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:25:11.686645Z","iopub.execute_input":"2023-11-26T16:25:11.687105Z","iopub.status.idle":"2023-11-26T16:25:21.570830Z","shell.execute_reply.started":"2023-11-26T16:25:11.687067Z","shell.execute_reply":"2023-11-26T16:25:21.569246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = list(i+1 for i in range(len(train_losses)))\nplt.plot(epochs, train_losses, label='train_loss')\nplt.plot(epochs, val_losses, label='val_loss')\nplt.ylabel('MSE')\nplt.xlabel('epoch')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-26T16:36:21.913999Z","iopub.execute_input":"2023-11-26T16:36:21.914394Z","iopub.status.idle":"2023-11-26T16:36:22.213690Z","shell.execute_reply.started":"2023-11-26T16:36:21.914363Z","shell.execute_reply":"2023-11-26T16:36:22.212180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction in the test data","metadata":{}},{"cell_type":"code","source":"dataset = MyDataset(processed_test)\ndataloader = DataLoader(dataset, batch_size=2, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T15:34:51.679044Z","iopub.execute_input":"2023-11-26T15:34:51.679542Z","iopub.status.idle":"2023-11-26T15:34:51.685997Z","shell.execute_reply.started":"2023-11-26T15:34:51.679480Z","shell.execute_reply":"2023-11-26T15:34:51.684723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\nnet.eval()\nwith torch.no_grad():\n    with tqdm(dataloader) as t:\n        for batch, (X, num_assets) in enumerate(t):\n            batch_size = len(X)\n            X = X.to(device)\n            y = y.to(device)\n            y_pred = net(X)\n\n            y_pred = [y_pred[i, :num_assets[i]] for i in range(batch_size)]\n            y_pred = torch.cat(y_pred).view(-1).to(device)\n            predictions.append(y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-11-26T15:36:10.618242Z","iopub.execute_input":"2023-11-26T15:36:10.618677Z","iopub.status.idle":"2023-11-26T15:36:10.653141Z","shell.execute_reply.started":"2023-11-26T15:36:10.618643Z","shell.execute_reply":"2023-11-26T15:36:10.652038Z"},"trusted":true},"execution_count":null,"outputs":[]}]}