{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":" ","metadata":{}},{"cell_type":"markdown","source":"## If you find this notebook useful, support with an upvote 🙏","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"markdown","source":" ","metadata":{}},{"cell_type":"code","source":"import io\nimport json\nimport requests\nimport functools\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\nfrom sklearn.model_selection import train_test_split\n\npd.options.mode.chained_assignment = None","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:06.253799Z","iopub.execute_input":"2021-11-04T05:19:06.254152Z","iopub.status.idle":"2021-11-04T05:19:07.019729Z","shell.execute_reply.started":"2021-11-04T05:19:06.254073Z","shell.execute_reply":"2021-11-04T05:19:07.019006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom torch.utils import data\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn import functional as F\nfrom torchvision import datasets, models, transforms","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:07.021231Z","iopub.execute_input":"2021-11-04T05:19:07.021474Z","iopub.status.idle":"2021-11-04T05:19:08.605348Z","shell.execute_reply.started":"2021-11-04T05:19:07.021440Z","shell.execute_reply":"2021-11-04T05:19:08.604559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"# Start with 10k rows for testing\ndf_train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv', nrows=10000)\ndf_train.dropna(axis = 0, inplace = True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:08.606923Z","iopub.execute_input":"2021-11-04T05:19:08.607190Z","iopub.status.idle":"2021-11-04T05:19:08.679371Z","shell.execute_reply.started":"2021-11-04T05:19:08.607157Z","shell.execute_reply":"2021-11-04T05:19:08.678428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data, validation_data = train_test_split(df_train, test_size=0.2, shuffle=False)\n\nprint(f\"Training data size: {training_data.shape}\",\n      f\"Validation data size: {validation_data.shape}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:08.681812Z","iopub.execute_input":"2021-11-04T05:19:08.682111Z","iopub.status.idle":"2021-11-04T05:19:08.689994Z","shell.execute_reply.started":"2021-11-04T05:19:08.682074Z","shell.execute_reply":"2021-11-04T05:19:08.688963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters","metadata":{}},{"cell_type":"code","source":"EPOCHS        = 1000\nDROPOUT       = 0.2\nDIRECTIONS    = 1\nNUM_LAYERS    = 2\nBATCH_SIZE    = 5\nOUTPUT_SIZE   = 1\nSEQ_LENGTH    = 60\nNUM_FEATURES  = 6\nHIDDEN_SIZE   = 100\nLEARNING_RATE = 0.0001\nSTATE_DIM     = NUM_LAYERS * DIRECTIONS, BATCH_SIZE, HIDDEN_SIZE\nTARGET        = \"Target\"\nFEATURES      = ['Close','High', 'Low', 'Open', 'VWAP', 'Volume']","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:08.692367Z","iopub.execute_input":"2021-11-04T05:19:08.693646Z","iopub.status.idle":"2021-11-04T05:19:08.700624Z","shell.execute_reply.started":"2021-11-04T05:19:08.693602Z","shell.execute_reply":"2021-11-04T05:19:08.699557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class CryptoDataset(Dataset):\n    \"\"\"Onchain dataset.\"\"\"\n\n    def __init__(self, csv_file, seq_length, features, target):\n        \"\"\"\n        Args:\n        \"\"\"\n        self.csv_file = csv_file\n        self.target = target\n        self.features = features\n        self.seq_length = seq_length\n        self.data_length = len(csv_file)\n\n        self.metrics = self.create_xy_pairs()\n\n    def create_xy_pairs(self):\n        pairs = []\n        for idx in range(self.data_length - self.seq_length):\n            x = self.csv_file[idx:idx + self.seq_length][self.features].values\n            y = self.csv_file[idx + self.seq_length:idx + self.seq_length + 1][self.target].values\n            pairs.append((x, y))\n        return pairs\n\n    def __len__(self):\n        return len(self.metrics)\n\n    def __getitem__(self, idx):\n        return self.metrics[idx]","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:08.702186Z","iopub.execute_input":"2021-11-04T05:19:08.702471Z","iopub.status.idle":"2021-11-04T05:19:08.712605Z","shell.execute_reply.started":"2021-11-04T05:19:08.702434Z","shell.execute_reply":"2021-11-04T05:19:08.711631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'batch_size': BATCH_SIZE,\n          'shuffle': False,\n          'drop_last': True, # Disregard last incomplete batch\n          'num_workers': 2}\n\nparams_test = {'batch_size': 1,\n          'shuffle': False,\n          'drop_last': False, # Disregard last incomplete batch\n          'num_workers': 2}\n\ntraining_ds = CryptoDataset(training_data, SEQ_LENGTH, FEATURES, TARGET)\ntraining_dl = DataLoader(training_ds, **params)\n\nvalidation_ds = CryptoDataset(validation_data, SEQ_LENGTH, FEATURES, TARGET)\nvalidation_dl = DataLoader(validation_ds, **params)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:08.713890Z","iopub.execute_input":"2021-11-04T05:19:08.714354Z","iopub.status.idle":"2021-11-04T05:19:15.370036Z","shell.execute_reply.started":"2021-11-04T05:19:08.714315Z","shell.execute_reply":"2021-11-04T05:19:15.369278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Settings","metadata":{}},{"cell_type":"code","source":"# Transfer to accelerator\nuse_cuda = torch.cuda.is_available()\ndevice = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\nnp.random.seed(0)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:15.371265Z","iopub.execute_input":"2021-11-04T05:19:15.371537Z","iopub.status.idle":"2021-11-04T05:19:15.425258Z","shell.execute_reply.started":"2021-11-04T05:19:15.371490Z","shell.execute_reply":"2021-11-04T05:19:15.424428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LSTM(nn.Module):\n  def __init__(self, input_size, hidden_size, num_layers, output_size, dropout_prob, directions=1):\n    super(LSTM, self).__init__()\n\n    self.num_layers = num_layers\n    self.hidden_size = hidden_size\n    self.directions = directions\n\n    self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True, dropout=dropout_prob)\n    self.dropout = nn.Dropout(dropout_prob)\n    self.linear = nn.Linear(hidden_size, output_size)\n\n  def init_hidden_states(self, batch_size):\n    state_dim = (self.num_layers * self.directions, batch_size, self.hidden_size)\n    return (torch.zeros(state_dim).to(device), torch.zeros(state_dim).to(device))\n\n  def forward(self, x, states):\n    x, (h, c) = self.lstm(x, states)\n    out = self.linear(x)\n    return out, (h, c)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:15.427278Z","iopub.execute_input":"2021-11-04T05:19:15.427933Z","iopub.status.idle":"2021-11-04T05:19:15.437310Z","shell.execute_reply.started":"2021-11-04T05:19:15.427754Z","shell.execute_reply":"2021-11-04T05:19:15.436379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LSTM(\n    NUM_FEATURES,\n    HIDDEN_SIZE,\n    NUM_LAYERS,\n    OUTPUT_SIZE,\n    DROPOUT\n).to(device)\n\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.linear.parameters(), lr=LEARNING_RATE, weight_decay=0.01)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:15.439851Z","iopub.execute_input":"2021-11-04T05:19:15.440349Z","iopub.status.idle":"2021-11-04T05:19:19.052528Z","shell.execute_reply.started":"2021-11-04T05:19:15.440276Z","shell.execute_reply":"2021-11-04T05:19:19.051787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def save_checkpoint(epoch, min_val_loss, model_state, opt_state):\n  print(f\"New minimum reached at epoch #{epoch + 1}, saving model state...\")\n  checkpoint = {\n    'epoch': epoch + 1,\n    'min_val_loss': min_val_loss,\n    'model_state': model_state,\n    'opt_state': opt_state,\n  }\n  torch.save(checkpoint, \"./model_state.pt\")\n\n\ndef load_checkpoint(path, model, optimizer):\n    # load check point\n    checkpoint = torch.load(path)\n    min_val_loss = checkpoint[\"min_val_loss\"]\n    model.load_state_dict(checkpoint[\"model_state\"])\n    optimizer.load_state_dict(checkpoint[\"opt_state\"])\n    return model, optimizer, checkpoint[\"epoch\"], min_val_loss\n\n\ndef training(model, epochs, validate_every=2):\n\n  training_losses = []\n  validation_losses = []\n  min_validation_loss = np.Inf\n\n  # Set to train mode\n  model.train()\n\n  for epoch in tqdm(range(epochs)):\n\n    # Initialize hidden and cell states with dimension:\n    # (num_layers * num_directions, batch, hidden_size)\n    states = model.init_hidden_states(BATCH_SIZE)\n    running_training_loss = 0.0\n\n    # Begin training\n    for idx, (x_batch, y_batch) in enumerate(training_dl):\n      # Convert to Tensors\n      x_batch = x_batch.float().to(device)\n      y_batch = y_batch.float().to(device)\n      \n      # Truncated Backpropagation\n      states = [state.detach() for state in states]          \n\n      optimizer.zero_grad()\n\n      # Make prediction\n      output, states = model(x_batch, states)\n\n      # Calculate loss\n      loss = criterion(output[:, -1, :], y_batch)\n      loss.backward()\n      running_training_loss += loss.item()\n\n      torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n      optimizer.step()\n        \n    # Average loss across timesteps\n    training_losses.append(running_training_loss / len(training_dl))\n        \n    if epoch % validate_every == 0:\n\n      # Set to eval mode\n      model.eval()\n\n      validation_states = model.init_hidden_states(BATCH_SIZE)\n      running_validation_loss = 0.0\n\n      for idx, (x_batch, y_batch) in enumerate(validation_dl):\n\n        # Convert to Tensors\n        x_batch = x_batch.float().to(device)\n        y_batch = y_batch.float().to(device)\n      \n        validation_states = [state.detach() for state in validation_states]\n        output, validation_states = model(x_batch, validation_states)\n        validation_loss = criterion(output[:, -1, :], y_batch)\n        running_validation_loss += validation_loss.item()\n        \n    validation_losses.append(running_validation_loss / len(validation_dl))\n    # Reset to training mode\n    model.train()\n\n    is_best = running_validation_loss / len(validation_dl) < min_validation_loss\n\n    if is_best:\n      min_validation_loss = running_validation_loss / len(validation_dl)\n      save_checkpoint(epoch + 1, min_validation_loss, model.state_dict(), optimizer.state_dict())\n        \n\n  # Visualize loss\n  epoch_count = range(1, len(training_losses) + 1)\n  plt.plot(epoch_count, training_losses, 'r--')\n  plt.legend(['Training Loss'])\n  plt.xlabel('Epoch')\n  plt.ylabel('Loss')\n  plt.show()\n\n  val_epoch_count = range(1, len(validation_losses) + 1)\n  plt.plot(val_epoch_count, validation_losses, 'b--')\n  plt.legend(['Validation loss'])\n  plt.xlabel('Epoch')\n  plt.ylabel('Loss')\n  plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:19.054601Z","iopub.execute_input":"2021-11-04T05:19:19.055015Z","iopub.status.idle":"2021-11-04T05:19:19.072877Z","shell.execute_reply.started":"2021-11-04T05:19:19.054978Z","shell.execute_reply":"2021-11-04T05:19:19.072203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training(model, 100)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:19:19.074466Z","iopub.execute_input":"2021-11-04T05:19:19.074960Z","iopub.status.idle":"2021-11-04T05:41:10.429765Z","shell.execute_reply.started":"2021-11-04T05:19:19.074922Z","shell.execute_reply":"2021-11-04T05:41:10.429022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"./model_state.pt\"\nmodel, optimizer, start_epoch, valid_loss_min = load_checkpoint(path, model, optimizer)\nprint(\"model = \", model)\nprint(\"optimizer = \", optimizer)\nprint(\"start_epoch = \", start_epoch)\nprint(\"valid_loss_min = \", valid_loss_min)\nprint(\"valid_loss_min = {:.6f}\".format(valid_loss_min))","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:41:10.431440Z","iopub.execute_input":"2021-11-04T05:41:10.431724Z","iopub.status.idle":"2021-11-04T05:41:10.444222Z","shell.execute_reply.started":"2021-11-04T05:41:10.431685Z","shell.execute_reply":"2021-11-04T05:41:10.443202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:41:10.446019Z","iopub.execute_input":"2021-11-04T05:41:10.446324Z","iopub.status.idle":"2021-11-04T05:41:10.476037Z","shell.execute_reply.started":"2021-11-04T05:41:10.446286Z","shell.execute_reply":"2021-11-04T05:41:10.475269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nfor (test_df, sample_prediction_df) in iter_test:\n    selected_features = test_df[FEATURES]\n    x = torch.Tensor(selected_features.values)\n    x = x.float().to(device)\n    x = x.view(1, -1, NUM_FEATURES) # Batch size x Sequence length x Number of features\n    validation_states = model.init_hidden_states(1)\n    validation_states = [state.detach() for state in validation_states]\n    output, _ = model(x, validation_states)\n    sample_prediction_df['Target'] = output[:, -1, :].item()\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T05:41:10.477285Z","iopub.execute_input":"2021-11-04T05:41:10.477870Z","iopub.status.idle":"2021-11-04T05:41:10.520890Z","shell.execute_reply.started":"2021-11-04T05:41:10.477830Z","shell.execute_reply":"2021-11-04T05:41:10.520238Z"},"trusted":true},"execution_count":null,"outputs":[]}]}