{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":" ","metadata":{}},{"cell_type":"markdown","source":"## If you find this notebook useful, support with an upvote 🙏","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"markdown","source":" ","metadata":{}},{"cell_type":"code","source":"import io\nimport json\nimport requests\nimport functools\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\nfrom sklearn.model_selection import train_test_split\n\npd.options.mode.chained_assignment = None","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:28.757328Z","iopub.execute_input":"2022-01-04T05:43:28.757942Z","iopub.status.idle":"2022-01-04T05:43:28.764956Z","shell.execute_reply.started":"2022-01-04T05:43:28.757898Z","shell.execute_reply":"2022-01-04T05:43:28.764324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils import data\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn import functional as F\nfrom torchvision import datasets, models, transforms","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:28.766805Z","iopub.execute_input":"2022-01-04T05:43:28.767405Z","iopub.status.idle":"2022-01-04T05:43:28.778413Z","shell.execute_reply.started":"2022-01-04T05:43:28.767352Z","shell.execute_reply":"2022-01-04T05:43:28.777681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"# Start with 10k rows for testing\ndf_train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv', nrows=10000)\ndf_train.dropna(axis = 0, inplace = True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:28.779752Z","iopub.execute_input":"2022-01-04T05:43:28.780210Z","iopub.status.idle":"2022-01-04T05:43:28.820189Z","shell.execute_reply.started":"2022-01-04T05:43:28.780173Z","shell.execute_reply":"2022-01-04T05:43:28.819501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data, validation_data = train_test_split(df_train, test_size=0.2, shuffle=False)\nprint(f\"Training data size: {training_data.shape}\",\n      f\"Validation data size: {validation_data.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:28.821434Z","iopub.execute_input":"2022-01-04T05:43:28.821958Z","iopub.status.idle":"2022-01-04T05:43:28.830095Z","shell.execute_reply.started":"2022-01-04T05:43:28.821919Z","shell.execute_reply":"2022-01-04T05:43:28.829160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Hyperparameters","metadata":{}},{"cell_type":"code","source":"EPOCHS        = 1000\nDROPOUT       = 0.15\nDIRECTIONS    = 1\nNUM_LAYERS    = 2 #为什么只有两层\nBATCH_SIZE    = 5\nOUTPUT_SIZE   = 1\nSEQ_LENGTH    = 60 #60个时间单位\nNUM_FEATURES  = 6\nHIDDEN_SIZE   = 100\nLEARNING_RATE = 0.00001\nSTATE_DIM     = NUM_LAYERS * DIRECTIONS, BATCH_SIZE, HIDDEN_SIZE\nTARGET        = \"Target\"\nFEATURES      = ['Close','High', 'Low', 'Open', 'VWAP', 'Volume']#这个变量是乱序？？？","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:28.832916Z","iopub.execute_input":"2022-01-04T05:43:28.833377Z","iopub.status.idle":"2022-01-04T05:43:28.839901Z","shell.execute_reply.started":"2022-01-04T05:43:28.833334Z","shell.execute_reply":"2022-01-04T05:43:28.838798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class CryptoDataset(Dataset):\n    \"\"\"Onchain dataset.\"\"\"\n    def __init__(self, csv_file, seq_length, features, target):\n        \"\"\"\n        Args:\n        \"\"\"\n        self.csv_file = csv_file\n        self.target = target\n        self.features = features\n        self.seq_length = seq_length\n        self.data_length = len(csv_file)\n\n        self.metrics = self.create_xy_pairs()\n\n    def create_xy_pairs(self):\n        pairs = []\n        for idx in range(self.data_length - self.seq_length):\n            x = self.csv_file[idx:idx + self.seq_length][self.features].values\n            y = self.csv_file[idx + self.seq_length:idx + self.seq_length + 1][self.target].values\n            pairs.append((x, y))\n        return pairs\n\n    def __len__(self):\n        return len(self.metrics)\n\n    def __getitem__(self, idx):\n        return self.metrics[idx]","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:28.841221Z","iopub.execute_input":"2022-01-04T05:43:28.842086Z","iopub.status.idle":"2022-01-04T05:43:28.852400Z","shell.execute_reply.started":"2022-01-04T05:43:28.842016Z","shell.execute_reply":"2022-01-04T05:43:28.851567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'batch_size': BATCH_SIZE,\n          'shuffle': False,\n          'drop_last': True, # Disregard last incomplete batch\n          'num_workers': 2}\nparams_test = {'batch_size': 1,\n          'shuffle': False,\n          'drop_last': False, # Disregard last incomplete batch\n          'num_workers': 2}\n\ntraining_ds = CryptoDataset(training_data, SEQ_LENGTH, FEATURES, TARGET)\ntraining_dl = DataLoader(training_ds, **params)\nvalidation_ds = CryptoDataset(validation_data, SEQ_LENGTH, FEATURES, TARGET)\nvalidation_dl = DataLoader(validation_ds, **params)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:28.853611Z","iopub.execute_input":"2022-01-04T05:43:28.854185Z","iopub.status.idle":"2022-01-04T05:43:34.545725Z","shell.execute_reply.started":"2022-01-04T05:43:28.854104Z","shell.execute_reply":"2022-01-04T05:43:34.544922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Settings","metadata":{}},{"cell_type":"code","source":"# Transfer to accelerator\nuse_cuda = torch.cuda.is_available()\ndevice = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\nnp.random.seed(0)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:34.547255Z","iopub.execute_input":"2022-01-04T05:43:34.547540Z","iopub.status.idle":"2022-01-04T05:43:34.556630Z","shell.execute_reply.started":"2022-01-04T05:43:34.547502Z","shell.execute_reply":"2022-01-04T05:43:34.555828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LSTM(nn.Module):\n  def __init__(self, input_size, hidden_size, num_layers, output_size, dropout_prob, directions=1):\n    super(LSTM, self).__init__()\n\n    self.num_layers = num_layers\n    self.hidden_size = hidden_size\n    self.directions = directions\n\n    self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True, dropout=dropout_prob)\n    self.dropout = nn.Dropout(dropout_prob)\n    self.linear = nn.Linear(hidden_size, output_size)\n\n  def init_hidden_states(self, batch_size):\n    state_dim = (self.num_layers * self.directions, batch_size, self.hidden_size)\n    return (torch.zeros(state_dim).to(device), torch.zeros(state_dim).to(device))\n\n  def forward(self, x, states):\n    x, (h, c) = self.lstm(x, states)\n    out = self.linear(x)\n    return out, (h, c)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:34.558079Z","iopub.execute_input":"2022-01-04T05:43:34.558357Z","iopub.status.idle":"2022-01-04T05:43:34.569664Z","shell.execute_reply.started":"2022-01-04T05:43:34.558317Z","shell.execute_reply":"2022-01-04T05:43:34.568836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LSTM(\n    NUM_FEATURES,\n    HIDDEN_SIZE,\n    NUM_LAYERS,\n    OUTPUT_SIZE,\n    DROPOUT\n).to(device)\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.linear.parameters(), lr=LEARNING_RATE, weight_decay=0.01)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:34.572812Z","iopub.execute_input":"2022-01-04T05:43:34.573393Z","iopub.status.idle":"2022-01-04T05:43:38.225199Z","shell.execute_reply.started":"2022-01-04T05:43:34.573349Z","shell.execute_reply":"2022-01-04T05:43:38.224451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def save_checkpoint(epoch, min_val_loss, model_state, opt_state):\n  print(f\"New minimum reached at epoch #{epoch + 1}, saving model state...\")\n  checkpoint = {\n    'epoch': epoch + 1,\n    'min_val_loss': min_val_loss,\n    'model_state': model_state,\n    'opt_state': opt_state,\n  }\n  torch.save(checkpoint, \"./model_state.pt\")\ndef load_checkpoint(path, model, optimizer):\n    # load check point\n    checkpoint = torch.load(path)\n    min_val_loss = checkpoint[\"min_val_loss\"]\n    model.load_state_dict(checkpoint[\"model_state\"])\n    optimizer.load_state_dict(checkpoint[\"opt_state\"])\n    return model, optimizer, checkpoint[\"epoch\"], min_val_loss\n\n\ndef training(model, epochs, validate_every=2):\n\n  training_losses = []\n  validation_losses = []\n  min_validation_loss = np.Inf\n\n  # Set to train mode\n  model.train()\n\n  for epoch in tqdm(range(epochs)):\n\n    # Initialize hidden and cell states with dimension:\n    # (num_layers * num_directions, batch, hidden_size)\n    states = model.init_hidden_states(BATCH_SIZE)\n    running_training_loss = 0.0\n\n    # Begin training\n    for idx, (x_batch, y_batch) in enumerate(training_dl):\n      # Convert to Tensors\n      x_batch = x_batch.float().to(device)\n      y_batch = y_batch.float().to(device)\n      \n      # Truncated Backpropagation\n      states = [state.detach() for state in states]          \n\n      optimizer.zero_grad()\n\n      # Make prediction\n      output, states = model(x_batch, states)\n\n      # Calculate loss\n      loss = criterion(output[:, -1, :], y_batch)\n      loss.backward()\n      running_training_loss += loss.item()\n\n      torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n      optimizer.step()\n        \n    # Average loss across timesteps\n    training_losses.append(running_training_loss / len(training_dl))\n        \n    if epoch % validate_every == 0:\n\n      # Set to eval mode\n      model.eval()\n\n      validation_states = model.init_hidden_states(BATCH_SIZE)\n      running_validation_loss = 0.0\n\n      for idx, (x_batch, y_batch) in enumerate(validation_dl):\n\n        # Convert to Tensors\n        x_batch = x_batch.float().to(device)\n        y_batch = y_batch.float().to(device)\n      \n        validation_states = [state.detach() for state in validation_states]\n        output, validation_states = model(x_batch, validation_states)\n        validation_loss = criterion(output[:, -1, :], y_batch)\n        running_validation_loss += validation_loss.item()\n        \n    validation_losses.append(running_validation_loss / len(validation_dl))\n    # Reset to training mode\n    model.train()\n\n    is_best = running_validation_loss / len(validation_dl) < min_validation_loss\n\n    if is_best:\n      min_validation_loss = running_validation_loss / len(validation_dl)\n      save_checkpoint(epoch + 1, min_validation_loss, model.state_dict(), optimizer.state_dict())\n        \n\n  # Visualize loss\n  epoch_count = range(1, len(training_losses) + 1)\n  plt.plot(epoch_count, training_losses, 'r--')\n  plt.legend(['Training Loss'])\n  plt.xlabel('Epoch')\n  plt.ylabel('Loss')\n  plt.show()\n\n  val_epoch_count = range(1, len(validation_losses) + 1)\n  plt.plot(val_epoch_count, validation_losses, 'b--')\n  plt.legend(['Validation loss'])\n  plt.xlabel('Epoch')\n  plt.ylabel('Loss')\n  plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:38.226591Z","iopub.execute_input":"2022-01-04T05:43:38.226889Z","iopub.status.idle":"2022-01-04T05:43:38.245282Z","shell.execute_reply.started":"2022-01-04T05:43:38.226851Z","shell.execute_reply":"2022-01-04T05:43:38.244487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training(model, 100)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T05:43:38.247156Z","iopub.execute_input":"2022-01-04T05:43:38.247549Z","iopub.status.idle":"2022-01-04T06:04:39.935949Z","shell.execute_reply.started":"2022-01-04T05:43:38.247511Z","shell.execute_reply":"2022-01-04T06:04:39.935245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"./model_state.pt\"\nmodel, optimizer, start_epoch, valid_loss_min = load_checkpoint(path, model, optimizer)\nprint(\"model = \", model)\nprint(\"optimizer = \", optimizer)\nprint(\"start_epoch = \", start_epoch)\nprint(\"valid_loss_min = \", valid_loss_min)\nprint(\"valid_loss_min = {:.6f}\".format(valid_loss_min))","metadata":{"execution":{"iopub.status.busy":"2022-01-04T06:04:39.937603Z","iopub.execute_input":"2022-01-04T06:04:39.937873Z","iopub.status.idle":"2022-01-04T06:04:39.950192Z","shell.execute_reply.started":"2022-01-04T06:04:39.937838Z","shell.execute_reply":"2022-01-04T06:04:39.949362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T06:04:39.951618Z","iopub.execute_input":"2022-01-04T06:04:39.951943Z","iopub.status.idle":"2022-01-04T06:04:39.975092Z","shell.execute_reply.started":"2022-01-04T06:04:39.951901Z","shell.execute_reply":"2022-01-04T06:04:39.974434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nfor (test_df, sample_prediction_df) in iter_test:\n    selected_features = test_df[FEATURES]\n    x = torch.Tensor(selected_features.values)\n    x = x.float().to(device)\n    x = x.view(1, -1, NUM_FEATURES) # Batch size x Sequence length x Number of features\n    validation_states = model.init_hidden_states(1)\n    validation_states = [state.detach() for state in validation_states]\n    output, _ = model(x, validation_states)\n    sample_prediction_df['Target'] = output[:, -1, :].item()\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T06:04:39.977732Z","iopub.execute_input":"2022-01-04T06:04:39.977927Z","iopub.status.idle":"2022-01-04T06:04:40.018059Z","shell.execute_reply.started":"2022-01-04T06:04:39.977904Z","shell.execute_reply":"2022-01-04T06:04:40.017411Z"},"trusted":true},"execution_count":null,"outputs":[]}]}