{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import io\nimport json\nimport requests\nimport functools\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\nfrom sklearn.model_selection import train_test_split\n\npd.options.mode.chained_assignment = None\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom torch.utils import data\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn import functional as F\nfrom torchvision import datasets, models, transforms\nfrom queue import Queue","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:57:59.467555Z","iopub.execute_input":"2022-01-21T14:57:59.4683Z","iopub.status.idle":"2022-01-21T14:58:01.826786Z","shell.execute_reply.started":"2022-01-21T14:57:59.468198Z","shell.execute_reply":"2022-01-21T14:58:01.825884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS        = 5\nDROPOUT       = 0.2\nDIRECTIONS    = 1\nNUM_LAYERS    = 2\nBATCH_SIZE    = 5\nOUTPUT_SIZE   = 1\nSEQ_LENGTH    = 30\n\nHIDDEN_SIZE   = 100 \nLEARNING_RATE = 0.001\nSHIFT_K       = 5  # feaute_1 ~ feaute_k\nSTATE_DIM     = NUM_LAYERS * DIRECTIONS, BATCH_SIZE, HIDDEN_SIZE\nTARGET        = \"Target\"\nFEATURES      = ['Close','High', 'Low', 'Open', 'VWAP', 'Volume']\n# add feaute_1 ~ feaute_k\nfor i in range(1, SHIFT_K+1):\n    FEATURES.append(f\"feature_{i}\")\nNUM_FEATURES  = len(FEATURES)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:58:01.82922Z","iopub.execute_input":"2022-01-21T14:58:01.829753Z","iopub.status.idle":"2022-01-21T14:58:01.836148Z","shell.execute_reply.started":"2022-01-21T14:58:01.82971Z","shell.execute_reply":"2022-01-21T14:58:01.835387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:58:01.837108Z","iopub.execute_input":"2022-01-21T14:58:01.837349Z","iopub.status.idle":"2022-01-21T14:58:01.84592Z","shell.execute_reply.started":"2022-01-21T14:58:01.837315Z","shell.execute_reply":"2022-01-21T14:58:01.845154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"# Start with 10k rows for testing\ndf_train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv', nrows=10000000) #### train data\n# df_train = pd.read_csv('../input/my_data/my_data.csv')\ndetails = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv') # asset_details\ndf_train.dropna(axis = 0, inplace = True) # dropna\ndf_train['Upper_Shadow'] = upper_shadow(df_train) # Upper_Shadow\ndf_train['Lower_Shadow'] = lower_shadow(df_train) # Lower_Shadow","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:58:01.847114Z","iopub.execute_input":"2022-01-21T14:58:01.847403Z","iopub.status.idle":"2022-01-21T14:58:25.166776Z","shell.execute_reply.started":"2022-01-21T14:58:01.847365Z","shell.execute_reply":"2022-01-21T14:58:25.166012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_df_dict = {}\n\n# split train data to 14 asset dataframe \nfor asset_id in range(14):\n    asset_df = df_train[df_train[\"Asset_ID\"]==asset_id][:10000].reset_index(drop=True) #### debug just 10000 row\n    # asset_df = df_train[df_train[\"Asset_ID\"]==asset_id].reset_index(drop=True)\n    asset_df_dict[asset_id] = asset_df\nlen(asset_df_dict.keys())","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:58:25.169006Z","iopub.execute_input":"2022-01-21T14:58:25.169217Z","iopub.status.idle":"2022-01-21T14:58:27.060219Z","shell.execute_reply.started":"2022-01-21T14:58:25.169191Z","shell.execute_reply":"2022-01-21T14:58:27.059497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get feaute_1 ~ feaute_k\ndef feature_k(df, k=SHIFT_K, target_variable=\"Target\"):\n    for i in range(1, k+1):\n        df[f\"feature_{i}\"] = df[target_variable].shift(i)\n    return df\n\ntarget_dict = {}\nfor asset_id in range(14):\n    # save target of last row \n    asset_df = feature_k(asset_df_dict[asset_id]).fillna(0)\n    asset_df_dict[asset_id] = asset_df\n    target_dict[asset_id] = Queue(maxsize=SHIFT_K+1) # queue\n    last_row = asset_df.iloc[-1]\n    for k in range(SHIFT_K-1, 0, -1):\n        target_dict[asset_id].put(last_row[f\"feature_{k}\"])\n    target_dict[asset_id].put(last_row[\"Target\"])    ","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:58:27.061363Z","iopub.execute_input":"2022-01-21T14:58:27.061604Z","iopub.status.idle":"2022-01-21T14:58:27.117031Z","shell.execute_reply.started":"2022-01-21T14:58:27.06157Z","shell.execute_reply":"2022-01-21T14:58:27.116405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data_list = []\nvalidation_data_list = []\n\n# train_test_split for 14 asset\nfor i in range(14):\n    training_data, validation_data = train_test_split(asset_df_dict[i], test_size=0.2, shuffle=False)\n    training_data_list.append(training_data)\n    validation_data_list.append(validation_data)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:58:27.118169Z","iopub.execute_input":"2022-01-21T14:58:27.118498Z","iopub.status.idle":"2022-01-21T14:58:27.136048Z","shell.execute_reply.started":"2022-01-21T14:58:27.118461Z","shell.execute_reply":"2022-01-21T14:58:27.135454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{}},{"cell_type":"code","source":"class CryptoDataset(Dataset):\n    \"\"\"Onchain dataset.\"\"\"\n\n    def __init__(self, csv_file, seq_length, features, target):\n        \"\"\"\n        Args:\n        \"\"\"\n        self.csv_file = csv_file\n        self.target = target\n        self.features = features\n        self.seq_length = seq_length\n        self.data_length = len(csv_file)\n\n        self.metrics = self.create_xy_pairs()\n\n    def create_xy_pairs(self):\n        pairs = []\n        for idx in range(self.data_length - self.seq_length):\n            x = self.csv_file[idx:idx + self.seq_length][self.features].values\n            y = self.csv_file[idx + self.seq_length:idx + self.seq_length + 1][self.target].values\n            pairs.append((x, y))\n        return pairs\n\n    def __len__(self):\n        return len(self.metrics)\n\n    def __getitem__(self, idx):\n        return self.metrics[idx]","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:58:27.137182Z","iopub.execute_input":"2022-01-21T14:58:27.137482Z","iopub.status.idle":"2022-01-21T14:58:27.145966Z","shell.execute_reply.started":"2022-01-21T14:58:27.137446Z","shell.execute_reply":"2022-01-21T14:58:27.145276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'batch_size': BATCH_SIZE,\n          'shuffle': False,\n          'drop_last': True, # Disregard last incomplete batch\n          'num_workers': 2}\n\nparams_test = {'batch_size': 1,\n          'shuffle': False,\n          'drop_last': False, # Disregard last incomplete batch\n          'num_workers': 2}\n\n#  datasets and dataloader for 14 asset\ntraining_ds_list = [CryptoDataset(training_data, SEQ_LENGTH, FEATURES, TARGET) for training_data in training_data_list]\ntraining_dl_list = [DataLoader(training_ds, **params) for training_ds in training_ds_list]\n\nvalidation_ds_list = [CryptoDataset(validation_data, SEQ_LENGTH, FEATURES, TARGET) for validation_data in validation_data_list]\nvalidation_dl_list = [DataLoader(validation_ds, **params) for validation_ds in validation_ds_list]","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:58:27.147312Z","iopub.execute_input":"2022-01-21T14:58:27.14775Z","iopub.status.idle":"2022-01-21T14:59:49.467887Z","shell.execute_reply.started":"2022-01-21T14:58:27.147713Z","shell.execute_reply":"2022-01-21T14:59:49.46715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Settings","metadata":{}},{"cell_type":"code","source":"# Transfer to accelerator\nuse_cuda = torch.cuda.is_available()\ndevice = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\nnp.random.seed(0)\n\nclass LSTM(nn.Module):\n    def __init__(self, input_size, hidden_size, num_layers, output_size, dropout_prob, directions=1):\n        super(LSTM, self).__init__()\n\n        self.num_layers = num_layers\n        self.hidden_size = hidden_size\n        self.directions = directions\n\n        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True, dropout=dropout_prob)\n        self.dropout = nn.Dropout(dropout_prob)\n        self.linear = nn.Linear(hidden_size, output_size)\n\n    def init_hidden_states(self, batch_size):\n        state_dim = (self.num_layers * self.directions, batch_size, self.hidden_size)\n        return (torch.zeros(state_dim).to(device), torch.zeros(state_dim).to(device))\n\n    def forward(self, x, states):\n        x, (h, c) = self.lstm(x, states)\n        out = self.linear(x)\n        return out, (h, c)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:59:49.469116Z","iopub.execute_input":"2022-01-21T14:59:49.469377Z","iopub.status.idle":"2022-01-21T14:59:49.53109Z","shell.execute_reply.started":"2022-01-21T14:59:49.469341Z","shell.execute_reply":"2022-01-21T14:59:49.53031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 14 models\nmodels_list = [LSTM(NUM_FEATURES,HIDDEN_SIZE,NUM_LAYERS,OUTPUT_SIZE,DROPOUT).to(device) for _ in range(14)]\n# 14 criterion\ncriterion_list = [nn.MSELoss() for _ in range(14)]\n# 14 optimizer\noptimizer_list = [optim.AdamW(model.linear.parameters(), lr=LEARNING_RATE, weight_decay=0.01) for model in models_list]","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:59:49.53408Z","iopub.execute_input":"2022-01-21T14:59:49.534303Z","iopub.status.idle":"2022-01-21T14:59:53.021574Z","shell.execute_reply.started":"2022-01-21T14:59:49.534265Z","shell.execute_reply":"2022-01-21T14:59:53.020807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def save_checkpoint(epoch, min_val_loss, model_state, opt_state, asset_id):\n    print(f\"New minimum reached at epoch #{epoch + 1}, saving model state...\")\n    checkpoint = {\n    'epoch': epoch + 1,\n    'min_val_loss': min_val_loss,\n    'model_state': model_state,\n    'opt_state': opt_state,\n    }\n    torch.save(checkpoint, f\"./model_state_{asset_id}.pt\")\n\n\ndef load_checkpoint(path, model, optimizer):\n    # load check point\n    checkpoint = torch.load(path)\n    min_val_loss = checkpoint[\"min_val_loss\"]\n    model.load_state_dict(checkpoint[\"model_state\"])\n    optimizer.load_state_dict(checkpoint[\"opt_state\"])\n    return model, optimizer, checkpoint[\"epoch\"], min_val_loss\n\n\ndef training(asset_id, model, criterion, optimizer, epochs, validate_every=2):\n\n    training_losses = []\n    validation_losses = []\n    min_validation_loss = np.Inf\n\n    # Set to train mode\n    model.train()\n\n    for epoch in tqdm(range(epochs)):\n\n        # Initialize hidden and cell states with dimension:\n        # (num_layers * num_directions, batch, hidden_size)\n        states = model.init_hidden_states(BATCH_SIZE)\n        running_training_loss = 0.0\n\n        # Begin training\n        for idx, (x_batch, y_batch) in enumerate(training_dl_list[asset_id]):\n            # Convert to Tensors\n            x_batch = x_batch.float().to(device)\n            y_batch = y_batch.float().to(device)\n\n            # Truncated Backpropagation\n            states = [state.detach() for state in states]          \n\n            optimizer.zero_grad()\n\n            # Make prediction\n            output, states = model(x_batch, states)\n\n            # Calculate loss\n            loss = criterion(output[:, -1, :], y_batch)\n            loss.backward()\n            running_training_loss += loss.item()\n\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n            optimizer.step()\n\n        # Average loss across timesteps\n        training_losses.append(running_training_loss / len(training_dl_list[asset_id]))\n\n        if epoch % validate_every == 0:\n\n            # Set to eval mode\n            model.eval()\n\n            validation_states = model.init_hidden_states(BATCH_SIZE)\n            running_validation_loss = 0.0\n\n            for idx, (x_batch, y_batch) in enumerate(validation_dl_list[asset_id]):\n\n                # Convert to Tensors\n                x_batch = x_batch.float().to(device)\n                y_batch = y_batch.float().to(device)\n\n                validation_states = [state.detach() for state in validation_states]\n                output, validation_states = model(x_batch, validation_states)\n                validation_loss = criterion(output[:, -1, :], y_batch)\n                running_validation_loss += validation_loss.item()\n\n        validation_losses.append(running_validation_loss / len(validation_dl_list[asset_id]))\n        # Reset to training mode\n        model.train()\n\n        is_best = running_validation_loss / len(validation_dl_list[asset_id]) < min_validation_loss\n\n        if is_best:\n            min_validation_loss = running_validation_loss / len(validation_dl_list[asset_id])\n            save_checkpoint(epoch + 1, min_validation_loss, model.state_dict(), optimizer.state_dict(), asset_id)\n\n\n    # Visualize loss\n    epoch_count = range(1, len(training_losses) + 1)\n    plt.plot(epoch_count, training_losses, 'r--')\n    plt.legend(['Training Loss'])\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.show()\n\n    val_epoch_count = range(1, len(validation_losses) + 1)\n    plt.plot(val_epoch_count, validation_losses, 'b--')\n    plt.legend(['Validation loss'])\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:59:53.023113Z","iopub.execute_input":"2022-01-21T14:59:53.023363Z","iopub.status.idle":"2022-01-21T14:59:53.043125Z","shell.execute_reply.started":"2022-01-21T14:59:53.023328Z","shell.execute_reply":"2022-01-21T14:59:53.042293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train\nfor asset_id, model in enumerate(models_list):\n    print(f\"Training asset {asset_id}\")\n    training(asset_id, model, criterion_list[asset_id], optimizer_list[asset_id], EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T14:59:53.04469Z","iopub.execute_input":"2022-01-21T14:59:53.045006Z","iopub.status.idle":"2022-01-21T15:04:37.672651Z","shell.execute_reply.started":"2022-01-21T14:59:53.044972Z","shell.execute_reply":"2022-01-21T15:04:37.671921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load checkpoint\nsaved_model_list = []\nfor asset_id, model in enumerate(models_list):\n    print(f\"load_checkpoint asset {asset_id}\")\n    model, optimizer, start_epoch, valid_loss_min = load_checkpoint(f\"./model_state_{asset_id}.pt\", model, optimizer_list[asset_id])\n    saved_model_list.append(model)\n    # print(\"model = \", model)\n    # print(\"optimizer = \", optimizer)\n    print(\"valid_loss_min = \", valid_loss_min)\n    print(\"valid_loss_min = {:.6f}\".format(valid_loss_min))","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:04:37.67588Z","iopub.execute_input":"2022-01-21T15:04:37.676083Z","iopub.status.idle":"2022-01-21T15:04:37.72236Z","shell.execute_reply.started":"2022-01-21T15:04:37.676057Z","shell.execute_reply":"2022-01-21T15:04:37.721725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:04:37.723663Z","iopub.execute_input":"2022-01-21T15:04:37.723923Z","iopub.status.idle":"2022-01-21T15:04:37.748413Z","shell.execute_reply.started":"2022-01-21T15:04:37.723889Z","shell.execute_reply":"2022-01-21T15:04:37.747794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get test data feature_1 ~ feature_k\ndef test_feature_k(df, k=SHIFT_K, target_variable=\"Target\"):\n    new_df = []\n    for row in df.iterrows():\n        row = row[1]\n        row_asset_id = int(row.Asset_ID)\n        asset_df = asset_df_dict[row_asset_id]\n        target_q = target_dict[row_asset_id]\n        for i in range(k):\n            row[f\"feature_{k-i}\"] = target_q.queue[i]\n        new_df.append(row)\n    return pd.DataFrame(new_df).astype({'timestamp': 'int64', 'Asset_ID':'int8', 'Count':'int32', 'row_id':'int32'})","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:04:37.751057Z","iopub.execute_input":"2022-01-21T15:04:37.751248Z","iopub.status.idle":"2022-01-21T15:04:37.758391Z","shell.execute_reply.started":"2022-01-21T15:04:37.751226Z","shell.execute_reply":"2022-01-21T15:04:37.757656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (test_df, sample_prediction_df) = next(iter_test)\n# test_df['Upper_Shadow'] = upper_shadow(test_df)\n# test_df['Lower_Shadow'] = lower_shadow(test_df)\n# test_df\n# test_df = test_feature_k(test_df)\n# test_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_asset_id_list = test_df.Asset_ID.to_list()\n# selected_features = test_df[FEATURES]\n# x = torch.Tensor(selected_features.values)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:08:26.934531Z","iopub.execute_input":"2022-01-21T15:08:26.935142Z","iopub.status.idle":"2022-01-21T15:08:26.941823Z","shell.execute_reply.started":"2022-01-21T15:08:26.935089Z","shell.execute_reply":"2022-01-21T15:08:26.941114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# idx = 1\n# asset_id = 3\n\n# x = x[idx].unsqueeze(0)\n# x = x.float().to(device)\n# x = x.view(1, -1, NUM_FEATURES)\n# model = saved_model_list[asset_id]\n# model.eval()\n# validation_states = model.init_hidden_states(1)\n# validation_states = [state.detach() for state in validation_states]\n# output, _ = model(x, validation_states)\n# pred = output[:, -1, :].item()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:08:16.346335Z","iopub.execute_input":"2022-01-21T15:08:16.346906Z","iopub.status.idle":"2022-01-21T15:08:16.367607Z","shell.execute_reply.started":"2022-01-21T15:08:16.346865Z","shell.execute_reply":"2022-01-21T15:08:16.366523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (test_df, sample_prediction_df) in iter_test:\n    pred_list = []\n    test_df['Upper_Shadow'] = upper_shadow(test_df) # test Upper_Shadow\n    test_df['Lower_Shadow'] = lower_shadow(test_df) # test Lower_Shadow\n    \n    test_df = test_feature_k(test_df) # get test data feature_1 ~ feature_k\n    test_asset_id_list = test_df.Asset_ID.to_list() # get asset_id_list\n\n    selected_features = test_df[FEATURES]\n    x_values = torch.Tensor(selected_features.values)\n    for idx, asset_id in enumerate(test_asset_id_list):\n        x = x_values[idx].unsqueeze(0)\n        x = x.float().to(device)\n        x = x.view(1, -1, NUM_FEATURES)\n        model = saved_model_list[asset_id]\n        model.eval()\n        validation_states = model.init_hidden_states(1)\n        validation_states = [state.detach() for state in validation_states]\n        output, _ = model(x, validation_states)\n        pred = output[:, -1, :].item()\n        target_dict[asset_id].get()\n        target_dict[asset_id].put(pred)\n        pred_list.append(pred)\n    sample_prediction_df['Target'] = pred_list\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:09:20.187249Z","iopub.execute_input":"2022-01-21T15:09:20.188069Z","iopub.status.idle":"2022-01-21T15:09:20.349331Z","shell.execute_reply.started":"2022-01-21T15:09:20.188016Z","shell.execute_reply":"2022-01-21T15:09:20.348663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}