{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"> # Cryptocurrency Forecasting in R  \n\n> ### The objective of this R language notebook is to apply ML techniques to predict future real cryptocurrency market data for G-Research, (Europe’s leading quantitative finance research firm). Specifically, the forecasting of short term returns in 14 popular cryptocurrencies within a dataset of millions of rows of high-frequency market data dating back to 2018.  \n\n> #### All R Language programming by John Akwei, ECMp ERMp Data Scientist  \n> #### ContextBase, contextbase.github.io  ","metadata":{}},{"cell_type":"markdown","source":"> # Section 1 - Required Libraries","metadata":{}},{"cell_type":"code","source":"suppressMessages(library(tidyverse))\nsuppressMessages(library(tidyr))\nsuppressMessages(library(data.table))\nsuppressMessages(library(dplyr))\nsuppressMessages(library(keras))\nsuppressMessages(library(reticulate))\nsuppressMessages(library(tibble))\nsuppressMessages(library(ggplot2))\nsuppressMessages(library(knitr))\nsuppressMessages(library(timeSeries))\nsuppressMessages(library(tseries))\nsuppressMessages(library(xts))\nsuppressMessages(library(zoo))\nsuppressMessages(library(quantmod))\nsuppressMessages(library(PerformanceAnalytics))\nsuppressMessages(library(forecast))\nsuppressMessages(library(rugarch))\nsuppressMessages(library(caret))\nsuppressMessages(library(lightgbm))\nsuppressMessages(library(xgboost))\nsuppressMessages(library(Matrix))","metadata":{"execution":{"iopub.status.busy":"2021-11-11T13:31:53.885098Z","iopub.execute_input":"2021-11-11T13:31:53.886688Z","iopub.status.idle":"2021-11-11T13:31:53.92525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 2 - Import Datasets","metadata":{}},{"cell_type":"code","source":"data_folder <- \"../input/g-research-crypto-forecasting/\"\n\ntrain <- fread(\"../input/g-research-crypto-forecasting/train.csv\")\nasset_data <- fread(\"../input/g-research-crypto-forecasting/asset_details.csv\")\nexample_sample_submission <- fread(\"../input/g-research-crypto-forecasting/example_sample_submission.csv\")\nexample_test <- fread(\"../input/g-research-crypto-forecasting/example_test.csv\")\nsupplemental_train <- fread(\"../input/g-research-crypto-forecasting/supplemental_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-11T13:32:02.832996Z","iopub.execute_input":"2021-11-11T13:32:02.834538Z","iopub.status.idle":"2021-11-11T13:44:16.150594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 3 - Exploratory Dataset Analysis","metadata":{}},{"cell_type":"code","source":"cat(\"First Five Rows of train.csv\")\nhead(train)\n\ncat(\"Content of asset_details.csv\")\nasset_data\n\ncat(\"First Five Rows of example_test.csv\")\nhead(example_test)\n\ncat(\"First Five Rows of supplemental_train.csv\")\nhead(supplemental_train)\n\ncat(\"First Five Rows of example_sample_submission.csv\")\nhead(example_sample_submission)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 4 - Data Optimization","metadata":{}},{"cell_type":"code","source":"# Remove Dataset Rows With Not Available Data\ncat(\"Number of Rows in the Train Dataset, before na.omit()\")\nnrow(train)\n\ntrain <- na.omit(train)\n\ncat(\"Number of Rows in the Train Dataset, after na.omit()\")\nnrow(train)\n\n# Convert Date Format\nDate <- as.Date(as.POSIXct(train$timestamp, origin=\"1970-01-01 00:00:00.000\"))\ntrain <- data.frame(Date, train)","metadata":{"execution":{"iopub.status.busy":"2021-11-11T13:56:58.829205Z","iopub.execute_input":"2021-11-11T13:56:58.830768Z","iopub.status.idle":"2021-11-11T13:57:05.95049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 5 - Bitcoin Trading Analysis","metadata":{}},{"cell_type":"code","source":"# Extract Asset #1 (Bitcoin)\ndata <- train[train$Asset_ID == \"1\",]\n\n# Time series for opening bitcoin prices\nbitcoin_time_series <- ts(data$Open, frequency=250)\n\n# Calculate Mean and Standard Deviation\nkable(mean(bitcoin_time_series), caption = \"Mean of Bitcoin Time Series\")\nkable(sd(bitcoin_time_series), caption = \"Standard Deviation of Bitcoin Time Series\")\n\n# Chart Series with Bollinger bands, highs & lows, and mean average convergence divergence.\nbitcoin_xts <- as.xts(bitcoin_time_series, dateFormat=\"POSIXct\")\nchartSeries(bitcoin_xts,\n            type = \"candlesticks\",\n            theme = chartTheme('white'),\n            TA=c(addVo(),\n                 addSMA(n=200,col = 'blue'),\n                 addSMA(n=50,col = 'red'),addSMA(n=22,col = 'green'),\n                 addBBands(), addMACD()))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 6 - Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Create Features for Train Dataset\ntrain$Upper_Shadow <- train$High / max(train$Close, train$Open)\ntrain$Lower_Shadow <- min(train$Close, train$Open) / train$Low\ntrain$open2close <- train$Close / train$Open\ntrain$high2low <- train$High / train$Low\ndata_subset <- data.frame(Open=train$Open, High=train$High, Low=train$Low, Close=train$Close)\nmean_price <- apply(data_subset, 1, mean)\nmedian_price <- apply(data_subset, 1, median)\ntrain$high2mean <- train$High / mean_price\ntrain$low2mean <- train$Low / mean_price\ntrain$high2median <- train$High / median_price\ntrain$low2median <- train$Low / median_price\ntrain$volume2count  <- train$Volume / train$Count\n\n# Create Features for example_test Dataset\nexample_test_subset <- data.frame(Open=example_test$Open,\n                                  High=example_test$High,\n                                  Low=example_test$Low,\n                                  Close=example_test$Close)\nexample_test_median <- apply(example_test_subset, 1, median)\nexample_test$high2median <- example_test$High / example_test_median\n\ncat(\"First Five Rows of Feature Engineered Train Dataset\")\nhead(train)\n\ncat(\"First Five Rows of Feature Engineered example_test dataset\")\nhead(example_test)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 7 - Correlation Testing","metadata":{}},{"cell_type":"code","source":"numericVars <- which(sapply(train, is.numeric)) \nall_numVar <- train[, numericVars]\ncor_numVar <- cor(all_numVar, use=\"pairwise.complete.obs\")\ncor_sorted <- as.matrix(sort(cor_numVar[,'Target'], decreasing=TRUE))\nCorHigh <- names(which(apply(cor_sorted, 1, function(x) abs(x)>0.005)))\n\nkable(print(CorHigh), caption = \"High Correlation Variables for Linear Modeling\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 8 - Linear Regression Predictive Analytics","metadata":{}},{"cell_type":"code","source":"# Linear Regression Model\nset.seed(123)\nlm_data <- data.frame(Target=as.numeric(train$Close), high2median=as.numeric(train$high2median))\nmodel_LN <- lm(Target ~ high2median, data = lm_data)\n\n# Linear Regression Predictions\npred_LN <- predict(model_LN, example_test)\nnames(pred_LN) <- NULL\n\n# Assemble Submission Dataframe\ntest_data <- data.frame(group_num=example_test$group_num, row_id=example_test$row_id)\nTarget <- as.integer(pred_LN)\nsubmission <- cbind(test_data, Target)\n\ncat(\"Linear Model of Target vs. high2median\")\nmodel_LN\n\ncat(\"Linear Modeling Predictions Dataframe\")\nsubmission","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 9 - Extreme Gradient Boosting Machine Learning","metadata":{}},{"cell_type":"code","source":"# Format Training Data for XGB Model\ntrainm <- sparse.model.matrix(Target ~ high2median, data = train)\ntrain_label <- train[,\"Target\"]\ntrain_matrix <- xgb.DMatrix(data = as.matrix(trainm), label = train_label)\n\n# Create Validation label variable\nexample_test$Target <- submission$Target\n\n# Create Validation Data\nvalm <- sparse.model.matrix(Target ~ high2median, data = example_test)\nval_label <- example_test$Target\nval_matrix <- xgb.DMatrix(data = as.matrix(valm), label = val_label)\n\n# Specify XGB Parameters\nxgb_params <- list(eval_metric = \"error\",\n                   max_depth = 3,\n                   eta= 0.01,\n                   gammma = 1,\n                   colsample_bytree = 0.5,\n                   min_child_weight = 1)\n\n# Create XGB Model\nModel_XGB <- xgb.train(params = xgb_params, data = train_matrix, nrounds = 1000)\n\n# Create Predictions\npred_XGB <- predict(Model_XGB, newdata = val_matrix)\n\n# Format XGB submission dataframe\nXGB_Predictions <- as.integer(pred_XGB)\nXGB_submission <- cbind(test_data, XGB_Predictions)\n\ncat(\"XGB Model of Target vs. high2median\")\nModel_XGB\n\ncat(\"Extreme Gradient Boosting Predictions Dataframe\")\nXGB_submission","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Section 10 - G-Research Submission Coding","metadata":{}},{"cell_type":"code","source":"# import python environment\ngresearch_crypto <-\nimport_from_path('competition','../input/g-research-crypto-forecasting/gresearch_crypto')\nenv <- gresearch_crypto$make_env()\niter <- env$iter_test()\n\npy_run_string(\"import io\")\npy_run_string(\"import pandas as pd\")\n\nwhile(TRUE) {\n    df <- iter_next(iter)\n    if(is.null(df)) break\n    \n    # pandas dataframe -> csv -> R dataframe\n    test_df <- read_csv(df[[1]]$to_csv(index = FALSE),\n                        show_col_types=FALSE)\n    pred_df <- read_csv(df[[2]]$to_csv(index = FALSE),\n                        show_col_types=FALSE)\n    \n    # Create high2median variable\n    test_df$high2median <- test_df$High / example_test_median[[1]]\n    \n    # make predictions\n    pred_df['Target'] <- predict(model_LN, test_df)\n    \n    # prediction R dataframe -> csv -> Python -> pandas dataframe\n    py$pred_df_string <- format_csv(pred_df)\n    py_run_string(\"pred_df = pd.read_csv(io.StringIO(pred_df_string))\")\n    \n    # submit prediction\n    # env$predict(pred_df)\n}\n\n# cat(\"G-Research Submission Dataframe\")\n# env$predictions","metadata":{},"execution_count":null,"outputs":[]}]}