{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"library(data.table)\nlibrary(dplyr)\nlibrary(keras)\nlibrary(tidyverse)\nlibrary(reticulate)\nlibrary(tibble)\n\noptions(scipen = 999)","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2022-01-23T06:08:56.644370Z","iopub.execute_input":"2022-01-23T06:08:56.646542Z","iopub.status.idle":"2022-01-23T06:08:59.329195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data input & New features**","metadata":{}},{"cell_type":"code","source":"df_train=fread(\"../input/g-research-crypto-forecasting/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:08:59.332797Z","iopub.execute_input":"2022-01-23T06:08:59.372485Z","iopub.status.idle":"2022-01-23T06:09:36.579381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train$Count=NULL\ndf_train$Volume=NULL\ndf_train$VWAP=NULL\ndf_train$NEW_x5=\n    (df_train$High-ifelse(df_train$Open>df_train$Close,df_train$Open,df_train$Close))\ndf_train$NEW_x6=\n    (ifelse(df_train$Open<df_train$Close,df_train$Open,df_train$Close)-df_train$Low)\ndf_train$High=NULL\ndf_train$Open=NULL\ndf_train$Close=NULL\ndf_train$Low=NULL","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:09:36.583056Z","iopub.execute_input":"2022-01-23T06:09:36.584589Z","iopub.status.idle":"2022-01-23T06:09:51.766443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids=c(0,1:13)\nfor (i in c(1:length(ids))){\n    ID=ids[i]\n    df_asset_i=df_train[df_train$Asset_ID==ID,]\n    fwrite(df_asset_i,paste(\"./\",\"asset_\",ID,\".csv\",sep=\"\"))\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:09:51.768918Z","iopub.execute_input":"2022-01-23T06:09:51.770400Z","iopub.status.idle":"2022-01-23T06:10:00.847645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(list=c(\"df_asset_i\",\"df_train\"))","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:10:00.851952Z","iopub.execute_input":"2022-01-23T06:10:00.854626Z","iopub.status.idle":"2022-01-23T06:10:00.873660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (i in c(1:length(ids))){\n    ID=ids[i]\n    df_asset_i=fread(paste(\"./\",\"asset_\",ID,\".csv\",sep=\"\"))\n\n    df_asset_i=df_asset_i[order(df_asset_i$timestamp),]\n    df_asset_i$gapTime60=c(1,\n                           ifelse(diff(df_asset_i$timestamp)==60,1,0))\n    df_asset_i$TargetNA=ifelse(is.na(df_asset_i$Target),0,1)\n    df_asset_i$TargetNA_gapTime60=(df_asset_i$gapTime60+\n                                     df_asset_i$TargetNA)\n    temp=split(df_asset_i, cumsum(c(1, diff(df_asset_i$TargetNA_gapTime60) != 0)))\n    whichOne=ifelse(temp[[length(temp)]]$TargetNA_gapTime60[1]==1,\n                      length(temp)-1,length(temp))\n    df_asset_i=temp[[whichOne]]\n    rm(list=c(\"temp\"))\n    df_asset_i$TargetNA_gapTime60=NULL\n    df_asset_i$gapTime60=NULL\n    df_asset_i$TargetNA=NULL\n    \n    fwrite(df_asset_i,paste(\"./\",\"asset_\",ID,\".csv\",sep=\"\"))\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:10:00.877861Z","iopub.execute_input":"2022-01-23T06:10:00.880304Z","iopub.status.idle":"2022-01-23T06:13:21.329918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for (i in c(1:length(ids))){\n    ID=ids[i]  \n    df_asset_i=fread(paste(\"./\",\"asset_\",ID,\".csv\",sep=\"\"))\n    df_asset_i=df_asset_i[order(df_asset_i$timestamp),]\n    \n    #New feature 1\n    NEW_x1=df_asset_i$Target[1:(nrow(df_asset_i)-1)]\n    df_asset_i=df_asset_i[-1,]\n    df_asset_i$NEW_x1=NEW_x1\n    rm(list=c(\"NEW_x1\"))\n    \n    #New feature 2\n    NEW_x2=df_asset_i$NEW_x1[1:(nrow(df_asset_i)-1)]\n    df_asset_i=df_asset_i[-1,]\n    df_asset_i$NEW_x2=NEW_x2\n    rm(list=c(\"NEW_x2\"))\n    \n    #New feature 3\n    NEW_x3=df_asset_i$NEW_x2[1:(nrow(df_asset_i)-1)]\n    df_asset_i=df_asset_i[-1,]\n    df_asset_i$NEW_x3=NEW_x3\n    rm(list=c(\"NEW_x3\"))\n    \n    #New feature 4\n    NEW_x4=df_asset_i$NEW_x3[1:(nrow(df_asset_i)-1)]\n    df_asset_i=df_asset_i[-1,]\n    df_asset_i$NEW_x4=NEW_x4\n    rm(list=c(\"NEW_x4\"))\n    \n    fwrite(df_asset_i,paste(\"./\",\"asset_\",ID,\".csv\",sep=\"\"))\n    \n}\n   ","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:13:21.333917Z","iopub.execute_input":"2022-01-23T06:13:21.335499Z","iopub.status.idle":"2022-01-23T06:13:21.687806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(list=c(\"df_asset_i\"))","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:13:21.691557Z","iopub.execute_input":"2022-01-23T06:13:21.693155Z","iopub.status.idle":"2022-01-23T06:13:21.706179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Modeling**","metadata":{}},{"cell_type":"code","source":"epoch_num=40\nshape_size=6","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:13:21.709131Z","iopub.execute_input":"2022-01-23T06:13:21.710558Z","iopub.status.idle":"2022-01-23T06:13:21.723867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model_all=list()\nfor (i in c(1:length(ids)))\n{\n    ID=ids[i]\n    data=fread(paste(\"./\",\"asset_\",ID,\".csv\",sep=\"\"))\n    data=data[order(timestamp),]\n    \n    #modeling\n    set.seed(2022)\n    model_i <- keras_model_sequential()\n    model_i %>%\n      layer_dense(\n        units = 8,\n        input_shape = shape_size,\n        activation = 'relu') %>%\n      layer_dense(units = 1) %>%\n      compile(\n        loss = 'mean_squared_error',\n        optimizer = 'adam') %>%\n      fit(\n        as.matrix(data[,c(\"NEW_x1\",\"NEW_x2\",\"NEW_x3\",\"NEW_x4\",\"NEW_x5\",\"NEW_x6\")]),\n        as.matrix(data[,c(\"Target\")]),\n        epochs = epoch_num,\n        batch_size = 1024,\n        verbose = 2)\n    \n    model_i %>% save_model_hdf5(paste(\"./model_\",ID,\".hdf5\",sep=\"\"))\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:13:21.726313Z","iopub.execute_input":"2022-01-23T06:13:21.727750Z","iopub.status.idle":"2022-01-23T06:13:50.771283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(list=c(\"model_i\"))","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:13:50.775391Z","iopub.execute_input":"2022-01-23T06:13:50.776996Z","iopub.status.idle":"2022-01-23T06:13:50.791364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Submit**","metadata":{}},{"cell_type":"code","source":"#call python\ngresearch_crypto <- import_from_path('competition','../input/g-research-crypto-forecasting/gresearch_crypto')\nenv <- gresearch_crypto$make_env()\niter <- env$iter_test()\npy_run_string(\"import io\")\npy_run_string(\"import pandas\")\n\n#submit\nj <- 0\n\nwhile(TRUE){\n    df <- iter_next(iter)\n    if (is.null(df)) break\n    \n    test_df <- fread(df[[1]]$to_csv(index = FALSE), data.table = TRUE)\n    test_df$NEW_x5=(test_df$High-ifelse(test_df$Open>test_df$Close,test_df$Open,test_df$Close))\n    test_df$NEW_x6=(ifelse(test_df$Open<test_df$Close,test_df$Open,test_df$Close)-test_df$Low)\n    test_df$NEW_x1=0\n    test_df$NEW_x2=0\n    test_df$NEW_x3=0\n    test_df$NEW_x4=0\n    test_df$Target=0\n    \n    for (i in c(1:length(ids))){\n         ID=ids[i]\n         df_train_i=fread(paste(\"./\",\"asset_\",ID,\".csv\",sep=\"\"))\n         df_train_i=df_train_i[order(timestamp),]\n         \n         rowid=nrow(df_train_i)\n         m=matrix(c(df_train_i$Target[(rowid)],\n                    df_train_i$Target[(rowid-1)],\n                    df_train_i$Target[(rowid-2)],\n                    df_train_i$Target[(rowid-3)],\n                    test_df$NEW_x5[test_df$Asset_ID==ID],\n                    test_df$NEW_x6[test_df$Asset_ID==ID]),1)\n         #test_df[test_df$Asset_ID==ID,c(\"NEW_x4\",\"NEW_x3\",\"NEW_x2\",\"NEW_x1\")]=df_train_i$Target[(rowid-4+1):(rowid)]\n         model_i= load_model_hdf5(paste(\"./model_\",ID,\".hdf5\",sep=\"\"))\n         test_df$Target[test_df$Asset_ID==ID]=model_i%>% predict(m)\n         \n         test_df$NEW_x1=df_train_i$Target[(rowid)]\n         test_df$NEW_x2=df_train_i$Target[(rowid-1)]\n         test_df$NEW_x3=df_train_i$Target[(rowid-2)]\n         test_df$NEW_x4=df_train_i$Target[(rowid-3)]\n         df_train_i=rbind(test_df[,colnames(df_train_i),with=FALSE],df_train_i)\n        \n         fwrite(df_train_i,paste(\"./\",\"asset_\",ID,\".csv\",sep=\"\"))\n     }\n     \n    \n    test_df=test_df[order(row_id),]\n    pred_df <- fread(df[[2]]$to_csv(index = FALSE), data.table = FALSE)\n    pred_df$group_num <- j\n    pred_df$Target <- test_df$Target\n    pred_df <- pred_df[,c(\"group_num\",\"row_id\",\"Target\")]\n     \n    #summit\n    py$pred_df_string <- format_csv(pred_df)\n    py_run_string(\"pred_df = pandas.read_csv(io.StringIO(pred_df_string))\")\n    py_run_string(paste0(\"pred_df.index.values[0] = \", i))\n    pred_df_pointer <- py_get_attr(py, \"pred_df\")\n    env$predict(pred_df_pointer)\n    \n    j <- j + 1\n}","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:13:50.795013Z","iopub.execute_input":"2022-01-23T06:13:50.796577Z","iopub.status.idle":"2022-01-23T06:14:00.323835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fread(\"./submission.csv\")[1:7,]","metadata":{"execution":{"iopub.status.busy":"2022-01-23T06:17:06.410271Z","iopub.execute_input":"2022-01-23T06:17:06.411944Z","iopub.status.idle":"2022-01-23T06:17:06.436568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}