library(randomForest)
library(readr)
library(caret)
library(xgboost)


set.seed(0)

#parameters
    #rf:
numTrees <- 150
mtry<-30
nodesize<-1


#read in data
train <- read_csv("../input/train.csv")
test <- read_csv("../input/test.csv")

#subset labels and training data into CV and train sets
inCV<-sample(1:nrow(train), size = .33*nrow(train))
CV.labels <- as.factor(train[inCV,1])
CV<-train[inCV,-1]

train.labels <- as.factor(train[-inCV,1])
bst.train.labels<-train[-inCV,]$label
train<-train[-inCV,-1]

#preprocess train, CV and test sets
nzv<-nearZeroVar(train)
train<-train[,-nzv]
test<-test[,-nzv]
CV<-CV[,-nzv]

pca<-preProcess(train, method=c("center","scale"))
train<-predict(pca,train)
test<-predict(pca,test)
CV<-predict(pca,CV)

#RF predictions and training
rf <- randomForest(train, train.labels, ntree=numTrees,nodesize=nodesize,mtry=mtry)
RF_test_predictions <- predict(rf,test)
RF_CV_predictions<-predict(rf,CV)

#xgboost
    #data prep
train<-as.matrix(train)
train<- matrix(as.numeric(train),nrow(train),ncol(train))


test<-as.matrix(test)
test<- matrix(as.numeric(test),nrow(test),ncol(test))

CV<-as.matrix(CV)
CV<-matrix(as.numeric(CV),nrow(CV),ncol(CV))

#Fit model

bst<-xgboost(data = train, label = bst.train.labels, max.depth = 6, eta = .3, nround =10,
               nthread = 2, objective = "multi:softmax",num_class=10)


bst_Test_pred<-predict(bst,test)
bst_CV_pred<-predict(bst,CV)



#Combine into ensemble
###ensemble<-glm(CV.labels~bst_CV_pred+RF_CV_predictions)
###CV_predictions<-predict(ensemble,CV)
CV_predictions <- (bst_CV_pred+RF_CV_predictions)/2
sum(CV_predictions==CV.labels)/nrow(CV)

predictions<-data.frame(Imageid=1:nrow(test),Label=NA)
###predictions[,2]<-predict(ensemble,test)
predictions[,2] <- (bst_Test_pred+RF_test_predictions)/2

head(data.frame(bst_Test_pred,RF_test_predictions))

write.csv(predictions,'ensemble.csv',row.names=F)