# This R environment comes with all of CRAN preinstalled, as well as many other helpful packages
# The environment is defined by the kaggle/rstats docker image: https://github.com/kaggle/docker-rstats
# For example, here's several helpful packages to load in 

library(readr)
library(NLP)
library(tm)
library(SnowballC)
library(skmeans)
library(varhandle)##unfactor() function
library(Hmisc)
product.df <- read.csv('../input/producto_tabla.csv')
product.df<-product.df[2:nrow(product.df),]
dim(product.df)
product.clean.df=product.df

##extract promotion factor
extract.promo=function(x){
 # x=product.clean.df[1,2]
  x=strsplit(as.character(x)," ")[[1]]
  
  return("Promo"%in%x||"promo"%in%x||"prom"%in%x||"Prom"%in%x)
}

##extract the weight factor
extract.weight=function(x){
  x=strsplit(as.character(x)," ")[[1]]
  tryCatch({
    weight=grep(pattern="[0-9][kKgGml]", x, value=TRUE)##get the match pattern. weight can be g or Kg
    if(length(weight)==0){
      return("NA")
      }
   return(weight)
  },error=function(e){
    return ("NA")
  })
  
}

##extract package factor
extract.pack=function(x){
  x=strsplit(as.character(x)," ")[[1]]
  tryCatch({
    pack=grep(pattern="[0-9][pP]", x, value=TRUE)##get the match pattern. 
    if(length(pack)==0){
      return("NA")
    }
    return(pack)
  },error=function(e){
    return ("NA")
  })
  
}

##extract extra infomation
extract.info=function(x,y){
  x=strsplit(as.character(x)," ")[[1]]
  info=x[length(x)-1]
  tryCatch({
  if(x[length(x)-2]%in%c(y,"promo","prom","Promo","Prom")||grepl("[[:digit:]]", x[length(x)-2])==TRUE){##grepl return logical (TRUE / FALSE).
    return(info)
  }else{
    
    return(paste(x[length(x)-2],x[length(x)-1], collapse = " "))
  }
  
  })
}

##extract brand name
extract.bname=function(x){
  x=strsplit(as.character(x)," ")[[1]]
  index=grep(pattern="[0-9][pPkKgGmlML]", x)##if no number, index will be "-1"
  if(length(index)!=0 &length(index)!=length(x)){
    bname=paste(x[1:(index[1]-1)], collapse = " ")
    return(bname)
    
  }else{
    return(paste(x[1:length(x)-1],collapse = " "))
  }
  
}



product.clean.df$promo=sapply(product.clean.df$NombreProducto, extract.promo )
product.clean.df$weight=sapply(product.clean.df$NombreProducto, extract.weight )
product.clean.df$pack=sapply(product.clean.df$NombreProducto, extract.pack )
product.clean.df$info=sapply(product.clean.df$NombreProducto,extract.info, y= product.clean.df$weight)
product.clean.df$bname=sapply(product.clean.df$NombreProducto, extract.bname)

# Short Names Preprocessing
##A vector source interprets each element of the vector x as a document.
# Examples
# docs <- c("This is a text.", "This another one.")
# (vs <- VectorSource(docs))
# inspect(VCorpus(vs))
CorpusShort <-Corpus(VectorSource(product.clean.df$bname))
# #tm_map
# #tm_filter returns a corpus containing documents where FUN matches, whereas tm_index only
# returns the corresponding indices.
# tolower
# tolower Either a logical value indicating whether characters should be translated
# to lower case or a custom function converting characters to lower case.
CorpusShort <- tm_map(CorpusShort, tolower)
CorpusShort <- tm_map(CorpusShort, PlainTextDocument)##list to vcorpus
# Remove Punctuation
CorpusShort <- tm_map(CorpusShort, removePunctuation)

# Remove Stopwords
CorpusShort <- tm_map(CorpusShort, removeWords, stopwords("es"))

# Stemming
CorpusShort <- tm_map(CorpusShort, stemDocument, language="es")

#In a document-term matrix, rows correspond to documents in the collection and columns correspond to terms
# Create DTM
CorpusShort <- Corpus(VectorSource(CorpusShort))
dtmShort <- DocumentTermMatrix(CorpusShort)

# Delete Sparse Terms (all the words now)
sparseShort <- removeSparseTerms(dtmShort, 0.9999)
ShortWords <- as.data.frame(as.matrix(sparseShort))

# Create valid names
colnames(ShortWords) <- make.names(colnames(ShortWords))

# Spherical k-means for product clustering (30 clusters at the moment)
set.seed(123)
mod <- skmeans(as.matrix(ShortWords), 30, method = "genetic")
product.clean.df$cluster <- mod$cluster

product.clean.df$weight=as.character(product.clean.df$weight)
product.clean.df$pack=as.character(product.clean.df$pack)
# Example for one of the clusters
write_csv(product.clean.df[c(1:15),], 'product.clean.csv')