```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

```{r}
library(tidyverse)
library(rjson)
library(purrr)
library(tidytext)
library(wordcloud)
library(stopwords)
#library(textrecipes)
library(tidymodels)
```

## Basic notebook for loading competition data
This is my first public notebook. I made this because 
- I want to dive deeper into the problem
- did not find any R notebook
- want to try how public notebooks works

### load json files from directory train

```{r}
files.list <- list.files(path = "../input/coleridgeinitiative-show-us-the-data/train/", full.names = TRUE, pattern = "\\.json$")
  listoftext=tibble(
          id = character(),
          text = character()
  )
for (i in seq_along(files.list)) {
  file_1 <- fromJSON(file=files.list[i])
  idi = str_sub(files.list[i], start=21, end= -6)
  past_1 <- pmap(file_1,paste0)
  listoftext <- listoftext %>% add_row(id = idi, text= as.character(past_1[2]))
}

```

```{r}
slice_head(listoftext,n= 10)
```


## Tokenisation and removal of stopwords

```{r}
text_token <- listoftext %>% unnest_tokens(word,text)
text_token <- text_token %>% anti_join(get_stopwords())
```

## Count words and filter words with less then 3 character 
```{r}
word_count <- text_token %>% 
        count(word, sort = TRUE)
## remove all words with length < 3
word_count <- word_count %>% filter(str_length(word)>3)
pal <- brewer.pal(8,"Dark2")
word_count %>%   with(wordcloud(word, freq=n , random.order = FALSE, max.words = 50, colors = pal))

```

## Read train.csv
```{r}
train <- read_csv("../input/coleridgeinitiative-show-us-the-data/train.csv")
```




### Number of referenced datasets per publication top 20  
```{r}
ref_data <- train %>% count(Id) %>% slice_max(n, n=20)
ref_data
```

### word count train cleaned_label

```{r}

cleaned_token <- train %>% unnest_tokens(word,cleaned_label)
cleaned_token <- cleaned_token %>% anti_join(get_stopwords())
word_count_train <- cleaned_token %>% 
        count(word, sort = TRUE)
## remove all words with length < 3
word_count_train <- word_count_train %>% filter(str_length(word)>3)
pal <- brewer.pal(8,"Dark2")
word_count_train %>%   with(wordcloud(word, freq=n , random.order = FALSE, max.words = 50, colors = pal))
```

## Work in progress