{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"3.6.3"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"library(tidyverse)\nlibrary(jsonlite)\nlibrary(tidytext)\nlibrary(tokenizers)\nlibrary(parallel)\nlibrary(doMC)\nlibrary(stringr)\nlibrary(e1071)\nlibrary(NLP)\n# library(openNLP)\n# library(openNLPdata)\nlibrary(udpipe)\ncores = detectCores()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"udmodel_eng = udpipe_load_model(file = '../input//udpipe/english-ewt-ud-2.4-190531.udpipe')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load.text.row = function(train.df, r) {\n    json.file.name = paste0(train.df[[r, 'Id']], '.json')\n    json.file.path = file.path('../input//coleridgeinitiative-show-us-the-data/train/', json.file.name)\n    return(load.text.json(json.file.path))\n}\n\nload.text.json = function(json.file.path) {\n    paper.json = read_json(json.file.path)\n    paper.df = bind_rows(paper.json)\n    paper.df = paper.df %>% mutate(text.lower = tolower(text))\n    return(paper.df)\n}\n\nclean.text = function(text) {\n    return(str_replace_all(tolower(text), pattern = '[^A-Za-z0-9]+', replacement = ' '))\n}\n\ndataset.name.exist = function(train.df, r) {\n    paper.df = load.text.row(train.df, r)\n    paper.text = paper.df$text.lower\n    label = train.df[[r, 'cleaned_label']]\n    res = length(paper.text[grep(label, paper.text)]) > 0\n    return(res)\n}\n\nfind.dataset.label = function(train.df, r) {\n    paper.df = load.text.row(train.df, r)\n    paper.text = paper.df$text.lower\n    label = train.df[[r, 'cleaned_label']]\n    sentences = paper.text %>% paste0(collapse = \"\")\n    return(paper.text[grep(label, paper.text)])\n}\n\nfind.matching.label.sens = function(train.df, r) {\n    paper.df = load.text.row(train.df, r)\n    cleaned.label = train.df[[r, 'cleaned_label']]\n    sens.full = paper.df$text %>% tokenize_sentences %>% unlist\n    sens.cleaned = sens.full %>% clean.text\n    return(sens.full[grep(cleaned.label, sens.cleaned)])\n}\n\nextract.dataset.label = function(paper.df) {\n    sens.full = paper.df$text %>% \n        tokenize_sentences %>% \n        unlist\n    #data|collect|study|survey\n    sens.filtered = sens.full[grepl('(?i)data|collect|study|survey', sens.full)]\n    if (length(sens.filtered) == 0) {\n        return(NULL)\n    }\n    sens.annotated = udpipe_annotate(udmodel_eng, x = sens.filtered, parser = \"none\") %>% \n        as.data.frame %>% \n        mutate(rownum = 1:nrow(.))\n    propn = sens.annotated\n    filtered.df = data.frame()\n    res.df = data.frame()\n    skip = 0\n    for (i in 1:nrow(propn)) {\n        if (skip > 0) {\n            skip = skip - 1\n            next\n        }\n        next.i = i\n        row = propn[i,]\n        if(!is.na(row$upos) & row$upos == 'PROPN') {\n            while(next.i + 1 <= nrow(propn)) {\n                next.row = propn[next.i + 1,]\n                if(is.na(next.row$upos) | !grepl('PROPN|ADP|PART', next.row$upos)) {\n                    break\n                }\n                next.i = next.i + 1\n            }      \n            if (next.i - i >= 2) {\n                res.words = paste0(as.vector(propn[i:next.i, 'token']), collapse = ' ')\n                res.list = list(words = res.words, sentence = row$sentence)\n                res.df = bind_rows(res.df, res.list)\n                filtered.df = bind_rows(filtered.df, propn[i:next.i,])\n                skip = next.i - i\n            } \n        }\n    }\n    return(res.df)\n}\n\njaccard = function(str1, str2) {\n    a = strsplit(tolower(str1), ' ')[[1]]\n    b = strsplit(tolower(str2), ' ')[[1]]\n    c = intersect(a, b)\n    return((length(c)) / (length(a) + length(b) - length(c)))\n}\n\nprecision = function(tp, fp) {\n    return(tp/(tp + fp))\n}\n\nrecall = function(tp, fn) {\n    return(tp/(tp + fn))\n}\n\nfbeta = function(precision, recall, beta) {\n    fbeta = (1 + beta^2) * (precision * recall) / (beta^2 * precision + recall)\n    return(fbeta)\n}\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.csv = '../input/coleridgeinitiative-show-us-the-data//train.csv'\ntrain.df = read_csv(train.csv)\ntest.folder = '../input//coleridgeinitiative-show-us-the-data//test/'\ntest.files = list.files(test.folder, full.names = TRUE)\n# found.label = mclapply(1:nrow(train.df), function(r) {\n#     return(dataset.name.exist(train.df, r))\n# }, mc.cores = 4) %>% unlist()\n# train.df = train.df %>% mutate(found.label.cleaned = found.label)\n\n# There are 14665/19661 rows with the exact label text\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 305\nres.list = mclapply(1:1000, function(r) {\n    sens = find.matching.label.sens(train.df, r)\n    if (length(sens) == 0) {\n        return(NULL)\n    }\n    label = train.df[[r, 'cleaned_label']]\n    pid = train.df[[r, 'Id']]\n    res.df = data.frame(row = r, label = label, sens = sens, stringsAsFactors = FALSE)\n    return(res.df)\n}, mc.cores = cores)\nres.df = bind_rows(res.list)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res.df = res.df %>% \n    mutate(cleaned.sens = clean.text(sens)) %>% \n    mutate(neighbour.words = str_replace_all(cleaned.sens, pattern = label, replacement = ''))\nneighbour.words = res.df$neighbour.words\ntokenized.nw = tokenize_words(neighbour.words) %>% unlist\ntokenized.nw %>% table %>% sort(decreasing = TRUE) %>% head(n = 100) \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res.df %>% \n#     filter(grepl('data|Data|collect|Study|study|survey', sens)) %>% \n    filter(grepl('data', sens)) %>% \n    select(row) %>% \n    unique %>% \n    nrow","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# annotated = udpipe_annotate(udmodel_eng, x = res.df[1:10, 'sens'])\n# as.data.frame(annotated) %>% filter (token == 'and')\n# udpipe_annotate(udmodel_eng, x = 'Data on Ph.D. diversity was obtained from the National Science Foundation Survey of Earned Doctorates, which reports demographics of Ph.D. earners (Survey of Earned Doctorates, 2015).') %>% as.data.frame","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique.labels = train.df$cleaned_label %>% unique \nlabels.df = data.frame(label = unique.labels, length = sapply(strsplit(unique.labels, \" \"), length))\nlabels.df$length %>% hist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# r = 86\n# # paper.df = load.text.row(train.df, r)\n# fn = test.files[r]\n# # fn %>% str_extract('(?<=\\\\/)(((?!\\\\/).)*)(?=.json)')\n# paper.df = load.text.json(fn)\n# res.df = extract.dataset.label(paper.df)\n# res.df\n# # system.time({\n# #     sens.full = paper.df$text %>% \n# #         tokenize_sentences %>% \n# #         unlist\n#     #data|collect|study|survey\n# #     sens.filtered = sens.full[grepl('(?i)data', sens.full)]\n# #      sens.annotated = udpipe_annotate(udmodel_eng, x = sens.filtered, tagger = \"default\", parser = \"none\") %>% \n# #         as.data.frame %>% \n# #         mutate(rownum = 1:nrow(.))\n# # })\n# # sens.annotated","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final.res = mclapply(test.files, function(fn){\n    Id = fn %>% str_extract('(?<=\\\\/)(((?!\\\\/).)*)(?=.json)')\n#     write(fn,file=\"myfile.txt\",append=TRUE)\n    paper.df = load.text.json(fn)\n    res.df = extract.dataset.label(paper.df)\n    PredictionString = res.df$words %>% clean.text %>% unique %>% paste0(collapse = \"|\")\n    return(list(Id = Id, PredictionString = PredictionString))\n}, mc.cores = cores)\nfinal.res.df = bind_rows(final.res)\nwrite.csv(final.res.df, 'submission.csv', row.names = FALSE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sentence = 'a Survey'\n# grepl('(?i)data|collect|study|survey', sentence)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}