{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"93b57a34-5e99-cd3c-1477-6e088f042f72"},"outputs":[],"source":"library(data.table)\nlibrary(digest)\ndtrain = fread(\"../input/train_date.csv\",drop = \"Id\",   nrows=500000)"},{"cell_type":"markdown","metadata":{"_cell_guid":"55edcbd1-26f3-48b0-d0a6-6e7201c73f31"},"source":"Get all the Stations"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e373e5bc-4f46-0773-c2f3-02d3bfc9e6cf"},"outputs":[],"source":"statNames = function(y) unique(unlist(lapply(strsplit(grep(\"_\", names(y), value=T), \"_\"), function(x) paste(x[1], x[2], sep=\"_\"))))\ndstations = statNames(dtrain)\nprint(dstations)"},{"cell_type":"markdown","metadata":{"_cell_guid":"b279e41b-76e7-b4bd-eb00-3f8e28fd54f3"},"source":"Find duplicates in each station"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"6090c094-8486-e21e-ab7d-6ea7ef38d9f4"},"outputs":[],"source":"yesDuplicated = noDuplicated = c()\nfor(ii in 1:length(dstations)){\n  selcol = grep(dstations[ii], names(dtrain), value=T) # \n  dgst = duplicated(lapply(dtrain[,selcol,with=F], digest))\n  yesDuplicated = c(yesDuplicated, selcol[dgst])\n  noDuplicated = c(noDuplicated, selcol[!dgst])\n  gc();gc()\n}"},{"cell_type":"markdown","metadata":{"_cell_guid":"12d61f42-12bf-6b30-3bfb-dba30c4c471f"},"source":"Print them out"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ab9e31c3-3f02-ac5e-12b6-eb0182aafde4"},"outputs":[],"source":"noDuplicated"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f7afa17c-01b3-26d8-a524-50219c88cb06"},"outputs":[],"source":"yesDuplicated"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4d2f6a80-cbcb-fd23-8ead-5e04b7d84414"},"outputs":[],"source":"Print them as a vector..."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"dd7330d9-0aa7-1426-af94-1e0ddb0270e7"},"outputs":[],"source":"print(paste0(noDuplicated, collapse='\",\"', sep=\"\"))"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"codemirror_mode":"r","file_extension":".r","mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.3.1"}},"nbformat":4,"nbformat_minor":0}