{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"eaa19822-2b3d-496e-b6d9-958246d0df80"},"source":"This is a new version of my older script which instead of visualizing a number of jobs presents the whole data. This allows assessing typical paths across the production facility. I provide three variants - one for all jobs and two additional ones filtered to Response 0 and 1. Widths of the edges indicate the number of jobs on this station pair."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"fd74c71c-ec4c-e9cd-ff3c-fd132dcb0568"},"outputs":[],"source":"library(data.table)\nlibrary(dplyr)\n\n\n#imports for plotting\nrequire(GGally)\nlibrary(network)\nlibrary(sna)\nlibrary(ggplot2)\nrequire(igraph)\nrequire(intergraph)\nlibrary(stringr)\n\n\ntopFeaturesDate <- c(\"L3_S33_D3856\",\"L3_S34_D3875\",\"L3_S30_D3496\",\"L3_S29_D3316\",\"L3_S37_D3942\",\n                       \"L3_S29_D3474\",\"L0_S0_D1\" ,    \"L3_S30_D3506\",\"L3_S30_D3501\",\"L3_S30_D3726\",\n                       \"L3_S30_D3566\",\"L0_S3_D70\" ,   \"L3_S35_D3895\",\"L0_S8_D145\"  , \"L2_S26_D3037\",\n                       \"L3_S35_D3886\",\"L3_S35_D3900\",\"L0_S13_D355\" , \"L3_S36_D3919\",\"L0_S1_D26\"   ,\n                       \"L3_S36_D3928\",\"L2_S26_D3081\",\"L1_S24_D1511\",\"L0_S12_D331\" , \"L1_S24_D1809\",\n                       \"L0_S2_D34\",\"L1_S24_D1536\",\"L0_S14_D360\" , \"L1_S24_D1558\",\"L0_S23_D617\" ,\n                       \"L0_S6_D120\",\"L1_S24_D1566\",\"L1_S24_D1576\",\"L0_S20_D462\" , \"L0_S4_D106\"  ,\n                       \"L0_S1_D30\",\"L0_S19_D454\",  \"L0_S7_D137\",\"L0_S9_D152\" ,  \"L1_S24_D1826\",\n                       \"L0_S16_D423\",\"L0_S17_D432\" , \"L1_S24_D1568\",\"L0_S5_D115\" ,  \"L1_S24_D1674\",\n                       \"L1_S24_D1570\",\"L0_S18_D437\"  ,\"L0_S10_D216\",\"L0_S9_D157\"  , \"L1_S24_D1522\",\n                       \"L0_S4_D111\",\"L2_S27_D3130\",\"L0_S11_D280\" , \"L0_S22_D543\" , \"L0_S9_D162\"  ,\n                       \"L0_S10_D221\",\"L0_S15_D395\",  \"L0_S9_D167\"  , \"L1_S24_D1765\",\"L3_S31_D3836\",\n                       \"L0_S22_D548\",\"L1_S24_D1583\",\"L0_S21_D469\",  \"L1_S24_D1770\",\"L0_S10_D231\" ,\n                       \"L0_S22_D553\",\"L0_S21_D474\" , \"L1_S25_D1854\",\"L3_S32_D3852\",\n                       \"L3_S39_D3966\",\"L0_S21_D484\" , \"L1_S25_D1867\",\"L3_S44_D4101\",\"L3_S40_D3981\",\n                       \"L3_S38_D3953\",\"L0_S22_D558\" , \"L1_S25_D2780\",\"L3_S43_D4062\",\"L3_S49_D4208\",\n                       \"L2_S28_D3223\",\"L1_S24_D1116\",\"L1_S25_D2788\",\"L3_S45_D4125\",\"L3_S49_D4218\",\n                       \"L3_S50_D4242\",\"L3_S40_D3985\",\"L1_S25_D2471\",\"L1_S25_D2058\",\"L1_S25_D1891\",\n                       \"L1_S24_D677\",\"L1_S24_D1135\",\"L3_S43_D4082\",\"L1_S25_D2792\",\"L1_S25_D1883\",\n                       \"L3_S41_D3997\",\"L1_S25_D2098\",\"L1_S25_D1887\",\"L1_S25_D2754\",\"L1_S25_D2138\",\n                       \"L3_S51_D4255\",\"L1_S24_D1155\",\"L1_S25_D1980\",\"L1_S25_D2790\",\"L3_S47_D4140\",\n                     \"L1_S25_D3011\",\"L1_S25_D2240\",\"L1_S25_D2230\",\"L1_S25_D2798\",\"L1_S25_D1902\",\n                       \"L1_S24_D702\",\"L1_S24_D697\",\"L3_S48_D4194\",\"L1_S25_D2996\",\"L1_S24_D1413\",\n                       \"L1_S24_D818\",\"L1_S24_D1168\",\"L1_S24_D1163\",\"L1_S24_D1171\",\"L1_S24_D804\", \n                       \"L1_S24_D909\",\"L1_S24_D801\",\"L1_S25_D2801\",\"L1_S24_D1018\",\"L1_S25_D2206\",\"L1_S25_D2505\")\n\n#one expansion option would be to select a time frame to visualized\ndtNum <- fread(\"../input/train_numeric.csv\", select = c(\"Id\", \"Response\"))\ndtDate <- fread(\"../input/train_date.csv\",select = c(\"Id\",topFeaturesDate))\n\n#for each job identify which stations are passed through and for those store the minimum time\nfor (station in paste0(\"S\",0:51))\n{\n  cols = min(which((grepl(station,colnames(dtDate)))))\n  if(!cols==Inf){\n    dtDate[,paste0(station) := dtDate[,cols,with = FALSE]]\n  }\n}\n\n#limit data to only when passed through station X\ndtStations = dtDate[,!grepl(\"L\",colnames(dtDate)),with=F]\n\n#melt data to go from wide to long format\n#join with numeric to have Response\n#remove NA entries - these are plentiful as after melting each station-job combination has its own row\n\nmelt(dtStations,id.vars=c(\"Id\")) %>%\n  left_join(dtNum, by = \"Id\")  %>%\n  filter(!is.na(value)) -> dtStationsMFiltered\n\ndtStationsMFiltered %>%\n  group_by(variable) %>%\n  summarize(stationCount=n()) -> stationCount\n\n\nstationCount = rbind(stationCount,data.frame(variable=c(\"result-0\",\"result-1\"),\n                                             stationCount=c(table(dtNum$Response)[1],\n                                                            table(dtNum$Response)[2])))\n\ncolnames(stationCount) = c(\"Node\",\"stationCount\")\n\n\n  \n\n#sort entries by ascending time\ndtStationsMFiltered %>%\n  arrange(value) -> dtStationsMFiltered\n\n\n\ndtStationsMFiltered %>%\n  group_by(Id) %>%\n  mutate(nextStation = lead(variable)) -> edgelistsComplete\n\nrm(dtStations)\nrm(dtStationsMFiltered)\nrm(dtNum)\ninvisible(gc())\n\nedgelistsComplete %>%\n  group_by(Id) %>%\n  filter(!(variable %in% nextStation)) -> startingPoints\n\n\nstartingPoints %>%\n  mutate(nextStation = variable) -> startingPoints\n\nstartingPoints$variable = as.character(startingPoints$variable)\nstartingPoints$nextStation = as.character(startingPoints$nextStation)\n\nstartingPoints$variable = \"start\"\n\nedgelistsComplete %>%\n  group_by(Id) %>%\n  filter(!(nextStation %in% variable)) -> endPoints\n\nendPoints$variable = as.character(endPoints$variable)\n\nendPoints$nextStation = paste0(\"result-\",endPoints$Response)\n\nedgelistsComplete$variable = as.character(edgelistsComplete$variable)\nedgelistsComplete$nextStation = as.character(edgelistsComplete$nextStation)\n\nedgelistsComplete %>% filter(nextStation!=\"NA\") -> edgelistsComplete\n\ndataPlot = rbind(startingPoints,edgelistsComplete,endPoints)\ndataPlot$variable = as.factor(dataPlot$variable)\ndataPlot$nextStation = as.factor(dataPlot$nextStation)\n\nrm(endPoints)\nrm(startingPoints)\ninvisible(gc())\n\ndataPlot$edge = paste0(dataPlot$variable,\"%\",dataPlot$nextStation)\n\ndataPlot %>%\n  group_by(edge) %>%\n  summarize(weight = n()) -> plotDataSummarized\n\n\nplotDataSummarized %>%\n  mutate(Source = str_split_fixed(edge,\"%\",2)[,1],\n         Target = str_split_fixed(edge,\"%\",2)[,2]) %>%\n  select(-edge) -> plotDataSummarized\n\n\nplotDataSummarized %>%\n  select(Source,Target,weight) -> plotDataSummarized\n\n\n#for each row obtain the subsequent statoin\n\n\n\n#create a station-line mapping lookup\nLineStations = NULL\nfor (station in unique(plotDataSummarized$Source)){\n  if(station!=\"start\")\n  {\n    x=paste0(\"_\",station,\"_\")\n    y=head(colnames(dtDate)[which(grepl(x,colnames(dtDate)))],1)\n    y=strsplit(y,\"_\")[[1]][1]\n    LineStations = rbind(LineStations,data.frame(Node=station,Line=y))\n  }\n}\nLineStations = rbind(LineStations,data.frame(Node=c(\"result-1\",\"result-0\",\"start\"),Line=c(\"Outcome\",\"Outcome\",\"START\")))\n\nrm(dtDate)\ninvisible()\n\n#plotting format\noptions(repr.plot.width=15, repr.plot.height=5)\n\n\n#create network from edgelist\nplotDataSummarized$weight = plotDataSummarized$weight / 100000\n\nig = graph.data.frame(plotDataSummarized)\n\ny = data.frame(Node=V(ig)$name)\ny %>% left_join(LineStations,by=\"Node\") -> y\n\nig %>% set_vertex_attr(\"line\", value = as.character(y$Line)) -> ig\n\ny %>% left_join(stationCount,by=\"Node\") -> y\ny[y$Node==\"start\",]$stationCount=1\n\nig %>% set_vertex_attr(\"count\", value = y$stationCount) -> ig\n\n\nnodeCoordinates=data.frame(label=c(\"start\",\"S0\",\"S1\",\"S2\",\"S3\",\"S4\",\"S5\",\"S6\",\n                                   \"S7\",\"S8\",\"S9\",\"S10\",\"S11\",\"S12\",\"S13\",\n                                   \"S14\",\"S15\",\"S16\",\"S17\",\"S18\",\"S19\",\n                                   \"S20\",\"S21\",\"S22\",\"S23\",\"S24\",\"S25\",\n                                   \"S26\",\"S27\",\"S28\",\"S29\",\"S30\",\"S31\",\n                                   \"S32\",\"S33\",\"S34\",\"S35\",\"S36\",\"S37\",\n                                   \"S38\",\"S39\",\"S40\",\"S41\",\"S43\",\n                                   \"S44\",\"S45\",\"S47\",\"S48\",\"S49\",\n                                   \"S50\",\"S51\",\"result-0\",\"result-1\"),\n                           x=c(0,\n                               1,2,3,3,4,4,5,5,6,6.75,8,7.25,\n                               1,2,3,3,4,4,5,5,6,6.75,8,7.25,\n                               6,6,6.75,8,7.25,\n                               9,10,11,11,11,12,12,13,14,15,\n                               9,10,11,12,12,13,14,15,16,16,17,\n                               18,18),\n                           y=c(5,\n                               9,9,10,8,10,8,10,8,9,10,9,8,\n                               5,5,6,4,6,4,6,4,5,6,5,4,\n                               2,0,2,1,0,\n                               7,7,8,7,6,8,6,7,7,7,\n                               3,3,3,4,2,3,3,3,4,2,3,\n                               7,3))\n\n\n#setup plot\nnetwork = ggnet2(ig)\n\nnetCoordinates = select(network$data,label)\nnetCoordinates = left_join(netCoordinates,nodeCoordinates,by = \"label\")\nnetCoordinates = as.matrix(select(netCoordinates,x,y))\n\nnetwork = ggnet2(ig,\n                 alpha = .5,\n                 size=\"count\",\n                 label = T,\n                 color = \"line\",palette = \"Set1\",\n                 mode = netCoordinates,\n                 size.cut=5,\n                 arrow.size = 20,\n                 edge.alpha = 0.2, edge.size = \"weight\",\n                 legend.position = \"bottom\")\n\nnetwork\n\n#### Response 0\n\n\n\ndataPlot %>%\n  filter(Response == 0) %>%\n  group_by(edge) %>%\n  summarize(weight = n()) -> plotDataSummarized\n\nplotDataSummarized %>%\n  mutate(Source = str_split_fixed(edge,\"%\",2)[,1],\n         Target = str_split_fixed(edge,\"%\",2)[,2]) %>%\n  select(-edge) -> plotDataSummarized\n\n\n\n\nplotDataSummarized %>%\n  select(Source,Target,weight) -> plotDataSummarized\n\n\n\n\n\n\n#create network from edgelist\nplotDataSummarized$weight = plotDataSummarized$weight /100000\n\nig = graph.data.frame(plotDataSummarized)\n\ny = data.frame(Node=V(ig)$name)\ny %>% left_join(LineStations,by=\"Node\") -> y\n\nig %>% set_vertex_attr(\"line\", value = as.character(y$Line)) -> ig\n\ny %>% left_join(stationCount,by=\"Node\") -> y\ny[y$Node==\"start\",]$stationCount=1\n\nig %>% set_vertex_attr(\"count\", value = y$stationCount) -> ig\n\n\n\n#setup plot\nnetwork = ggnet2(ig)\n\nnetCoordinates = select(network$data,label)\nnetCoordinates = left_join(netCoordinates,nodeCoordinates,by = \"label\")\nnetCoordinates = as.matrix(select(netCoordinates,x,y))\n\nnetwork = ggnet2(ig,\n                 alpha = .5,\n                 size=\"count\",\n                 label = T,\n                 color = \"line\",palette = \"Set1\",\n                 mode = netCoordinates,\n                 size.cut=5,\n                 edge.alpha = 0.2, edge.size = \"weight\",\n                 arrow.size = 20,\n                 legend.position = \"bottom\")\n\nnetwork\n\n##### Response 1\n\ndataPlot %>%\n  filter(Response==1) %>%\n  group_by(edge) %>%\n  summarize(weight = n()) -> plotDataSummarized\n\nplotDataSummarized %>%\n  mutate(Source = str_split_fixed(edge,\"%\",2)[,1],\n         Target = str_split_fixed(edge,\"%\",2)[,2]) %>%\n  select(-edge) -> plotDataSummarized\n\n\nplotDataSummarized %>%\n  select(Source,Target,weight) -> plotDataSummarized\n\n\n\n\n\n\n\n\n#create network from edgelist\nplotDataSummarized$weight = plotDataSummarized$weight / 500\n\nig = graph.data.frame(plotDataSummarized)\n\ny = data.frame(Node=V(ig)$name)\ny %>% left_join(LineStations,by=\"Node\") -> y\n\nig %>% set_vertex_attr(\"line\", value = as.character(y$Line)) -> ig\n\ny %>% left_join(stationCount,by=\"Node\") -> y\ny[y$Node==\"start\",]$stationCount=1\n\nig %>% set_vertex_attr(\"count\", value = y$stationCount) -> ig\n\n\n\n\n\n#setup plot\nnetwork = ggnet2(ig)\n\nnetCoordinates = select(network$data,label)\nnetCoordinates = left_join(netCoordinates,nodeCoordinates,by = \"label\")\nnetCoordinates = as.matrix(select(netCoordinates,x,y))\n\nnetwork = ggnet2(ig,\n                 alpha = .5,\n                 size=\"count\",\n                 label = T,\n                 color = \"line\",palette = \"Set1\",\n                 mode = netCoordinates,\n                 size.cut=5,\n                 edge.alpha = 0.2, edge.size = \"weight\",\n                 arrow.size = 20,\n                 legend.position = \"bottom\")\n\n#output plot on graphics device\nprint(network)"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"R","language":"R","name":"ir"},"language_info":{"codemirror_mode":"r","file_extension":".r","mimetype":"text/x-r-source","name":"R","pygments_lexer":"r","version":"3.3.1"}},"nbformat":4,"nbformat_minor":0}