{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "eaa19822-2b3d-496e-b6d9-958246d0df80"
      },
      "source": [
        "This is a new version of my older script which instead of visualizing a number of jobs presents the whole data. This allows assessing typical paths across the production facility. I provide three variants - one for all jobs and two additional ones filtered to Response 0 and 1. Widths of the edges indicate the number of jobs on this station pair."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "fd74c71c-ec4c-e9cd-ff3c-fd132dcb0568"
      },
      "outputs": [],
      "source": [
        "library(data.table)\n",
        "library(dplyr)\n",
        "\n",
        "\n",
        "#imports for plotting\n",
        "require(GGally)\n",
        "library(network)\n",
        "library(sna)\n",
        "library(ggplot2)\n",
        "require(igraph)\n",
        "require(intergraph)\n",
        "library(stringr)\n",
        "\n",
        "\n",
        "topFeaturesDate <- c(\"L3_S33_D3856\",\"L3_S34_D3875\",\"L3_S30_D3496\",\"L3_S29_D3316\",\"L3_S37_D3942\",\n",
        "                       \"L3_S29_D3474\",\"L0_S0_D1\" ,    \"L3_S30_D3506\",\"L3_S30_D3501\",\"L3_S30_D3726\",\n",
        "                       \"L3_S30_D3566\",\"L0_S3_D70\" ,   \"L3_S35_D3895\",\"L0_S8_D145\"  , \"L2_S26_D3037\",\n",
        "                       \"L3_S35_D3886\",\"L3_S35_D3900\",\"L0_S13_D355\" , \"L3_S36_D3919\",\"L0_S1_D26\"   ,\n",
        "                       \"L3_S36_D3928\",\"L2_S26_D3081\",\"L1_S24_D1511\",\"L0_S12_D331\" , \"L1_S24_D1809\",\n",
        "                       \"L0_S2_D34\",\"L1_S24_D1536\",\"L0_S14_D360\" , \"L1_S24_D1558\",\"L0_S23_D617\" ,\n",
        "                       \"L0_S6_D120\",\"L1_S24_D1566\",\"L1_S24_D1576\",\"L0_S20_D462\" , \"L0_S4_D106\"  ,\n",
        "                       \"L0_S1_D30\",\"L0_S19_D454\",  \"L0_S7_D137\",\"L0_S9_D152\" ,  \"L1_S24_D1826\",\n",
        "                       \"L0_S16_D423\",\"L0_S17_D432\" , \"L1_S24_D1568\",\"L0_S5_D115\" ,  \"L1_S24_D1674\",\n",
        "                       \"L1_S24_D1570\",\"L0_S18_D437\"  ,\"L0_S10_D216\",\"L0_S9_D157\"  , \"L1_S24_D1522\",\n",
        "                       \"L0_S4_D111\",\"L2_S27_D3130\",\"L0_S11_D280\" , \"L0_S22_D543\" , \"L0_S9_D162\"  ,\n",
        "                       \"L0_S10_D221\",\"L0_S15_D395\",  \"L0_S9_D167\"  , \"L1_S24_D1765\",\"L3_S31_D3836\",\n",
        "                       \"L0_S22_D548\",\"L1_S24_D1583\",\"L0_S21_D469\",  \"L1_S24_D1770\",\"L0_S10_D231\" ,\n",
        "                       \"L0_S22_D553\",\"L0_S21_D474\" , \"L1_S25_D1854\",\"L3_S32_D3852\",\n",
        "                       \"L3_S39_D3966\",\"L0_S21_D484\" , \"L1_S25_D1867\",\"L3_S44_D4101\",\"L3_S40_D3981\",\n",
        "                       \"L3_S38_D3953\",\"L0_S22_D558\" , \"L1_S25_D2780\",\"L3_S43_D4062\",\"L3_S49_D4208\",\n",
        "                       \"L2_S28_D3223\",\"L1_S24_D1116\",\"L1_S25_D2788\",\"L3_S45_D4125\",\"L3_S49_D4218\",\n",
        "                       \"L3_S50_D4242\",\"L3_S40_D3985\",\"L1_S25_D2471\",\"L1_S25_D2058\",\"L1_S25_D1891\",\n",
        "                       \"L1_S24_D677\",\"L1_S24_D1135\",\"L3_S43_D4082\",\"L1_S25_D2792\",\"L1_S25_D1883\",\n",
        "                       \"L3_S41_D3997\",\"L1_S25_D2098\",\"L1_S25_D1887\",\"L1_S25_D2754\",\"L1_S25_D2138\",\n",
        "                       \"L3_S51_D4255\",\"L1_S24_D1155\",\"L1_S25_D1980\",\"L1_S25_D2790\",\"L3_S47_D4140\",\n",
        "                     \"L1_S25_D3011\",\"L1_S25_D2240\",\"L1_S25_D2230\",\"L1_S25_D2798\",\"L1_S25_D1902\",\n",
        "                       \"L1_S24_D702\",\"L1_S24_D697\",\"L3_S48_D4194\",\"L1_S25_D2996\",\"L1_S24_D1413\",\n",
        "                       \"L1_S24_D818\",\"L1_S24_D1168\",\"L1_S24_D1163\",\"L1_S24_D1171\",\"L1_S24_D804\", \n",
        "                       \"L1_S24_D909\",\"L1_S24_D801\",\"L1_S25_D2801\",\"L1_S24_D1018\",\"L1_S25_D2206\",\"L1_S25_D2505\")\n",
        "\n",
        "#one expansion option would be to select a time frame to visualized\n",
        "dtNum <- fread(\"../input/train_numeric.csv\", select = c(\"Id\", \"Response\"))\n",
        "dtDate <- fread(\"../input/train_date.csv\",select = c(\"Id\",topFeaturesDate))\n",
        "\n",
        "#for each job identify which stations are passed through and for those store the minimum time\n",
        "for (station in paste0(\"S\",0:51))\n",
        "{\n",
        "  cols = min(which((grepl(station,colnames(dtDate)))))\n",
        "  if(!cols==Inf){\n",
        "    dtDate[,paste0(station) := dtDate[,cols,with = FALSE]]\n",
        "  }\n",
        "}\n",
        "\n",
        "#limit data to only when passed through station X\n",
        "dtStations = dtDate[,!grepl(\"L\",colnames(dtDate)),with=F]\n",
        "\n",
        "#melt data to go from wide to long format\n",
        "#join with numeric to have Response\n",
        "#remove NA entries - these are plentiful as after melting each station-job combination has its own row\n",
        "\n",
        "melt(dtStations,id.vars=c(\"Id\")) %>%\n",
        "  left_join(dtNum, by = \"Id\")  %>%\n",
        "  filter(!is.na(value)) -> dtStationsMFiltered\n",
        "\n",
        "dtStationsMFiltered %>%\n",
        "  group_by(variable) %>%\n",
        "  summarize(stationCount=n()) -> stationCount\n",
        "\n",
        "\n",
        "stationCount = rbind(stationCount,data.frame(variable=c(\"result-0\",\"result-1\"),\n",
        "                                             stationCount=c(table(dtNum$Response)[1],\n",
        "                                                            table(dtNum$Response)[2])))\n",
        "\n",
        "colnames(stationCount) = c(\"Node\",\"stationCount\")\n",
        "\n",
        "\n",
        "  \n",
        "\n",
        "#sort entries by ascending time\n",
        "dtStationsMFiltered %>%\n",
        "  arrange(value) -> dtStationsMFiltered\n",
        "\n",
        "\n",
        "\n",
        "dtStationsMFiltered %>%\n",
        "  group_by(Id) %>%\n",
        "  mutate(nextStation = lead(variable)) -> edgelistsComplete\n",
        "\n",
        "rm(dtStations)\n",
        "rm(dtStationsMFiltered)\n",
        "rm(dtNum)\n",
        "invisible(gc())\n",
        "\n",
        "edgelistsComplete %>%\n",
        "  group_by(Id) %>%\n",
        "  filter(!(variable %in% nextStation)) -> startingPoints\n",
        "\n",
        "\n",
        "startingPoints %>%\n",
        "  mutate(nextStation = variable) -> startingPoints\n",
        "\n",
        "startingPoints$variable = as.character(startingPoints$variable)\n",
        "startingPoints$nextStation = as.character(startingPoints$nextStation)\n",
        "\n",
        "startingPoints$variable = \"start\"\n",
        "\n",
        "edgelistsComplete %>%\n",
        "  group_by(Id) %>%\n",
        "  filter(!(nextStation %in% variable)) -> endPoints\n",
        "\n",
        "endPoints$variable = as.character(endPoints$variable)\n",
        "\n",
        "endPoints$nextStation = paste0(\"result-\",endPoints$Response)\n",
        "\n",
        "edgelistsComplete$variable = as.character(edgelistsComplete$variable)\n",
        "edgelistsComplete$nextStation = as.character(edgelistsComplete$nextStation)\n",
        "\n",
        "edgelistsComplete %>% filter(nextStation!=\"NA\") -> edgelistsComplete\n",
        "\n",
        "dataPlot = rbind(startingPoints,edgelistsComplete,endPoints)\n",
        "dataPlot$variable = as.factor(dataPlot$variable)\n",
        "dataPlot$nextStation = as.factor(dataPlot$nextStation)\n",
        "\n",
        "rm(endPoints)\n",
        "rm(startingPoints)\n",
        "invisible(gc())\n",
        "\n",
        "dataPlot$edge = paste0(dataPlot$variable,\"%\",dataPlot$nextStation)\n",
        "\n",
        "dataPlot %>%\n",
        "  group_by(edge) %>%\n",
        "  summarize(weight = n()) -> plotDataSummarized\n",
        "\n",
        "\n",
        "plotDataSummarized %>%\n",
        "  mutate(Source = str_split_fixed(edge,\"%\",2)[,1],\n",
        "         Target = str_split_fixed(edge,\"%\",2)[,2]) %>%\n",
        "  select(-edge) -> plotDataSummarized\n",
        "\n",
        "\n",
        "plotDataSummarized %>%\n",
        "  select(Source,Target,weight) -> plotDataSummarized\n",
        "\n",
        "\n",
        "#for each row obtain the subsequent statoin\n",
        "\n",
        "\n",
        "\n",
        "#create a station-line mapping lookup\n",
        "LineStations = NULL\n",
        "for (station in unique(plotDataSummarized$Source)){\n",
        "  if(station!=\"start\")\n",
        "  {\n",
        "    x=paste0(\"_\",station,\"_\")\n",
        "    y=head(colnames(dtDate)[which(grepl(x,colnames(dtDate)))],1)\n",
        "    y=strsplit(y,\"_\")[[1]][1]\n",
        "    LineStations = rbind(LineStations,data.frame(Node=station,Line=y))\n",
        "  }\n",
        "}\n",
        "LineStations = rbind(LineStations,data.frame(Node=c(\"result-1\",\"result-0\",\"start\"),Line=c(\"Outcome\",\"Outcome\",\"START\")))\n",
        "\n",
        "rm(dtDate)\n",
        "invisible()\n",
        "\n",
        "#plotting format\n",
        "options(repr.plot.width=15, repr.plot.height=5)\n",
        "\n",
        "\n",
        "#create network from edgelist\n",
        "plotDataSummarized$weight = plotDataSummarized$weight / 100000\n",
        "\n",
        "ig = graph.data.frame(plotDataSummarized)\n",
        "\n",
        "y = data.frame(Node=V(ig)$name)\n",
        "y %>% left_join(LineStations,by=\"Node\") -> y\n",
        "\n",
        "ig %>% set_vertex_attr(\"line\", value = as.character(y$Line)) -> ig\n",
        "\n",
        "y %>% left_join(stationCount,by=\"Node\") -> y\n",
        "y[y$Node==\"start\",]$stationCount=1\n",
        "\n",
        "ig %>% set_vertex_attr(\"count\", value = y$stationCount) -> ig\n",
        "\n",
        "\n",
        "nodeCoordinates=data.frame(label=c(\"start\",\"S0\",\"S1\",\"S2\",\"S3\",\"S4\",\"S5\",\"S6\",\n",
        "                                   \"S7\",\"S8\",\"S9\",\"S10\",\"S11\",\"S12\",\"S13\",\n",
        "                                   \"S14\",\"S15\",\"S16\",\"S17\",\"S18\",\"S19\",\n",
        "                                   \"S20\",\"S21\",\"S22\",\"S23\",\"S24\",\"S25\",\n",
        "                                   \"S26\",\"S27\",\"S28\",\"S29\",\"S30\",\"S31\",\n",
        "                                   \"S32\",\"S33\",\"S34\",\"S35\",\"S36\",\"S37\",\n",
        "                                   \"S38\",\"S39\",\"S40\",\"S41\",\"S43\",\n",
        "                                   \"S44\",\"S45\",\"S47\",\"S48\",\"S49\",\n",
        "                                   \"S50\",\"S51\",\"result-0\",\"result-1\"),\n",
        "                           x=c(0,\n",
        "                               1,2,3,3,4,4,5,5,6,6.75,8,7.25,\n",
        "                               1,2,3,3,4,4,5,5,6,6.75,8,7.25,\n",
        "                               6,6,6.75,8,7.25,\n",
        "                               9,10,11,11,11,12,12,13,14,15,\n",
        "                               9,10,11,12,12,13,14,15,16,16,17,\n",
        "                               18,18),\n",
        "                           y=c(5,\n",
        "                               9,9,10,8,10,8,10,8,9,10,9,8,\n",
        "                               5,5,6,4,6,4,6,4,5,6,5,4,\n",
        "                               2,0,2,1,0,\n",
        "                               7,7,8,7,6,8,6,7,7,7,\n",
        "                               3,3,3,4,2,3,3,3,4,2,3,\n",
        "                               7,3))\n",
        "\n",
        "\n",
        "#setup plot\n",
        "network = ggnet2(ig)\n",
        "\n",
        "netCoordinates = select(network$data,label)\n",
        "netCoordinates = left_join(netCoordinates,nodeCoordinates,by = \"label\")\n",
        "netCoordinates = as.matrix(select(netCoordinates,x,y))\n",
        "\n",
        "network = ggnet2(ig,\n",
        "                 alpha = .5,\n",
        "                 size=\"count\",\n",
        "                 label = T,\n",
        "                 color = \"line\",palette = \"Set1\",\n",
        "                 mode = netCoordinates,\n",
        "                 size.cut=5,\n",
        "                 arrow.size = 20,\n",
        "                 edge.alpha = 0.2, edge.size = \"weight\",\n",
        "                 legend.position = \"bottom\")\n",
        "\n",
        "network\n",
        "\n",
        "#### Response 0\n",
        "\n",
        "\n",
        "\n",
        "dataPlot %>%\n",
        "  filter(Response == 0) %>%\n",
        "  group_by(edge) %>%\n",
        "  summarize(weight = n()) -> plotDataSummarized\n",
        "\n",
        "plotDataSummarized %>%\n",
        "  mutate(Source = str_split_fixed(edge,\"%\",2)[,1],\n",
        "         Target = str_split_fixed(edge,\"%\",2)[,2]) %>%\n",
        "  select(-edge) -> plotDataSummarized\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "plotDataSummarized %>%\n",
        "  select(Source,Target,weight) -> plotDataSummarized\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#create network from edgelist\n",
        "plotDataSummarized$weight = plotDataSummarized$weight /100000\n",
        "\n",
        "ig = graph.data.frame(plotDataSummarized)\n",
        "\n",
        "y = data.frame(Node=V(ig)$name)\n",
        "y %>% left_join(LineStations,by=\"Node\") -> y\n",
        "\n",
        "ig %>% set_vertex_attr(\"line\", value = as.character(y$Line)) -> ig\n",
        "\n",
        "y %>% left_join(stationCount,by=\"Node\") -> y\n",
        "y[y$Node==\"start\",]$stationCount=1\n",
        "\n",
        "ig %>% set_vertex_attr(\"count\", value = y$stationCount) -> ig\n",
        "\n",
        "\n",
        "\n",
        "#setup plot\n",
        "network = ggnet2(ig)\n",
        "\n",
        "netCoordinates = select(network$data,label)\n",
        "netCoordinates = left_join(netCoordinates,nodeCoordinates,by = \"label\")\n",
        "netCoordinates = as.matrix(select(netCoordinates,x,y))\n",
        "\n",
        "network = ggnet2(ig,\n",
        "                 alpha = .5,\n",
        "                 size=\"count\",\n",
        "                 label = T,\n",
        "                 color = \"line\",palette = \"Set1\",\n",
        "                 mode = netCoordinates,\n",
        "                 size.cut=5,\n",
        "                 edge.alpha = 0.2, edge.size = \"weight\",\n",
        "                 arrow.size = 20,\n",
        "                 legend.position = \"bottom\")\n",
        "\n",
        "network\n",
        "\n",
        "##### Response 1\n",
        "\n",
        "dataPlot %>%\n",
        "  filter(Response==1) %>%\n",
        "  group_by(edge) %>%\n",
        "  summarize(weight = n()) -> plotDataSummarized\n",
        "\n",
        "plotDataSummarized %>%\n",
        "  mutate(Source = str_split_fixed(edge,\"%\",2)[,1],\n",
        "         Target = str_split_fixed(edge,\"%\",2)[,2]) %>%\n",
        "  select(-edge) -> plotDataSummarized\n",
        "\n",
        "\n",
        "plotDataSummarized %>%\n",
        "  select(Source,Target,weight) -> plotDataSummarized\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#create network from edgelist\n",
        "plotDataSummarized$weight = plotDataSummarized$weight / 500\n",
        "\n",
        "ig = graph.data.frame(plotDataSummarized)\n",
        "\n",
        "y = data.frame(Node=V(ig)$name)\n",
        "y %>% left_join(LineStations,by=\"Node\") -> y\n",
        "\n",
        "ig %>% set_vertex_attr(\"line\", value = as.character(y$Line)) -> ig\n",
        "\n",
        "y %>% left_join(stationCount,by=\"Node\") -> y\n",
        "y[y$Node==\"start\",]$stationCount=1\n",
        "\n",
        "ig %>% set_vertex_attr(\"count\", value = y$stationCount) -> ig\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#setup plot\n",
        "network = ggnet2(ig)\n",
        "\n",
        "netCoordinates = select(network$data,label)\n",
        "netCoordinates = left_join(netCoordinates,nodeCoordinates,by = \"label\")\n",
        "netCoordinates = as.matrix(select(netCoordinates,x,y))\n",
        "\n",
        "network = ggnet2(ig,\n",
        "                 alpha = .5,\n",
        "                 size=\"count\",\n",
        "                 label = T,\n",
        "                 color = \"line\",palette = \"Set1\",\n",
        "                 mode = netCoordinates,\n",
        "                 size.cut=5,\n",
        "                 edge.alpha = 0.2, edge.size = \"weight\",\n",
        "                 arrow.size = 20,\n",
        "                 legend.position = \"bottom\")\n",
        "\n",
        "#output plot on graphics device\n",
        "print(network)"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "R",
      "language": "R",
      "name": "ir"
    },
    "language_info": {
      "codemirror_mode": "r",
      "file_extension": ".r",
      "mimetype": "text/x-r-source",
      "name": "R",
      "pygments_lexer": "r",
      "version": "3.3.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}