{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DATA PROCESS","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import zipfile\n\nzf = zipfile.ZipFile('../input/bosch-production-line-performance/train_date.csv.zip') \ntrain_date_chunks = pd.read_csv(zf.open('train_date.csv'), iterator=True, chunksize=100000)\npd.options.display.max_columns = None\npd.options.display.max_rows = None\npd.options.display.max_colwidth = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_date_frame():\n    for data_frame in train_date_chunks:\n        yield data_frame\n        \nget_df_date = get_date_frame()\ndf_date = next(get_df_date)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"station_list = []\nfirst_features_in_each_station = [] \n\ndf_date_columns = df_date.columns.tolist()\n\nfor feature in df_date_columns[1:]:\n    station = feature[:feature.index('_D')]\n    if station in station_list:\n        continue\n    else:\n        station_list.append(station)\n        first_features_in_each_station.append(feature)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"global_station_pairs = {}\n\nL0_to_L0 = {}\nL0_to_L1 = {}\nL0_to_L2 = {}\nL0_to_L3 = {}\n\nL1_to_L1 = {}\nL1_to_L2 = {}\nL1_to_L3 = {}\n\nL2_to_L2 = {}\nL2_to_L3 = {}\n\nL3_to_L3 = {}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_line_dict(pair):\n    from_line = pair[0][:2]\n    to_line = pair[1][:2]\n    \n    if from_line == 'L0' and to_line == 'L0':\n        place_line_dict(pair, L0_to_L0)\n    elif from_line == 'L0' and to_line == 'L1': \n        place_line_dict(pair, L0_to_L1)\n    elif from_line == 'L0' and to_line == 'L2': \n        place_line_dict(pair, L0_to_L2)\n    elif from_line == 'L0' and to_line == 'L3': \n        place_line_dict(pair, L0_to_L3)\n        \n    \n    elif from_line == 'L1' and to_line == 'L1': \n        place_line_dict(pair, L1_to_L1)\n    elif from_line == 'L1' and to_line == 'L2': \n        place_line_dict(pair, L1_to_L2)\n    elif from_line == 'L1' and to_line == 'L3': \n        place_line_dict(pair, L1_to_L3)\n\n       \n    elif from_line == 'L2' and to_line == 'L2': \n        place_line_dict(pair, L2_to_L2)\n    elif from_line == 'L2' and to_line == 'L3': \n        place_line_dict(pair, L2_to_L3)\n    \n    elif from_line == 'L3' and to_line == 'L3': \n        place_line_dict(pair, L3_to_L3)\n\n        \ndef place_line_dict(pair, line_dict):\n    try:\n        line_dict[pair] += 1\n    except:\n        line_dict[pair] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"while True:\n    temp_df = pd.DataFrame (np.array(df_date[first_features_in_each_station]), columns = station_list)\n    station_list_for_each_part = temp_df.stack().reset_index(level=1).groupby(level=0, sort=False)['level_1'].apply(list)\n    \n    for each_part in station_list_for_each_part:\n        for station_cursor in range(1, len(each_part)):\n            pair = (each_part[station_cursor-1], each_part[station_cursor])\n            \n            find_line_dict(pair)\n            try:\n                global_station_pairs[pair] += 1\n            except:\n                global_station_pairs[pair] = 1\n\n    try:\n        df_date = next(get_df_date)\n    except:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_max_min_dict(station_pair):\n    values = station_pair.values()\n    return {\"max\":max(values), \"min\":min(values)}\n\nmin_max_global = find_max_min_dict(global_station_pairs)\n\nmin_max_L0_L0 = find_max_min_dict(L0_to_L0)\nmin_max_L0_L1 = find_max_min_dict(L0_to_L1)\nmin_max_L0_L2 = find_max_min_dict(L0_to_L2)\nmin_max_L0_L3 = find_max_min_dict(L0_to_L3)\n\nmin_max_L1_L1 = find_max_min_dict(L1_to_L1)\nmin_max_L1_L2 = find_max_min_dict(L1_to_L2)\nmin_max_L1_L3 = find_max_min_dict(L1_to_L3)\n\nmin_max_L2_L2 = find_max_min_dict(L2_to_L2)\nmin_max_L2_L3 = find_max_min_dict(L2_to_L3)\n\nmin_max_L3_L3 = find_max_min_dict(L3_to_L3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# min_max_global\nmin_max_L0_L0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sorting_method(item):\n    el_1 = item[0][0]\n    x = int(el_1[1:el_1.index('_')])\n    y = int(el_1[el_1.index('S')+1:])\n    \n    el_2 = item[0][1]\n    z = int(el_2[1:el_2.index('_')])\n    t = int(el_2[el_2.index('S')+1:])\n    return(x, y, z, t)\n\nglobal_station_pairs = sorted(global_station_pairs.items(), key = sorting_method)\n\nL0_to_L0 = sorted(L0_to_L0.items(), key = sorting_method)\nL0_to_L1 = sorted(L0_to_L1.items(), key = sorting_method)\nL0_to_L2 = sorted(L0_to_L2.items(), key = sorting_method)\nL0_to_L3 = sorted(L0_to_L3.items(), key = sorting_method)\n\nL1_to_L1 = sorted(L1_to_L1.items(), key = sorting_method)\nL1_to_L2 = sorted(L1_to_L2.items(), key = sorting_method)\nL1_to_L3 = sorted(L1_to_L3.items(), key = sorting_method)\n\nL2_to_L2 = sorted(L2_to_L2.items(), key = sorting_method)\nL2_to_L3 = sorted(L2_to_L3.items(), key = sorting_method)\n\nL3_to_L3 = sorted(L3_to_L3.items(), key = sorting_method)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# global_station_pairs\nL0_to_L0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"node_occurrences = {}\nfor line in global_station_pairs:\n    pair = line[0]\n    try:\n        node_occurrences[pair[0]] += 1\n    except:\n        node_occurrences[pair[0]] = 1\n    try:\n        node_occurrences[pair[1]] += 1\n    except:\n        node_occurrences[pair[1]] = 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NETWORK VISUALIZATION","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import networkx as nx\nimport matplotlib.pyplot as plt\n\nimport plotly.offline as py\nimport plotly.graph_objects as go","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prog = ['dot', 'neato', 'fdp', 'twopi', 'circo'] #layout types\n\nmy_red = 'rgba(255,0,0,0.7)'\nmy_green = 'rgba(0,255,0,0.7)'\nmy_blue = 'rgba(0,0,255,0.7)'\nmy_darkorchid = 'rgba(153,50,204,0.5)'\nmy_yellowish = 'rgba(255,255,153,0.7)'\nmy_blue= 'rgba(221,243,245,1)'\n\nmapping_range_minimum = 0.5\nmapping_range_maximum = 10\n\nmapping_range = mapping_range_maximum - mapping_range_minimum","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_graph(pair_set, layout_style):\n    graph = nx.Graph()\n    for pair in pair_set:\n        stations = pair[0]\n        graph.add_node(stations[0], size = node_occurrences[stations[0]])\n        graph.add_node(stations[1], size = node_occurrences[stations[1]])\n        graph.add_edge(pair[0][0], pair[0][1], occurrence = pair[1])\n    pos_ = nx.drawing.nx_pydot.graphviz_layout(graph, prog=layout_style)\n    return graph, pos_\n\n\ndef make_edge(x, y, color, width, text):\n    return  go.Scatter(x         = x,\n                       y         = y,\n                       line      = dict(width = width,\n                                        color = color),\n                       mode      = 'lines',\n                       hoverinfo = \"text\",\n                       text      = text)\n\ndef make_edge_trace(min_max_flow, graph, pos_, color):\n    edge_trace = []\n    \n    min_flow = min_max_flow[\"min\"]\n    max_flow = min_max_flow[\"max\"]\n    \n    for edge in graph.edges():\n        ch1 = edge[0]\n        ch2 = edge[1]\n\n        x0, y0 = pos_[ch1]\n        x1, y1 = pos_[ch2]\n        \n        flow = graph.edges()[edge][\"occurrence\"]\n        \n        try:\n            line_width = (float(mapping_range * (flow - min_flow)) / (max_flow-min_flow)) + mapping_range_minimum\n        except:\n            line_width = 1 # exception occurs if there is a div by 0\n        \n        trace  = make_edge(x = [x0, (x0+x1)/2, x1], \n                           y = [y0, (y0+y1)/2, y1], \n                           color = color,\n                           width = line_width,\n                           text = [\"\",str(flow),\"\"])\n        \n        edge_trace.append(trace)\n    return edge_trace\n\n\ndef make_node_trace(graph, pos_, is_all_stations):\n    node_x = []\n    node_y = []\n    node_size_l = []\n    node_text_l = []\n    \n    for node in graph.nodes():\n        x, y = pos_[node]\n        node_x.append(x)\n        node_y.append(y)\n        \n        if is_all_stations:\n            node_size = graph.nodes()[node]['size']\n        else:\n            node_size = 20\n        \n        node_size_l.append(node_size)\n        node_text_l.append(str(node))\n    \n    node_trace = go.Scatter(x         = node_x,\n                            y         = node_y,\n                            text      = node_text_l,\n                            textposition = \"top center\",\n                            textfont_size = 12,\n                            mode      = 'markers+text',\n                            hoverinfo = \"text\",\n                            marker    = dict(size  = node_size_l,\n                                             line_width=3,\n                                             color=[],\n                                             showscale=True,\n                                             reversescale=True,\n                                             colorscale='Viridis',\n                                             colorbar=dict(\n                                                    thickness=15,\n                                                    title='Number of Node Connections',\n                                                    xanchor='left',\n                                                    titleside='right'),\n                                             ))\n    \n    node_adjacencies = []\n    for node, adjacencies in enumerate(graph.adjacency()):\n        node_adjacencies.append(len(adjacencies[1]))\n    node_trace.marker.color = node_adjacencies\n    \n    return node_trace\n        \n    \ndef draw(title, pair_set, min_max_flow, edge_color, is_all_stations = False, layout_style = prog[4],):\n    graph, pos_ = make_graph(pair_set, layout_style)\n    \n    edge_trace = make_edge_trace(min_max_flow, graph, pos_, edge_color)\n    node_trace = make_node_trace(graph, pos_, is_all_stations)\n    \n    layout = go.Layout(\n        title = {'text':title, 'x':0.5},\n        paper_bgcolor=my_blue, # transparent background\n        plot_bgcolor=my_blue, # transparent 2nd background\n        xaxis =  {'showgrid': False, 'zeroline': False}, # no gridlines\n        yaxis = {'showgrid': False, 'zeroline': False}, # no gridlines\n        title_font_color=\"red\",\n        hovermode='closest',\n        hoverlabel=dict(\n            font_size=30, \n            font_family=\"Rockwell\")\n    )\n    \n      \n    fig = go.Figure(layout = layout)\n    \n    fig.add_trace(node_trace)\n    for edge in edge_trace:\n        fig.add_trace(edge)\n        \n    fig.update_layout(showlegend = False)\n    fig.update_xaxes(showticklabels = False)\n    fig.update_yaxes(showticklabels = False)\n    fig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L0_to_L0\",\n     pair_set = L0_to_L0, \n     min_max_flow = min_max_L0_L0, \n     edge_color = my_darkorchid,\n     layout_style = prog[2]\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L0_to_L0_split_1\",\n     pair_set = L0_to_L0[:48], \n     min_max_flow = min_max_L0_L0, \n     edge_color = my_darkorchid,\n     layout_style = prog[1]\n    )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L0_to_L0_split_2\",\n     pair_set = L0_to_L0[48:], \n     min_max_flow = min_max_L0_L0, \n     edge_color = my_darkorchid\n    )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L0_to_L1\",\n     pair_set = L0_to_L1, \n     min_max_flow = min_max_L0_L1, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L0_to_L2\",\n     pair_set = L0_to_L2, \n     min_max_flow = min_max_L0_L2, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L0_to_L3\",\n     pair_set = L0_to_L3, \n     min_max_flow = min_max_L0_L3, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L1_to_L1\",\n     pair_set = L1_to_L1, \n     min_max_flow = min_max_L1_L1, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L1_to_L2\",\n     pair_set = L1_to_L2, \n     min_max_flow = min_max_L1_L2, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L1_to_L3\",\n     pair_set = L1_to_L3, \n     min_max_flow = min_max_L1_L3, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L2_to_L2\",\n     pair_set = L2_to_L2, \n     min_max_flow = min_max_L2_L2, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L2_to_L3\",\n     pair_set = L2_to_L3, \n     min_max_flow = min_max_L2_L3, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L3_to_L3\",\n     pair_set = L3_to_L3, \n     min_max_flow = min_max_L3_L3, \n     edge_color = my_darkorchid\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L3_to_L3_split_1\",\n     pair_set = L3_to_L3[:31], \n     min_max_flow = min_max_L3_L3, \n     edge_color = my_darkorchid,\n     layout_style = prog[1]\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"L3_to_L3_split_2\",\n     pair_set = L3_to_L3[31:], \n     min_max_flow = min_max_L3_L3, \n     edge_color = my_darkorchid,\n     layout_style = prog[1]\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"draw(title = \"ALL STATIONS\",\n     pair_set = global_station_pairs, \n     min_max_flow = min_max_global, \n     edge_color = my_darkorchid,\n     is_all_stations = True,\n     layout_style = prog[4],\n    )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}