{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"c9d29987-117e-687f-1c2d-8005a5c8eca9"},"source":"This is quick tutorial on how to use Network X to visualize the production workflow using dates"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ae5c19dd-e4f1-8711-6749-6272924f378d"},"outputs":[],"source":"import pandas as pd\nimport networkx as nx\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport operator\n%matplotlib inline"},{"cell_type":"markdown","metadata":{"_cell_guid":"22ce7ca8-8dab-0abb-f899-72cbb7ec34bf"},"source":"First get the responses for the first 20,000 rows"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a1b3e26b-8a3c-9f53-540e-34841be36ef6"},"outputs":[],"source":"directory = '../input/'"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"f58de7f1-05e1-bf21-f5e1-f745d5d30b4f"},"outputs":[],"source":"responses = None\nfor i, chunk in enumerate(pd.read_csv(directory+'train_numeric.csv', chunksize = 10000, usecols=['Id','Response'])):\n    if(responses is None):\n        responses = chunk.copy()\n    else:\n        responses = pd.concat([responses, chunk.copy()])\n    if i == 2:\n        break"},{"cell_type":"markdown","metadata":{"_cell_guid":"62056c49-13e2-1e52-2864-2c215421ca64"},"source":"Now get the date stuff"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"1f317cf2-5368-3433-70aa-1952c5ffedba"},"outputs":[],"source":"dates = None\nfor i, chunk in enumerate(pd.read_csv(directory+'train_date.csv', chunksize = 10000)):\n    if(dates is None):\n        dates = chunk.copy()\n    else:\n        dates = pd.concat([dates, chunk.copy()])\n    if i == 2:\n        break"},{"cell_type":"markdown","metadata":{"_cell_guid":"97aaf6b8-8ea3-6348-6e8e-dfff47455bf1"},"source":"Now merge responses into the date data"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"35d0d115-cf4f-189c-71ad-1d517ba342c1"},"outputs":[],"source":"alldata = pd.merge(dates, responses, on=\"Id\")"},{"cell_type":"markdown","metadata":{"_cell_guid":"5354c22b-4bb7-a3a1-ff83-6553ea7cd0ca"},"source":"Scale the dates and remove duplicates in order to get clearer graphs"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"464a5e6f-5e91-ee67-b9f3-2a538bc5384a"},"outputs":[],"source":"alldata[alldata.columns[1:-1]] = alldata[alldata.columns[1:-1]].astype(float)\nalldata[alldata.columns[1:-1]] = alldata[alldata.columns[1:-1]]-alldata[alldata.columns[1:-1]].min()\nalldata[alldata.columns[1:-1]] = alldata[alldata.columns[1:-1]]/alldata[alldata.columns[1:-1]].max()\nalldata.fillna(-1,inplace=1,axis=1)\nremove = []\ncols = alldata.columns[1:-1]\nfor i in range(len(cols)-1):\n    v = alldata[cols[i]].values\n    for j in range(i+1,len(cols)):\n        if np.array_equal(v,alldata[cols[j]].values):\n            remove.append(cols[j]),\nalldata.drop(remove, axis=1, inplace=True),\nremove = []\nfor col in alldata.columns[1:-1]:\n    if alldata[col].std() == 0.0:\n        remove.append(col)\nalldata.drop(remove, axis=1, inplace=True)\npdt = alldata[alldata.Response==1]\nndt = alldata[alldata.Response==0]"},{"cell_type":"markdown","metadata":{"_cell_guid":"9fbab74a-5065-8c4d-0996-bcdc7a142eca"},"source":"Now comes the good bit where we visualize the flow"},{"cell_type":"markdown","metadata":{"_cell_guid":"aa86d54d-ee57-fd00-837b-6a2578d5dd71"},"source":"Let us look at the connections for the first 100 rows of positive responses"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d0a3aac6-bc34-0f19-ee7b-007bb988226b"},"outputs":[],"source":"posnet = nx.DiGraph()\nfor row in range(100):\n    d = pdt.iloc[row][pdt.columns[1:-1]].to_dict()\n    d = { k:v for k, v in d.items() if v!=-1.0 }\n    sorted_x = sorted(d.items(), key=operator.itemgetter(1))\n    prevk = None\n    prevv = None\n    for i, (k,v) in enumerate(sorted_x):\n        if(prevk == None):\n            prevk = k\n            prevv = v\n        else:\n            posnet.add_weighted_edges_from([(prevk,k,v-prevv)])\n            prevk = k\n            prevv = v\n\nplt.figure(figsize=(10,10))\nnx.draw(posnet, with_labels=True)"},{"cell_type":"markdown","metadata":{"_cell_guid":"63eb0185-a4eb-0bcc-3ba4-9b8e81494dc6"},"source":"Let us look at the connections for the first 100 rows of negative responses"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"dcea046a-5913-34b9-2ec4-b94977df1f5d"},"outputs":[],"source":"negnet = nx.DiGraph()\nfor row in range(100):\n    d = ndt.iloc[row][ndt.columns[1:-1]].to_dict()\n    d = { k:v for k, v in d.items() if v!=-1.0 }\n    sorted_x = sorted(d.items(), key=operator.itemgetter(1))\n    prevk = None\n    prevv = None\n    for i, (k,v) in enumerate(sorted_x):\n        if(prevk == None):\n            prevk = k\n            prevv = v\n        else:\n            negnet.add_weighted_edges_from([(prevk,k,v-prevv)])\n            prevk = k\n            prevv = v\n\nplt.figure(figsize=(10,10))\nnx.draw(negnet, with_labels=True)"},{"cell_type":"markdown","metadata":{"_cell_guid":"4d797a74-a6f6-9e9c-a795-1dae3b35efd4"},"source":"One can see that the positive responses seem busier. However, we cannot really see the wood for the trees so lets produce subgraphs associated with to top 10 highest nodes in terms of connections"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"92f918fa-b9ca-cec9-4f92-4f4a60030f94"},"outputs":[],"source":"def GetTopXNode(dct, topX=10):\n    sorted_x = sorted(dct.items(), key=operator.itemgetter(1)) \n    top10 = [node[0] for node in sorted_x[-topX:]]\n    return top10"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"be03983a-57c4-790b-197a-addd0bd1bd1f"},"outputs":[],"source":"plt.figure(figsize=(10,10))\ntop10connections = GetTopXNode(nx.degree(posnet))\nprint(top10connections)\nsub = posnet.subgraph(top10connections)\nnx.draw(sub, with_labels=True, font_size=20)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"62fa1773-499c-6bcc-0dec-7dd004013b30"},"outputs":[],"source":"plt.figure(figsize=(10,10))\ntop10connections = GetTopXNode(nx.degree(negnet))\nprint(top10connections)\nsub = negnet.subgraph(top10connections)\nnx.draw(sub, with_labels=True, font_size=20)"},{"cell_type":"markdown","metadata":{"_cell_guid":"8afb3973-5956-e508-c30c-159b29f0f0fa"},"source":"Please go to [NetworkX][1] in order to see what you can do to analyze graphs\n\n\n  [1]: https://networkx.github.io/"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}