{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "c9d29987-117e-687f-1c2d-8005a5c8eca9"
      },
      "source": [
        "This is quick tutorial on how to use Network X to visualize the production workflow using dates"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ae5c19dd-e4f1-8711-6749-6272924f378d"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import networkx as nx\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import operator\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "22ce7ca8-8dab-0abb-f899-72cbb7ec34bf"
      },
      "source": [
        "First get the responses for the first 20,000 rows"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a1b3e26b-8a3c-9f53-540e-34841be36ef6"
      },
      "outputs": [],
      "source": [
        "directory = '../input/'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f58de7f1-05e1-bf21-f5e1-f745d5d30b4f"
      },
      "outputs": [],
      "source": [
        "responses = None\n",
        "for i, chunk in enumerate(pd.read_csv(directory+'train_numeric.csv', chunksize = 10000, usecols=['Id','Response'])):\n",
        "    if(responses is None):\n",
        "        responses = chunk.copy()\n",
        "    else:\n",
        "        responses = pd.concat([responses, chunk.copy()])\n",
        "    if i == 2:\n",
        "        break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "62056c49-13e2-1e52-2864-2c215421ca64"
      },
      "source": [
        "Now get the date stuff"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1f317cf2-5368-3433-70aa-1952c5ffedba"
      },
      "outputs": [],
      "source": [
        "dates = None\n",
        "for i, chunk in enumerate(pd.read_csv(directory+'train_date.csv', chunksize = 10000)):\n",
        "    if(dates is None):\n",
        "        dates = chunk.copy()\n",
        "    else:\n",
        "        dates = pd.concat([dates, chunk.copy()])\n",
        "    if i == 2:\n",
        "        break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "97aaf6b8-8ea3-6348-6e8e-dfff47455bf1"
      },
      "source": [
        "Now merge responses into the date data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "35d0d115-cf4f-189c-71ad-1d517ba342c1"
      },
      "outputs": [],
      "source": [
        "alldata = pd.merge(dates, responses, on=\"Id\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "5354c22b-4bb7-a3a1-ff83-6553ea7cd0ca"
      },
      "source": [
        "Scale the dates and remove duplicates in order to get clearer graphs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "464a5e6f-5e91-ee67-b9f3-2a538bc5384a"
      },
      "outputs": [],
      "source": [
        "alldata[alldata.columns[1:-1]] = alldata[alldata.columns[1:-1]].astype(float)\n",
        "alldata[alldata.columns[1:-1]] = alldata[alldata.columns[1:-1]]-alldata[alldata.columns[1:-1]].min()\n",
        "alldata[alldata.columns[1:-1]] = alldata[alldata.columns[1:-1]]/alldata[alldata.columns[1:-1]].max()\n",
        "alldata.fillna(-1,inplace=1,axis=1)\n",
        "remove = []\n",
        "cols = alldata.columns[1:-1]\n",
        "for i in range(len(cols)-1):\n",
        "    v = alldata[cols[i]].values\n",
        "    for j in range(i+1,len(cols)):\n",
        "        if np.array_equal(v,alldata[cols[j]].values):\n",
        "            remove.append(cols[j]),\n",
        "alldata.drop(remove, axis=1, inplace=True),\n",
        "remove = []\n",
        "for col in alldata.columns[1:-1]:\n",
        "    if alldata[col].std() == 0.0:\n",
        "        remove.append(col)\n",
        "alldata.drop(remove, axis=1, inplace=True)\n",
        "pdt = alldata[alldata.Response==1]\n",
        "ndt = alldata[alldata.Response==0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "9fbab74a-5065-8c4d-0996-bcdc7a142eca"
      },
      "source": [
        "Now comes the good bit where we visualize the flow"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "aa86d54d-ee57-fd00-837b-6a2578d5dd71"
      },
      "source": [
        "Let us look at the connections for the first 100 rows of positive responses"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d0a3aac6-bc34-0f19-ee7b-007bb988226b"
      },
      "outputs": [],
      "source": [
        "posnet = nx.DiGraph()\n",
        "for row in range(100):\n",
        "    d = pdt.iloc[row][pdt.columns[1:-1]].to_dict()\n",
        "    d = { k:v for k, v in d.items() if v!=-1.0 }\n",
        "    sorted_x = sorted(d.items(), key=operator.itemgetter(1))\n",
        "    prevk = None\n",
        "    prevv = None\n",
        "    for i, (k,v) in enumerate(sorted_x):\n",
        "        if(prevk == None):\n",
        "            prevk = k\n",
        "            prevv = v\n",
        "        else:\n",
        "            posnet.add_weighted_edges_from([(prevk,k,v-prevv)])\n",
        "            prevk = k\n",
        "            prevv = v\n",
        "\n",
        "plt.figure(figsize=(10,10))\n",
        "nx.draw(posnet, with_labels=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "63eb0185-a4eb-0bcc-3ba4-9b8e81494dc6"
      },
      "source": [
        "Let us look at the connections for the first 100 rows of negative responses"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "dcea046a-5913-34b9-2ec4-b94977df1f5d"
      },
      "outputs": [],
      "source": [
        "negnet = nx.DiGraph()\n",
        "for row in range(100):\n",
        "    d = ndt.iloc[row][ndt.columns[1:-1]].to_dict()\n",
        "    d = { k:v for k, v in d.items() if v!=-1.0 }\n",
        "    sorted_x = sorted(d.items(), key=operator.itemgetter(1))\n",
        "    prevk = None\n",
        "    prevv = None\n",
        "    for i, (k,v) in enumerate(sorted_x):\n",
        "        if(prevk == None):\n",
        "            prevk = k\n",
        "            prevv = v\n",
        "        else:\n",
        "            negnet.add_weighted_edges_from([(prevk,k,v-prevv)])\n",
        "            prevk = k\n",
        "            prevv = v\n",
        "\n",
        "plt.figure(figsize=(10,10))\n",
        "nx.draw(negnet, with_labels=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "4d797a74-a6f6-9e9c-a795-1dae3b35efd4"
      },
      "source": [
        "One can see that the positive responses seem busier. However, we cannot really see the wood for the trees so lets produce subgraphs associated with to top 10 highest nodes in terms of connections"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "92f918fa-b9ca-cec9-4f92-4f4a60030f94"
      },
      "outputs": [],
      "source": [
        "def GetTopXNode(dct, topX=10):\n",
        "    sorted_x = sorted(dct.items(), key=operator.itemgetter(1)) \n",
        "    top10 = [node[0] for node in sorted_x[-topX:]]\n",
        "    return top10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "be03983a-57c4-790b-197a-addd0bd1bd1f"
      },
      "outputs": [],
      "source": [
        "plt.figure(figsize=(10,10))\n",
        "top10connections = GetTopXNode(nx.degree(posnet))\n",
        "print(top10connections)\n",
        "sub = posnet.subgraph(top10connections)\n",
        "nx.draw(sub, with_labels=True, font_size=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "62fa1773-499c-6bcc-0dec-7dd004013b30"
      },
      "outputs": [],
      "source": [
        "plt.figure(figsize=(10,10))\n",
        "top10connections = GetTopXNode(nx.degree(negnet))\n",
        "print(top10connections)\n",
        "sub = negnet.subgraph(top10connections)\n",
        "nx.draw(sub, with_labels=True, font_size=20)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "8afb3973-5956-e508-c30c-159b29f0f0fa"
      },
      "source": [
        "Please go to [NetworkX][1] in order to see what you can do to analyze graphs\n",
        "\n",
        "\n",
        "  [1]: https://networkx.github.io/"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}