{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"09d1147c-6f94-ddcf-9e2a-46fa9f932f78"},"source":"I loaded the data and discovered a pattern:\nIt looks like the data set represents a kind of pipeline with different routes\n\nWhat we can do with that? Any comments, critics?"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5420d783-b32c-73a3-2d5b-d23b25737cb8"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\n%matplotlib inline\nimport matplotlib\nimport matplotlib.pyplot as plt\n\ndef load(N=None):\n    return pd.read_csv('../input/train_date.csv', engine='c', nrows=N, index_col='Id')\n\n# load some data ( you can increase - pattern stays the same)\n\ndf = load(100000)\ndf.shape"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a3c75016-80d3-3371-4469-bfcc0e6e00f3"},"outputs":[],"source":"# conver to boolean and drop similar rows and adjacent columns\ndf = np.logical_not(df.isnull())\n\ndf = df.drop_duplicates()\ndf = df.T.drop_duplicates().T\ndf.shape\n\n# pretty less"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"1d4f78fb-8088-19b5-3fdd-d0be50633b04"},"outputs":[],"source":"# visualize"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3a5bbe0f-8610-0b1b-41f1-40b571792a56"},"outputs":[],"source":"def map(df):\n    fig= plt.plot(figsize=(20,10))\n    ax=plt.gca()\n    heatmap = plt.pcolor(df, cmap=plt.cm.Blues)\n    #cmap='RdBu'\n    # want a more natural, table-like display\n    ax.invert_yaxis()\n    ax.xaxis.tick_top()\n\n    ax.set_xticklabels(df.columns, minor=False)\n    #ax.set_yticklabels(df.index, minor=False)\n    ax.settitle()\n    return plt"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"98665adf-e0d0-d536-db4b-e1d84c5ec784"},"outputs":[],"source":"# let's sort\n\ndf = df.sort_values(list(df.columns))\nplt=map(df)\nplt.show()\n\n# wow!"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"01e9d290-ea10-40c6-f25a-1b4ce6ac3da2"},"outputs":[],"source":"fig=map(df)\nfig.show()\n\n# there is a pattern - obviously"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c3684556-a923-ab59-9c77-a173c1012f74"},"outputs":[],"source":"def map(df):\n    fig= plt.plot(figsize=(20,6))\n    ax=plt.gca()\n    heatmap = plt.pcolor(df, cmap=plt.cm.Blues)\n    #cmap='RdBu'\n    # want a more natural, table-like display\n    ax.invert_yaxis()\n    #ax.xaxis.tick_top()\n\n    ax.set_xticklabels(df.columns, minor=False,rotation=90)\n    #ax.set_yticklabels(df.index, minor=False)\n    ax.set_title(\"Production flow across different station-line combinations\")\n    ax.set_xlabel(\"Station-Line-Date Feature\")\n    ax.yaxis.set_major_formatter(plt.NullFormatter())\n    plt.tight_layout()\n    return plt"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a3abd1fc-6b65-6580-41ba-b7576bf50957"},"outputs":[],"source":"# let's sort the tails\n\ndf = df.sort_values(list(df.columns[-20:-1]))\nfig=map(df)\nfig.show()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0996d5c6-0a9a-76ed-a6a1-ddd440c2d762"},"outputs":[],"source":"fig.xlabel(\"Line-Station-Date feature\")\n#plt.ylabel(\"\")\n#fig.gca().yaxis.set_major_formatter(plt.NullFormatter())\nfig.show()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"ee50157d-dd56-e762-0db3-357e1cd96df0"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}