{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"09d1147c-6f94-ddcf-9e2a-46fa9f932f78"},"source":"I loaded the data and discovered a pattern:\nIt looks like the data set represents a kind of pipeline with different routes\n\nWhat we can do with that? Any comments, critics?"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"5420d783-b32c-73a3-2d5b-d23b25737cb8"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\n%matplotlib inline\nimport matplotlib\nimport matplotlib.pyplot as plt\n\ndef load(N=None):\n    return pd.read_csv('../input/train_date.csv', engine='c', nrows=N, index_col='Id')\n\n# load some data ( you can increase - pattern stays the same)\n\ndf = load(100000)\ndf.shape"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a3c75016-80d3-3371-4469-bfcc0e6e00f3"},"outputs":[],"source":"# conver to boolean and drop similar rows and adjacent columns\ndf = np.logical_not(df.isnull())\n\ndf = df.drop_duplicates()\ndf = df.T.drop_duplicates().T\ndf.shape\n\n# pretty less"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"1d4f78fb-8088-19b5-3fdd-d0be50633b04"},"outputs":[],"source":"# visualize"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3a5bbe0f-8610-0b1b-41f1-40b571792a56"},"outputs":[],"source":"def map(df):\n    fig, ax = plt.subplots()\n    heatmap = ax.pcolor(df, cmap=plt.cm.Blues)\n\n    # want a more natural, table-like display\n    ax.invert_yaxis()\n    ax.xaxis.tick_top()\n\n    ax.set_xticklabels(df.columns, minor=False)\n    ax.set_yticklabels(df.index, minor=False)\n    return plt"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"01e9d290-ea10-40c6-f25a-1b4ce6ac3da2"},"outputs":[],"source":"map(df).show()\n\n# there is a pattern - obviously"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b3c1de91-d25b-9f5a-d7ca-5464bddb07fe"},"outputs":[],"source":"# let's sort\n\ndf = df.sort_values(list(df.columns))\nmap(df).show()\n\n# wow!"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a3abd1fc-6b65-6580-41ba-b7576bf50957"},"outputs":[],"source":"# let's sort the tails\n\ndf = df.sort_values(list(df.columns[-20:-1]))\nmap(df).show()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"22fda374-960f-8db5-5220-f5dfc4b50d76"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}