{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"c677ea82-1498-a082-ff5b-3dd5a093d377"},"source":"Visualize the data allignment"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e2efb5f9-2a51-8ea2-00c7-a4b087b3ef94"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\n%matplotlib inline\nimport matplotlib\nimport matplotlib.pyplot as plt"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b22a1975-2ef8-4fc8-c113-f0ace755250d"},"outputs":[],"source":"def load(N=None):\n    return pd.read_csv('../input/train_date.csv', engine='c', nrows=N, index_col='Id')"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"41997593-896e-49e3-7d18-b48e4653e24d"},"outputs":[],"source":"# load some\n\ndf = load(1000)\ndf.shape"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"bdc3b4f0-3885-66c7-83e9-be5eb6c0c4fa"},"outputs":[],"source":"# convert to boolean based on data presents\ndf = np.logical_not(df.isnull())"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3a129e03-1064-4df7-af88-c97983736747"},"outputs":[],"source":"# drop similar rows and columns\ndf = df.drop_duplicates()\ndf = df.T.drop_duplicates().T\ndf.shape\n\n# much less"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"c412b347-1c2f-44d1-80fd-022332ed1bc2"},"outputs":[],"source":"# visualize\n\ndef map(df):\n    fig, ax = plt.subplots()\n    heatmap = ax.pcolor(df, cmap=plt.cm.Blues)\n\n    # want a more natural, table-like display\n    ax.invert_yaxis()\n    ax.xaxis.tick_top()\n\n    ax.set_xticklabels(df.columns, minor=False)\n    ax.set_yticklabels(df.index, minor=False)\n    return plt"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"cb9b6e3f-69f3-40a3-374b-afff4b45253e"},"outputs":[],"source":"map(df).show()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"a4f68ad5-2a1b-909c-a626-52222684f53c"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}