{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "09d1147c-6f94-ddcf-9e2a-46fa9f932f78"
      },
      "source": [
        "I loaded the data and discovered a pattern:\n",
        "It looks like the data set represents a kind of pipeline with different routes\n",
        "\n",
        "What we can do with that? Any comments, critics?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5420d783-b32c-73a3-2d5b-d23b25737cb8"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "%matplotlib inline\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def load(N=None):\n",
        "    return pd.read_csv('../input/train_date.csv', engine='c', nrows=N, index_col='Id')\n",
        "\n",
        "# load some data ( you can increase - pattern stays the same)\n",
        "\n",
        "df = load(100000)\n",
        "df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "735592d0-8d45-aed8-b7de-e2681569593e"
      },
      "outputs": [],
      "source": [
        "# conver to boolean and drop similar rows and adjacent columns\n",
        "df = np.logical_not(df.isnull())\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "581ef0c0-edea-41a2-20e6-cf3bd55973b7"
      },
      "outputs": [],
      "source": [
        "df = df.drop_duplicates()\n",
        "print(df.shape)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a3c75016-80d3-3371-4469-bfcc0e6e00f3"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "df = df.T.drop_duplicates().T\n",
        "df.shape\n",
        "\n",
        "# pretty less"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1d4f78fb-8088-19b5-3fdd-d0be50633b04"
      },
      "outputs": [],
      "source": [
        "# visualize"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3a5bbe0f-8610-0b1b-41f1-40b571792a56"
      },
      "outputs": [],
      "source": [
        "def map(df):\n",
        "    fig, ax = plt.subplots()\n",
        "    heatmap = ax.pcolor(df, cmap=plt.cm.Blues)\n",
        "\n",
        "    # want a more natural, table-like display\n",
        "    ax.invert_yaxis()\n",
        "    ax.xaxis.tick_top()\n",
        "\n",
        "    ax.set_xticklabels(df.columns, minor=False)\n",
        "    ax.set_yticklabels(df.index, minor=False)\n",
        "    return plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "01e9d290-ea10-40c6-f25a-1b4ce6ac3da2"
      },
      "outputs": [],
      "source": [
        "map(df).show()\n",
        "\n",
        "# there is a pattern - obviously"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b3c1de91-d25b-9f5a-d7ca-5464bddb07fe"
      },
      "outputs": [],
      "source": [
        "# let's sort\n",
        "\n",
        "df = df.sort_values(list(df.columns))\n",
        "map(df).show()\n",
        "\n",
        "# wow!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a3abd1fc-6b65-6580-41ba-b7576bf50957"
      },
      "outputs": [],
      "source": [
        "# let's sort the tails\n",
        "\n",
        "df = df.sort_values(list(df.columns[-20:-1]))\n",
        "map(df).show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "22fda374-960f-8db5-5220-f5dfc4b50d76"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}