{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "cb207525-2807-81af-adab-a2c27b95cab7"
      },
      "source": [
        "Can we predict anomalies using the numerical feature density? It doesn't look like it...\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "74e5dd47-d604-6354-7502-4afe7b4af08e"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import sys"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "e4ab94c1-f12e-cc04-9a24-1d7c9aed4879"
      },
      "source": [
        "Author = Alex Galea   \n",
        "Date = Sept 12th, 2016"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e45d27d0-27ff-6c42-0675-ddcb48931e76"
      },
      "outputs": [],
      "source": [
        "sys.version"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "427db241-5f74-b9c0-9775-05d765cb1a89"
      },
      "source": [
        "**Loading the numerical dataset**:\n",
        "   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9d1e6f25-b2f3-a30f-a1da-660e782fcfeb"
      },
      "outputs": [],
      "source": [
        "def df_load(file, batch_size, skip = 1):\n",
        "    ''' Build dataframe by iterating over chunks. Option to skip chunks and\n",
        "        therefore read in less data. '''\n",
        "\n",
        "    reader = pd.read_csv(file, chunksize=batch_size,\n",
        "                         dtype=np.float16)\n",
        "\n",
        "    df = pd.concat((chunk for i, chunk in enumerate(reader) if i % skip == 0))\n",
        "\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "91b6f12d-9cc6-8d96-68cd-05a708053b9f"
      },
      "outputs": [],
      "source": [
        "input_path = '../input/'\n",
        "df_num = df_load(input_path+'train_numeric.csv',\n",
        "                 batch_size=100000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c1b8f01f-9cd7-a3b0-5694-6d836c3021de"
      },
      "outputs": [],
      "source": [
        "df_num.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d9548704-5529-46bc-8381-e4064dc197f2"
      },
      "source": [
        "Get a random selection of models:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2382381e-4b16-cf52-9512-458044d36e8d"
      },
      "outputs": [],
      "source": [
        "np.random.seed(117)\n",
        "num_samples = 200\n",
        "random_picks = (np.array(np.random.random(num_samples)) * df_num.shape[1]).astype(int)\n",
        "random_picks = sorted(random_picks)\n",
        "print(random_picks)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "c528835f-bf0e-8371-034d-93a926cb9b7c"
      },
      "source": [
        "Plot the feature densities, comparing response = 1 targets with response = 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "66202e9b-f5c3-8a34-38c4-95e119c471de"
      },
      "outputs": [],
      "source": [
        "for feature in random_picks[:150]:\n",
        "    fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n",
        "    # Plot linear scale feature density\n",
        "    df_num.iloc[:, feature][(df_num.Response == 0)].dropna().hist(bins=50, alpha=0.4, normed=True, ax=axes[0],\n",
        "                                                         label='%d response 0 samples' % len(df_num.iloc[:, feature][(df_num.Response == 0)].dropna()))\n",
        "    df_num.iloc[:, feature][(df_num.Response == 1)].dropna().hist(bins=50, alpha=0.4, color='r', normed=True, ax=axes[0],\n",
        "                                                         label='%d response 1 samples' % len(df_num.iloc[:, feature][(df_num.Response == 1)].dropna()))\n",
        "    # Plot log scale feature density\n",
        "    df_num.iloc[:, feature][(df_num.Response == 0)].dropna().hist(bins=50, alpha=0.4, normed=True, ax=axes[1], log=True)\n",
        "    df_num.iloc[:, feature][(df_num.Response == 1)].dropna().hist(bins=50, alpha=0.4, color='r', normed=True, ax=axes[1], log=True)\n",
        "\n",
        "    axes[0].legend(loc='upper left')\n",
        "    plt.suptitle('Feature column \"%s\" histogram' % df_num.columns[feature], fontsize=15)\n",
        "    plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "aace4901-0acd-1d15-3979-18297367be81"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}