{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "a3f3029b-2c29-44dd-74c8-3c9a74592663"
      },
      "source": [
        "Can we predict anomalies using the numerical feature density? It doesn't look like it..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "270a5113-a955-2bbb-2a38-e059717fe54b"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import sys\n",
        "from collections import defaultdict\n",
        "import random"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "e7761853-ba0e-671a-e23f-388332d628a6"
      },
      "source": [
        "Author = Eliot Andres"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "111f9d0a-375a-ea58-0849-dc2d3022c44d"
      },
      "outputs": [],
      "source": [
        "sys.version"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "510f5c96-2424-8dbd-76fc-d9787bd761ad"
      },
      "source": [
        "**Loading the numerical dataset in chunks:**\n",
        "   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3824394e-d849-001b-1fb6-982745976ec4"
      },
      "outputs": [],
      "source": [
        "def df_load(file, batch_size, skip = 1, usecolsCustom = None):\n",
        "    ''' Build dataframe by iterating over chunks. Option to skip chunks and\n",
        "        therefore read in less data. '''\n",
        "\n",
        "    reader = pd.read_csv(file, nrows=batch_size,\n",
        "                         dtype=np.float32, usecols=usecolsCustom)\n",
        "\n",
        "    '''df = pd.concat((chunk for i, chunk in enumerate(reader) if i % skip == 0))'''\n",
        "\n",
        "    return reader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a32d1324-947e-88c4-8b43-2f07df0be38f"
      },
      "outputs": [],
      "source": [
        "input_path = '../input/'\n",
        "df_num = df_load(input_path+'train_numeric.csv',\n",
        "                 batch_size=100000, skip=20, usecolsCustom=['Id', 'Response'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b11eaa09-fc74-7ac1-6bca-6b3a30e38825"
      },
      "outputs": [],
      "source": [
        "df_date = df_load(input_path+'train_date.csv',\n",
        "                 batch_size=100000, skip=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a28dc024-adb6-69a1-433a-38e60b8dd689"
      },
      "outputs": [],
      "source": [
        "df_date.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "492fdfe6-2f5a-4fd7-c4f6-9ecfba2b402c"
      },
      "source": [
        "**Create a set (for fast lookup) containing  parts with defects**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "af9e36de-100e-a286-9b8c-a60e4fc01d67"
      },
      "outputs": [],
      "source": [
        "responsesTemp = []\n",
        "for index, row in df_num.iterrows():\n",
        "    if bool(row['Response']):\n",
        "        responsesTemp.append(int(row['Id']))\n",
        "responses = set(responsesTemp)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "b3622053-04f7-e995-1d55-b3cdb857e7b4"
      },
      "source": [
        "**Store the timestamps at each stations. Split in 2, parts with defects and parts without. Note that we don't know when the part got the defect**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "225c2173-e878-ee3b-2753-2dd8891cd5d1"
      },
      "outputs": [],
      "source": [
        "stations = defaultdict(list)\n",
        "stationsdefaults = defaultdict(list)\n",
        "for index, row in df_date.iterrows():\n",
        "    current_station = ''\n",
        "    for column in df_date:\n",
        "        if(column != 'Id'):\n",
        "            if (current_station != column.split('_')[1]):\n",
        "                current_station = column.split('_')[1]\n",
        "                stations[current_station]\n",
        "                if not np.isnan(row[column]):\n",
        "                    if (row['Id'] in responses):\n",
        "                        stationsdefaults[current_station].append(row[column])\n",
        "                    else:\n",
        "                        stations[current_station].append(row[column])  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d22353e5-d18d-b96c-b972-3096682121b0"
      },
      "source": [
        "**Plot the timestamps for each station. We use jittering(the random coefficient) to see things better**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2dd7bf77-a58f-8e2c-89e8-a85de6b65461"
      },
      "outputs": [],
      "source": [
        "for j in range(51):\n",
        "    values = [(random.random()*0.5 + 0.5) for i in range(len(stationsdefaults['S' + str(j)]))]\n",
        "    plt.plot(stationsdefaults['S' + str(j)], values, marker='*', ls='', markersize='1')\n",
        "\n",
        "    values = [random.random()*0.5 for i in range(len(stations['S' + str(j)]))]\n",
        "    plt.plot(stations['S' + str(j)], values, marker='*', ls='', markersize='1')\n",
        "    plt.title('Station S' + str(j))\n",
        "    \n",
        "    plt.show()\n",
        "   "
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}