{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "9e0b1c94-764c-3a12-fb30-f1f53ba4baf1"
      },
      "source": [
        "# Feature combinations\n",
        "\n",
        "We have seen [station 32 has high (4.7%) error rate][1].\n",
        "\n",
        "Let's investigate that failure rate with station combinations.\n",
        "\n",
        "  [1]: https://www.kaggle.com/jeffd23/bosch-production-line-performance/what-s-wrong-with-station-32"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e8180192-9ca9-4c99-daa7-6d745e9ae117"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "sns.set_style('whitegrid')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "587a8ccd-9fe7-14f9-f7e6-3422aad61708"
      },
      "source": [
        "# Read station and response data\n",
        "\n",
        "['S32', 'S33', 'S34'] have the most interesting pattern. \n",
        "\n",
        "We read the full train set although only 5 columns."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "00ad541e-ac50-59a4-a013-f34b7ec713ed"
      },
      "outputs": [],
      "source": [
        "STATIONS = ['S32', 'S33', 'S34']\n",
        "train_date_part = pd.read_csv('../input/train_date.csv', nrows=10000)\n",
        "date_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\n",
        "date_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\n",
        "date_cols = date_cols[date_cols['station'].isin(STATIONS)]\n",
        "date_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\n",
        "print(date_cols)\n",
        "train_date = pd.read_csv('../input/train_date.csv', usecols=['Id'] + date_cols)\n",
        "print(train_date.columns)\n",
        "train_date.columns = ['Id'] + STATIONS\n",
        "for station in STATIONS:\n",
        "    train_date[station] = 1 * (train_date[station] >= 0)\n",
        "response = pd.read_csv('../input/train_numeric.csv', usecols=['Id', 'Response'])\n",
        "print(response.shape)\n",
        "train = response.merge(train_date, how='left', on='Id')\n",
        "# print(train.count())\n",
        "train.head(3)\n",
        "train.to_csv(\"tr_merge.csv\",index=False)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "train_date = pd.read_csv('../input/test_date.csv', usecols=['Id'] + date_cols)\n",
        "print(train_date.columns)\n",
        "train_date.columns = ['Id'] + STATIONS\n",
        "for station in STATIONS:\n",
        "    train_date[station] = 1 * (train_date[station] >= 0)\n",
        "response = pd.read_csv('../input/test_numeric.csv', usecols=['Id'])\n",
        "print(response.shape)\n",
        "train = response.merge(train_date, how='left', on='Id')\n",
        "# print(train.count())\n",
        "train.head(3)\n",
        "train.to_csv(\"ts_merge.csv\",index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d338b8d9-9f70-d115-b1f6-ea75248c8a53"
      },
      "source": [
        "#Aggregation\n",
        "\n",
        "Remove cases with less than 1000 records to show significant results."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9badda51-b401-d944-5948-374821e84fe7"
      },
      "outputs": [],
      "source": [
        "train['cnt'] = 1\n",
        "failure_rate = train.groupby(STATIONS).sum()[['Response', 'cnt']]\n",
        "failure_rate['failure_rate'] = failure_rate['Response'] / failure_rate['cnt']\n",
        "failure_rate = failure_rate[failure_rate['cnt'] > 1000]  # remove \n",
        "failure_rate.head(20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "44866b71-8de5-7913-976f-15e21ef3716b"
      },
      "outputs": [],
      "source": [
        "failure_rate_pretty = failure_rate.reset_index()\n",
        "failure_rate_pretty['group'] = ['-'.join([s if row[s] else '' for s in STATIONS]) \\\n",
        "                         for _, row in failure_rate_pretty.iterrows()]\n",
        "fig=plt.figure(figsize=(10, 4))\n",
        "sns.barplot(x='group', y=\"failure_rate\", data=failure_rate_pretty, color='r', alpha=0.8)\n",
        "plt.ylabel('failure rate')\n",
        "for i, row in failure_rate_pretty.iterrows():\n",
        "    plt.text(i, row['failure_rate']+0.01, np.round(row['failure_rate'], 3),\n",
        "             verticalalignment='top', horizontalalignment='center')\n",
        "plt.title('Station combinations %s' % str(STATIONS))\n",
        "fig.savefig('failure_rate.png', dpi=300)\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}