{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "9e0b1c94-764c-3a12-fb30-f1f53ba4baf1"
      },
      "source": [
        "# Station combinations\n",
        "\n",
        "We have seen [station 32 has high (4.7%) error rate][1].\n",
        "\n",
        "Let's investigate that failure rate with station combinations.\n",
        "\n",
        "  [1]: https://www.kaggle.com/jeffd23/bosch-production-line-performance/what-s-wrong-with-station-32"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e8180192-9ca9-4c99-daa7-6d745e9ae117"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import gc\n",
        "sns.set_style('whitegrid')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "587a8ccd-9fe7-14f9-f7e6-3422aad61708"
      },
      "source": [
        "# Read station and response data\n",
        "\n",
        "['S32', 'S33', 'S34'] have the most interesting pattern. \n",
        "\n",
        "We read the full train set although only 5 columns."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "00ad541e-ac50-59a4-a013-f34b7ec713ed"
      },
      "outputs": [],
      "source": [
        "STATIONS = ['S32', 'S33', 'S34']\n",
        "train_date_part = pd.read_csv('../input/train_date.csv', nrows=10000)\n",
        "date_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\n",
        "date_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\n",
        "date_cols = date_cols[date_cols['station'].isin(STATIONS)]\n",
        "date_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\n",
        "print(date_cols)\n",
        "train_date = pd.read_csv('../input/train_date.csv', usecols=['Id'] + date_cols)\n",
        "print(train_date.columns)\n",
        "train_date.columns = ['Id'] + STATIONS\n",
        "for station in STATIONS:\n",
        "    train_date[station] = 1 * (train_date[station] >= 0)\n",
        "response = pd.read_csv('../input/train_numeric.csv', usecols=['Id', 'Response'])\n",
        "print(response.shape)\n",
        "train = response.merge(train_date, how='left', on='Id')\n",
        "# print(train.count())\n",
        "train.head(3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "d338b8d9-9f70-d115-b1f6-ea75248c8a53"
      },
      "source": [
        "#Aggregation\n",
        "\n",
        "Remove cases with less than 1000 records to show significant results."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9badda51-b401-d944-5948-374821e84fe7"
      },
      "outputs": [],
      "source": [
        "train['cnt'] = 1\n",
        "failure_rate = train.groupby(STATIONS).sum()[['Response', 'cnt']]\n",
        "failure_rate['failure_rate'] = failure_rate['Response'] / failure_rate['cnt']\n",
        "failure_rate = failure_rate[failure_rate['cnt'] > 1000]  # remove \n",
        "failure_rate.head(20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "44866b71-8de5-7913-976f-15e21ef3716b"
      },
      "outputs": [],
      "source": [
        "failure_rate_pretty = failure_rate.reset_index()\n",
        "failure_rate_pretty['group'] = ['-'.join([s if row[s] else '' for s in STATIONS]) \\\n",
        "                         for _, row in failure_rate_pretty.iterrows()]\n",
        "fig=plt.figure(figsize=(10, 4))\n",
        "sns.barplot(x='group', y=\"failure_rate\", data=failure_rate_pretty, color='r', alpha=0.8)\n",
        "plt.ylabel('failure rate')\n",
        "for i, row in failure_rate_pretty.iterrows():\n",
        "    plt.text(i, row['failure_rate']+0.01, np.round(row['failure_rate'], 3),\n",
        "             verticalalignment='top', horizontalalignment='center')\n",
        "plt.title('Station combinations %s' % str(STATIONS))\n",
        "fig.savefig('failure_rate.png', dpi=300)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "793d4b41-9f84-9e7e-9b53-7ba154bea2d5"
      },
      "source": [
        "#Magic/Leakage features\n",
        "First we need to get the start times for both train and test."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "809cd358-829f-43a7-bf84-35e118847b5a"
      },
      "outputs": [],
      "source": [
        "train_date_part = pd.read_csv('../input/train_date.csv', nrows=10000)\n",
        "date_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\n",
        "date_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\n",
        "date_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\n",
        "# Train start dates\n",
        "train_start_date = pd.read_csv('../input/train_date.csv', usecols=['Id'] + date_cols)\n",
        "train_start_date['start_date'] = train_start_date[date_cols].min(axis=1)\n",
        "train_start_date = train_start_date.drop(date_cols, axis=1)\n",
        "print(train_start_date.shape)\n",
        "# Test start dates\n",
        "test_start_date = pd.read_csv('../input/test_date.csv', usecols=['Id'] + date_cols)\n",
        "test_start_date['start_date'] = test_start_date[date_cols].min(axis=1)\n",
        "test_start_date = test_start_date.drop(date_cols, axis=1)\n",
        "print(test_start_date.shape)\n",
        "start_date = pd.concat([train_start_date, test_start_date])\n",
        "print(start_date.shape)\n",
        "del train_start_date, test_start_date\n",
        "gc.collect()\n",
        "start_date.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "609306ab-03d3-4d2c-11af-dc51d0169223"
      },
      "source": [
        "Then we add Faron's features."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b33f0bf7-3a6f-ff32-7b92-e7f45b10bc1c"
      },
      "outputs": [],
      "source": [
        "train_id = pd.read_csv('../input/train_numeric.csv', usecols=['Id'])\n",
        "test_id = pd.read_csv('../input/test_numeric.csv', usecols=['Id'])\n",
        "train_id = train_id.merge(start_date, on='Id')\n",
        "test_id = test_id.merge(start_date, on='Id')\n",
        "train_test_id = pd.concat((train_id, test_id)).reset_index(drop=True).reset_index(drop=False)\n",
        "train_test_id = train_test_id.sort_values(by=['start_date', 'Id'], ascending=True)\n",
        "train_test_id['IdDiff1'] = train_test_id['Id'].diff().fillna(9999999).astype(int)\n",
        "train_test_id['IdDiff2'] = train_test_id['Id'].iloc[::-1].diff().fillna(9999999).astype(int)\n",
        "train_test_id['Magic'] = 1 + 2 * (train_test_id['IdDiff1'] > 1) + 1 * (train_test_id['IdDiff2'] < -1)\n",
        "\n",
        "train_with_magic = train.merge(train_test_id[['Id', 'Magic']], on='Id')\n",
        "train_with_magic.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4a1ec13b-2718-657b-3de2-94de7b3afae9"
      },
      "outputs": [],
      "source": [
        "magic_failure_rate = train_with_magic.groupby(['Magic']).sum()[['Response', 'cnt']]\n",
        "magic_failure_rate['failure_rate'] = magic_failure_rate['Response'] / magic_failure_rate['cnt']\n",
        "magic_failure_rate.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "0ee4537a-35f4-935a-bbae-80cb916a71f8"
      },
      "source": [
        "5% failure rate for 100K rows. Not bad for a single feature."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7ddd27ae-ad95-980a-60c2-ebc9045ff6f0"
      },
      "outputs": [],
      "source": [
        "magic_failure_rate_pretty = magic_failure_rate.reset_index()\n",
        "fig=plt.figure(figsize=(10, 4))\n",
        "sns.barplot(x='Magic', y=\"failure_rate\", data=magic_failure_rate_pretty, color='k', alpha=0.8)\n",
        "plt.ylabel('failure rate')\n",
        "for i, row in magic_failure_rate_pretty.iterrows():\n",
        "    plt.text(i, row['failure_rate']+0.01, np.round(row['failure_rate'], 3),\n",
        "             verticalalignment='top', horizontalalignment='center')\n",
        "fig.savefig('magic_failure_rate.png', dpi=300)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "6fdd6d38-fac1-e25d-a9c7-64c07e683b09"
      },
      "source": [
        "# Combining the results\n",
        "Let's check ifwe could get something even stronger by combining the previous results."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "527e241e-8a91-5b9e-e428-25b90e339412"
      },
      "outputs": [],
      "source": [
        "combined_failure_rate = train_with_magic.groupby(['Magic','S32', 'S33']).sum()[['Response', 'cnt']]\n",
        "combined_failure_rate['failure_rate'] = combined_failure_rate['Response'] / combined_failure_rate['cnt']\n",
        "combined_failure_rate.head(20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "23120d63-3eff-5eb3-5870-c9423bd79275"
      },
      "outputs": [],
      "source": [
        "full = combined_failure_rate.reset_index()\n",
        "full['group'] = 100 * full['Magic'] + 10 * full['S32'] + full['S33']\n",
        "fig=plt.figure(figsize=(10, 4))\n",
        "sns.barplot(x='group', y=\"failure_rate\", data=full, color='g', alpha=0.8)\n",
        "plt.ylabel('failure rate')\n",
        "for i, row in full.iterrows():\n",
        "    plt.text(i, row['failure_rate']+0.05, np.round(row['failure_rate'], 3),\n",
        "             verticalalignment='top', horizontalalignment='center')\n",
        "plt.title('Magic & S32 - S33')\n",
        "fig.savefig('magic_station_failure_rate.png', dpi=300)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "b9121a19-5932-a657-2cbf-cf6d4fb961fd"
      },
      "source": [
        "We are able to find **1000 products with almost 70% failure rate using only 3 features**."
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}