{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"9e0b1c94-764c-3a12-fb30-f1f53ba4baf1"},"source":"# Station combinations\n\nWe have seen [station 32 has high (4.7%) error rate][1].\n\nLet's investigate that failure rate with station combinations.\n\n  [1]: https://www.kaggle.com/jeffd23/bosch-production-line-performance/what-s-wrong-with-station-32"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"e8180192-9ca9-4c99-daa7-6d745e9ae117"},"outputs":[],"source":"%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\nsns.set_style('whitegrid')"},{"cell_type":"markdown","metadata":{"_cell_guid":"587a8ccd-9fe7-14f9-f7e6-3422aad61708"},"source":"# Read station and response data\n\n['S32', 'S33', 'S34'] have the most interesting pattern. \n\nWe read the full train set although only 5 columns."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"00ad541e-ac50-59a4-a013-f34b7ec713ed"},"outputs":[],"source":"STATIONS = ['S32', 'S33', 'S34']\ntrain_date_part = pd.read_csv('../input/train_date.csv', nrows=10000)\ndate_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\ndate_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\ndate_cols = date_cols[date_cols['station'].isin(STATIONS)]\ndate_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\nprint(date_cols)\ntrain_date = pd.read_csv('../input/train_date.csv', usecols=['Id'] + date_cols)\nprint(train_date.columns)\ntrain_date.columns = ['Id'] + STATIONS\nfor station in STATIONS:\n    train_date[station] = 1 * (train_date[station] >= 0)\nresponse = pd.read_csv('../input/train_numeric.csv', usecols=['Id', 'Response'])\nprint(response.shape)\ntrain = response.merge(train_date, how='left', on='Id')\n# print(train.count())\ntrain.head(3)"},{"cell_type":"markdown","metadata":{"_cell_guid":"d338b8d9-9f70-d115-b1f6-ea75248c8a53"},"source":"#Aggregation\n\nRemove cases with less than 1000 records to show significant results."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"9badda51-b401-d944-5948-374821e84fe7"},"outputs":[],"source":"train['cnt'] = 1\nfailure_rate = train.groupby(STATIONS).sum()[['Response', 'cnt']]\nfailure_rate['failure_rate'] = failure_rate['Response'] / failure_rate['cnt']\nfailure_rate = failure_rate[failure_rate['cnt'] > 1000]  # remove \nfailure_rate.head(20)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"44866b71-8de5-7913-976f-15e21ef3716b"},"outputs":[],"source":"failure_rate_pretty = failure_rate.reset_index()\nfailure_rate_pretty['group'] = ['-'.join([s if row[s] else '' for s in STATIONS]) \\\n                         for _, row in failure_rate_pretty.iterrows()]\nfig=plt.figure(figsize=(10, 4))\nsns.barplot(x='group', y=\"failure_rate\", data=failure_rate_pretty, color='r', alpha=0.8)\nplt.ylabel('failure rate')\nfor i, row in failure_rate_pretty.iterrows():\n    plt.text(i, row['failure_rate']+0.01, np.round(row['failure_rate'], 3),\n             verticalalignment='top', horizontalalignment='center')\nplt.title('Station combinations %s' % str(STATIONS))\nfig.savefig('failure_rate.png', dpi=300)\nplt.show()"},{"cell_type":"markdown","metadata":{"_cell_guid":"793d4b41-9f84-9e7e-9b53-7ba154bea2d5"},"source":"#Magic/Leakage features\nFirst we need to get the start times for both train and test."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"809cd358-829f-43a7-bf84-35e118847b5a"},"outputs":[],"source":"train_date_part = pd.read_csv('../input/train_date.csv', nrows=10000)\ndate_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\ndate_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\ndate_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\n# Train start dates\ntrain_start_date = pd.read_csv('../input/train_date.csv', usecols=['Id'] + date_cols)\ntrain_start_date['start_date'] = train_start_date[date_cols].min(axis=1)\ntrain_start_date = train_start_date.drop(date_cols, axis=1)\nprint(train_start_date.shape)\n# Test start dates\ntest_start_date = pd.read_csv('../input/test_date.csv', usecols=['Id'] + date_cols)\ntest_start_date['start_date'] = test_start_date[date_cols].min(axis=1)\ntest_start_date = test_start_date.drop(date_cols, axis=1)\nprint(test_start_date.shape)\nstart_date = pd.concat([train_start_date, test_start_date])\nprint(start_date.shape)\ndel train_start_date, test_start_date\ngc.collect()\nstart_date.head()"},{"cell_type":"markdown","metadata":{"_cell_guid":"609306ab-03d3-4d2c-11af-dc51d0169223"},"source":"Then we add Faron's features."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b33f0bf7-3a6f-ff32-7b92-e7f45b10bc1c"},"outputs":[],"source":"train_id = pd.read_csv('../input/train_numeric.csv', usecols=['Id'])\ntest_id = pd.read_csv('../input/test_numeric.csv', usecols=['Id'])\ntrain_id = train_id.merge(start_date, on='Id')\ntest_id = test_id.merge(start_date, on='Id')\ntrain_test_id = pd.concat((train_id, test_id)).reset_index(drop=True).reset_index(drop=False)\ntrain_test_id = train_test_id.sort_values(by=['start_date', 'Id'], ascending=True)\ntrain_test_id['IdDiff1'] = train_test_id['Id'].diff().fillna(9999999).astype(int)\ntrain_test_id['IdDiff2'] = train_test_id['Id'].iloc[::-1].diff().fillna(9999999).astype(int)\ntrain_test_id['Magic'] = 1 + 2 * (train_test_id['IdDiff1'] > 1) + 1 * (train_test_id['IdDiff2'] < -1)\n\ntrain_with_magic = train.merge(train_test_id[['Id', 'Magic']], on='Id')\ntrain_with_magic.head()"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"4a1ec13b-2718-657b-3de2-94de7b3afae9"},"outputs":[],"source":"magic_failure_rate = train_with_magic.groupby(['Magic']).sum()[['Response', 'cnt']]\nmagic_failure_rate['failure_rate'] = magic_failure_rate['Response'] / magic_failure_rate['cnt']\nmagic_failure_rate.head()"},{"cell_type":"markdown","metadata":{"_cell_guid":"0ee4537a-35f4-935a-bbae-80cb916a71f8"},"source":"5% failure rate for 100K rows. Not bad for a single feature."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"7ddd27ae-ad95-980a-60c2-ebc9045ff6f0"},"outputs":[],"source":"magic_failure_rate_pretty = magic_failure_rate.reset_index()\nfig=plt.figure(figsize=(10, 4))\nsns.barplot(x='Magic', y=\"failure_rate\", data=magic_failure_rate_pretty, color='k', alpha=0.8)\nplt.ylabel('failure rate')\nfor i, row in magic_failure_rate_pretty.iterrows():\n    plt.text(i, row['failure_rate']+0.01, np.round(row['failure_rate'], 3),\n             verticalalignment='top', horizontalalignment='center')\nfig.savefig('magic_failure_rate.png', dpi=300)\nplt.show()"},{"cell_type":"markdown","metadata":{"_cell_guid":"6fdd6d38-fac1-e25d-a9c7-64c07e683b09"},"source":"# Combining the results\nLet's check ifwe could get something even stronger by combining the previous results."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"527e241e-8a91-5b9e-e428-25b90e339412"},"outputs":[],"source":"combined_failure_rate = train_with_magic.groupby(['Magic','S32', 'S33']).sum()[['Response', 'cnt']]\ncombined_failure_rate['failure_rate'] = combined_failure_rate['Response'] / combined_failure_rate['cnt']\ncombined_failure_rate.head(20)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"23120d63-3eff-5eb3-5870-c9423bd79275"},"outputs":[],"source":"full = combined_failure_rate.reset_index()\nfull['group'] = 100 * full['Magic'] + 10 * full['S32'] + full['S33']\nfig=plt.figure(figsize=(10, 4))\nsns.barplot(x='group', y=\"failure_rate\", data=full, color='g', alpha=0.8)\nplt.ylabel('failure rate')\nfor i, row in full.iterrows():\n    plt.text(i, row['failure_rate']+0.05, np.round(row['failure_rate'], 3),\n             verticalalignment='top', horizontalalignment='center')\nplt.title('Magic & S32 - S33')\nfig.savefig('magic_station_failure_rate.png', dpi=300)\nplt.show()"},{"cell_type":"markdown","metadata":{"_cell_guid":"b9121a19-5932-a657-2cbf-cf6d4fb961fd"},"source":"We are able to find **1000 products with almost 70% failure rate using only 3 features**."}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}