{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "8cf8a39f-a732-7377-7415-c0e247855ea6" }, "source": [ "# Combining some findings from current kernels of other teams" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "a9517a21-beb6-a79f-d38f-06cf743c1cec" }, "source": [ "Load required packages" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a1c1aefe-9b7b-0a38-994c-577ef924da94" }, "outputs": [], "source": [ "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import xgboost as xgb # XGBoost is short for \u201cExtreme Gradient Boosting\u201d\n", "# See here for details about xgboost: http://xgboost.readthedocs.io/en/latest/model.html\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "import seaborn as sns\n", "import time\n", "import gc\n", "sns.set_style('whitegrid')" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "0a47e513-9622-64b8-a1bf-d81cf13290ff" }, "source": [ "# 1. Look at the so-called \"Station\" extracted from \"training_date\" variable names\n", "Source : https://www.kaggle.com/gaborfodor/bosch-production-line-performance/69-failure-rate" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "10794deb-33c6-6f6b-1b5c-51e27115870c" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "89e10c33-e470-ca52-3ac4-c3138a54a409" }, "outputs": [], "source": [ "STATIONS = ['S32', 'S33', 'S34']\n", "train_date_part = pd.read_csv('../input/train_date.csv', nrows=10000)\n", "# count missing value in each date column\n", "date_cols = train_date_part.drop('Id', axis=1).count().reset_index().sort_values(by=0, ascending=False)\n", "# create a new variable station which reads SXX in the date column name, like \"L3_S37_D3949\"\n", "date_cols['station'] = date_cols['index'].apply(lambda s: s.split('_')[1])\n", "\n", "# filter only S32, S33, and S34\n", "date_cols = date_cols[date_cols['station'].isin(STATIONS)]\n", "# save the date names for S32, 33,34 to a list\n", "date_cols = date_cols.drop_duplicates('station', keep='first')['index'].tolist()\n", "print(date_cols)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "37e95b5b-a21d-f67b-5484-ced09e9feec9" }, "outputs": [], "source": [ "# read train_date keep only Id and the 3 date_columns\n", "train_date1 = pd.read_csv('../input/train_date.csv', usecols=['Id'] + date_cols)\n", "print(train_date1.columns)\n", "train_date1.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8c8bfbb5-41c8-1689-19d1-1e2306083b09" }, "outputs": [], "source": [ "# rename 3 columns\n", "train_date1.columns = ['Id'] + STATIONS\n", "\n", "# convert values> 0 to 1, else to 0 \n", "for station in STATIONS:\n", " train_date1[station] = 1 * (train_date1[station] >= 0)\n", "train_date1.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ced16cf3-2963-9a6e-5672-db2f3d8d8823" }, "outputs": [], "source": [ "# get response from train_numeric\n", "response = pd.read_csv('../input/train_numeric.csv', usecols=['Id', 'Response'])\n", "print(response.shape)\n", "\n", "# merge predictors to response\n", "train1 = response.merge(train_date1, how='left', on='Id')\n", "# print(train.count())\n", "print(train1.head(3))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "93ec065b-1f59-394d-8850-bea3a0fd82d2" }, "source": [ "###Aggregation:\n", "Remove cases with less than 1000 records to show significant results." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d4b3bfd3-343e-6e48-71b4-9d24b8503bc5" }, "outputs": [], "source": [ "train1['cnt'] = 1\n", "failure_rate = train1.groupby(STATIONS).sum()[['Response', 'cnt']]\n", "failure_rate['failure_rate'] = failure_rate['Response'] / failure_rate['cnt']\n", "print(failure_rate.head(20))\n", "failure_rate = failure_rate[failure_rate['cnt'] > 1000] # remove \n", "print(failure_rate.head(20))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "d1acf0cb-e039-1bcc-8d0a-e6e945211251" }, "source": [ "###then add the magic feature below as predictor and fit a model" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "73aac4c4-a4e6-5501-f1d3-2be513bb650c" }, "source": [ "# 2. Generate ID features as predictors\n", " - Source: https://www.kaggle.com/mmueller/bosch-production-line-performance/road-2-0-4/code\n", " - This script generates ID-based features, which can be used to achieve\n", " a MCC > 0.4. They provide a very high AUC on their own" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "190dc93a-9041-95e2-8ff5-fe7e158e7c7a" }, "outputs": [], "source": [ "# set data directory\n", "DATA_DIR = \"../input\"\n", "\n", "ID_COLUMN = 'Id'\n", "TARGET_COLUMN = 'Response'\n", "\n", "# some parameters in reading in the files\n", "SEED = 0\n", "CHUNKSIZE = 50000\n", "NROWS = 250000\n", "\n", "#Read all test and training data sets:\n", "TRAIN_NUMERIC = \"{0}/train_numeric.csv\".format(DATA_DIR)\n", "TRAIN_DATE = \"{0}/train_date.csv\".format(DATA_DIR)\n", "TRAIN_CAT = \"{0}/train_categorical\".format(DATA_DIR)\n", "\n", "TEST_NUMERIC = \"{0}/test_numeric.csv\".format(DATA_DIR)\n", "TEST_DATE = \"{0}/test_date.csv\".format(DATA_DIR)\n", "TRAIN_CAT = \"{0}/test_categorical\".format(DATA_DIR)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "4c39d464-baa0-a268-30d9-727f6dbf185c" }, "source": [ "**Save ID, response in training data as train; Save ID in test as test**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "549c0cec-97fb-aed9-e134-bb05947c7b9a" }, "outputs": [], "source": [ "FILENAME = \"etimelhoods\"\n", "\n", "train = pd.read_csv(TRAIN_NUMERIC, usecols=[ID_COLUMN, TARGET_COLUMN], nrows=NROWS)\n", "test = pd.read_csv(TEST_NUMERIC, usecols=[ID_COLUMN], nrows=NROWS)\n", "\n", "train[\"StartTime\"] = -1\n", "test[\"StartTime\"] = -1\n", "# **Read training and test date**\n", "# print (\"Size of training data: %int\" % train.shape[0])\n", "# print (train.head())\n", "# print (\"Size of test data: %int\" % test.shape[0])\n", "# print (test.head())" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "84267b2b-6c62-b0d2-f32b-080eae3e033a" }, "source": [ "**Look at date data: 1157 columns....**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9c2b594e-db8b-a54b-2b2e-c9db0df78bd8" }, "outputs": [], "source": [ "train_date = pd.read_csv(TRAIN_DATE, nrows=10)\n", "# test_date = pd.read_csv(TEST_DATE, nrows=10)\n", "train_date.head()\n", "# test_date.head()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "b71306f3-8dcc-2871-1162-977a073a2a16" }, "source": [ "**Look at numeric data, 970 column, the last column is response 0/1:**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "065cea50-0257-4340-8fde-41838a07c04d" }, "outputs": [], "source": [ "train_num = pd.read_csv(TRAIN_NUMERIC, nrows=10)\n", "# test_num = pd.read_csv(TEST_NUMERIC, nrows=10)\n", "train_num.head()" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "af1b756f-8f8d-bbc9-60a4-d893d9fc0beb" }, "source": [ "**Look at categorical data, 970 column, the last column is response 0/1:**" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f9c7f900-a591-30e2-3c0f-afe2af3500b3" }, "outputs": [], "source": [ "train_cat = pd.read_csv(TRAIN_CAT, nrows=10)\n", "# test_cat = pd.read_csv(TEST_CAT, nrows=10)\n", "train_cat.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b25ccaf2-da43-09f1-a984-96f8849468a4" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f3ad7425-2f69-a7f2-aee9-312a0c0aa903" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "7dceedfc-f78d-3097-36ad-f988e656747f" }, "outputs": [], "source": [ "nrows = 0\n", "# Zip: Make an iterator that aggregates elements from each of the iterables;\n", "# Returns an iterator of tuples: zip('ABCD','xy') --> Ax By\n", "\n", "for tr, te in zip(pd.read_csv(TRAIN_DATE, chunksize=CHUNKSIZE), pd.read_csv(TEST_DATE, chunksize=CHUNKSIZE)):\n", " \n", " # numpy.setdiff1d(ar1, ar2, assume_unique=False). Find the set difference of two arrays. \n", " # Return the sorted, unique values in ar1 that are not in ar2. \n", " feats = np.setdiff1d(tr.columns, [ID_COLUMN])\n", " # feats are the column names in _date dataset, excluding ID\n", "\n", " # get the min date for each ID\n", " stime_tr = tr[feats].min(axis=1).values\n", " stime_te = te[feats].min(axis=1).values\n", " \n", " # save min date for each ID, if the ID exist in data 'train'/'test'\n", " train.loc[train.Id.isin(tr.Id), 'StartTime'] = stime_tr\n", " test.loc[test.Id.isin(te.Id), 'StartTime'] = stime_te\n", "\n", " nrows += CHUNKSIZE\n", " if nrows >= NROWS:\n", " break" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e0ef0fa7-d402-a5b4-4654-8c1e181e1b89" }, "outputs": [], "source": [ "ntrain = train.shape[0] # num of rows in training set\n", "train_test = pd.concat((train, test)).reset_index(drop=True).reset_index(drop=False)\n", "\n", "# **Create 4 predictors based solely on ID**\n", "# new col= kth Id - (k-1)th ID\n", "train_test['magic1'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\n", "# new col= kth Id - (k+1)th ID\n", "train_test['magic2'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n", "\n", "# Sort by StartTime and then by ID, create another 2 columns based on ID\n", "train_test = train_test.sort_values(by=['StartTime', 'Id'], ascending=True)\n", "train_test['magic3'] = train_test[ID_COLUMN].diff().fillna(9999999).astype(int)\n", "train_test['magic4'] = train_test[ID_COLUMN].iloc[::-1].diff().fillna(9999999).astype(int)\n", "\n", "print(train_test.head())\n", "print(train_test.tail())\n", "\n", "# sort data back to original order\n", "train_test = train_test.sort_values(by=['index']).drop(['index'], axis=1)\n", "# save new train data\n", "train = train_test.iloc[:ntrain, :]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "e8266568-bed2-f276-cad9-1537ae1a50b3" }, "outputs": [], "source": [ "'''# visualizing the magic features above\n", "This is an attempt at visualizing the magic feature(outed by Faron) in how well it separates responses. Can be used to visualize any random feature's discriminating power.\n", "\n", "**source:** https://www.kaggle.com/rithal/bosch-production-line-performance/magic-feature-visualization'''\n", "\n", "def twoplot(df, col, xaxis=None):\n", " ''' scatter plot a feature split into response values as two subgraphs '''\n", " if col not in df.columns.values:\n", " print('ERROR: %s not a column' % col)\n", " ndf = pd.DataFrame(index = df.index)\n", " ndf[col] = df[col]\n", " ndf[xaxis] = df[xaxis] if xaxis else df.index\n", " ndf['Response'] = df['Response']\n", " \n", " g = sns.FacetGrid(ndf, col=\"Response\", hue=\"Response\")\n", " g.map(plt.scatter, xaxis, col, alpha=.7, s=1)\n", " g.add_legend();\n", " \n", " del ndf" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a559af70-4e68-b81a-88db-7defec02e130" }, "outputs": [], "source": [ "twoplot(train, 'magic1')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9459ce0c-4d26-c486-234d-f3d4e5df6f50" }, "outputs": [], "source": [ "twoplot(train, 'magic2')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "d24811b9-dbdf-0b78-e50d-c9b2ab5f638b" }, "outputs": [], "source": [ "twoplot(train, 'magic3')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "1fd6e944-3649-e549-7f8b-b02c04621f2a" }, "outputs": [], "source": [ "twoplot(train, 'magic4')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "eabfbf28-702a-6938-ca7b-df0622d50918" }, "outputs": [], "source": [ "'''The following codes are commented out for now'''\n", "\n", "'''# features is all column in train, except ID and response:\n", "features = np.setdiff1d(list(train.columns), [TARGET_COLUMN, ID_COLUMN])\n", "# y: response of new training set\n", "y = train.Response.ravel() # numpy.ravel(): Return a flattened array\n", "# train: 4 predictors based on columns + StartTime\n", "train = np.array(train[features]) \n", "\n", "# print # rows and # cols of training predictors: 250K * 5:\n", "print('train: {0}'.format(train.shape))\n", "\n", "# Use failure rate in training set as prior to input into the model:\n", "prior = np.sum(y) / (1.*len(y))\n", "\n", "# set parameters for xgboost\n", "xgb_params = {\n", " 'seed': 0,\n", " 'colsample_bytree': 0.7,\n", " 'silent': 1,\n", " 'subsample': 0.7,\n", " 'learning_rate': 0.1,\n", " 'objective': 'binary:logistic',\n", " 'max_depth': 4,\n", " 'num_parallel_tree': 1,\n", " 'min_child_weight': 2,\n", " 'eval_metric': 'auc',\n", " 'base_score': prior\n", "}\n", "\n", "\n", "dtrain = xgb.DMatrix(train, label=y)\n", "res = xgb.cv(xgb_params, dtrain, num_boost_round=10, nfold=4, seed=0, stratified=True,\n", " early_stopping_rounds=1, verbose_eval=1, show_stdv=True)\n", "\n", "cv_mean = res.iloc[-1, 0]\n", "cv_std = res.iloc[-1, 1]\n", "\n", "print('CV-Mean: {0}+{1}'.format(cv_mean, cv_std))\n", "'''" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "4c47e4fc-e4b9-feb9-cee5-c2f221540461" }, "outputs": [], "source": "" }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3a442101-2eb8-6f6b-02f2-56f16b3d228e" }, "outputs": [], "source": "" } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 0 }