{"cells":[{"metadata":{},"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0\"></a>\n\n# Building the feature importance diagrams in 6 ways on the example of competition \"Titanic: Machine Learning from Disaster\":\n* XGB\n* LGBM\n* LogisticRegression (with Eli5)\n* LinearRegression (with Eli5)\n* Mean\n* Total with different weights"},{"metadata":{},"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0.1\"></a>\n\n## Table of Contents\n\n1. [Import libraries](#1)\n1. [Download datasets](#2)\n1. [FE & EDA](#3)\n1. [Preparing to modeling](#4)\n1. [Tuning models and building the feature importance diagrams](#5)\n    -  [LGBM](#5.1)\n    -  [XGB](#5.2)\n    -  [Logistic Regression](#5.3)\n    -  [Linear Regression](#5.4)\n1. [Comparison of the all feature importance diagrams](#6)"},{"metadata":{},"cell_type":"markdown","source":"## 1. Import libraries <a class=\"anchor\" id=\"1\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\n\nimport eli5\n\nimport lightgbm as lgbm\nimport xgboost as xgb\n\nfrom sklearn.linear_model import LogisticRegression, LinearRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Download datasets <a class=\"anchor\" id=\"2\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. FE & EDA <a class=\"anchor\" id=\"3\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{},"cell_type":"markdown","source":"While we take all the features"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('max_columns',100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"#train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. Preparing to modeling <a class=\"anchor\" id=\"4\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"trusted":false},"cell_type":"code","source":"# Thanks to : https://www.kaggle.com/aantonova/some-new-risk-and-clusters-features\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']\ncategorical_columns = []\nfeatures = train.columns.values.tolist()\nfor col in features:\n    if train[col].dtype in numerics: continue\n    categorical_columns.append(col)\nindexer = {}\nfor col in categorical_columns:\n    if train[col].dtype in numerics: continue\n    _, indexer[col] = pd.factorize(train[col])\n    \nfor col in categorical_columns:\n    if train[col].dtype in numerics: continue\n    train[col] = indexer[col].get_indexer(train[col])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = train.pop('target')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train = reduce_mem_usage(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. Tuning models and building the feature importance diagrams<a class=\"anchor\" id=\"5\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{},"cell_type":"markdown","source":"### 5.1 LGBM <a class=\"anchor\" id=\"5.1\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"trusted":false},"cell_type":"code","source":"X = train\nz = target","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#%% split training set to validation set\nXtrain, Xval, Ztrain, Zval = train_test_split(X, z, test_size=0.2, random_state=0)\ntrain_set = lgbm.Dataset(Xtrain, Ztrain, silent=False)\nvalid_set = lgbm.Dataset(Xval, Zval, silent=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"params = {\n        'boosting_type':'gbdt',\n        'objective': 'regression',\n        'num_leaves': 31,\n        'learning_rate': 0.05,\n        'max_depth': -1,\n        'subsample': 0.8,\n        'bagging_fraction' : 1,\n        'max_bin' : 5000 ,\n        'bagging_freq': 20,\n        'colsample_bytree': 0.6,\n        'metric': 'rmse',\n        'min_split_gain': 0.5,\n        'min_child_weight': 1,\n        'min_child_samples': 10,\n        'scale_pos_weight':1,\n        'zero_as_missing': True,\n        'seed':0,        \n    }\n\nmodelL = lgbm.train(params, train_set = train_set, num_boost_round=1000,\n                   early_stopping_rounds=50,verbose_eval=10, valid_sets=valid_set)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"fig =  plt.figure(figsize = (15,15))\naxes = fig.add_subplot(111)\nlgbm.plot_importance(modelL,ax = axes,height = 0.5)\nplt.show();plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"feature_score = pd.DataFrame(train.columns, columns = ['feature']) \nfeature_score['score_lgb'] = modelL.feature_importance()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5.2 XGB<a class=\"anchor\" id=\"5.2\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"trusted":false},"cell_type":"code","source":"#%% split training set to validation set \ndata_tr  = xgb.DMatrix(Xtrain, label=Ztrain)\ndata_cv  = xgb.DMatrix(Xval   , label=Zval)\nevallist = [(data_tr, 'train'), (data_cv, 'valid')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"parms = {'max_depth':8, #maximum depth of a tree\n         'objective':'reg:logistic',\n         'eta'      :0.3,\n         'subsample':0.8,#SGD will use this percentage of data\n         'lambda '  :4, #L2 regularization term,>1 more conservative \n         'colsample_bytree ':0.9,\n         'colsample_bylevel':1,\n         'min_child_weight': 10}\nmodelx = xgb.train(parms, data_tr, num_boost_round=200, evals = evallist,\n                  early_stopping_rounds=30, maximize=False, \n                  verbose_eval=10)\n\nprint('score = %1.5f, n_boost_round =%d.'%(modelx.best_score,modelx.best_iteration))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"fig =  plt.figure(figsize = (15,15))\naxes = fig.add_subplot(111)\nxgb.plot_importance(modelx,ax = axes,height = 0.5)\nplt.show();plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"feature_score['score_xgb'] = feature_score['feature'].map(modelx.get_score(importance_type='weight'))\nfeature_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5.3 Logistic Regression <a class=\"anchor\" id=\"5.3\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"trusted":false},"cell_type":"code","source":"# Standardization for regression models\ntrain = pd.DataFrame(\n    preprocessing.MinMaxScaler().fit_transform(train),\n    columns=train.columns,\n    index=train.index\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Logistic Regression\n\nlogreg = LogisticRegression()\nlogreg.fit(train, target)\ncoeff_logreg = pd.DataFrame(train.columns.delete(0))\ncoeff_logreg.columns = ['feature']\ncoeff_logreg[\"score_logreg\"] = pd.Series(logreg.coef_[0])\ncoeff_logreg.sort_values(by='score_logreg', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"len(coeff_logreg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# the level of importance of features is not associated with the sign\ncoeff_logreg[\"score_logreg\"] = coeff_logreg[\"score_logreg\"].abs()\nfeature_score = pd.merge(feature_score, coeff_logreg, on='feature')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Eli5 visualization\neli5.show_weights(logreg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 5.4 Linear Regression <a class=\"anchor\" id=\"5.4\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"trusted":false},"cell_type":"code","source":"# Linear Regression\n\nlinreg = LinearRegression()\nlinreg.fit(train, target)\ncoeff_linreg = pd.DataFrame(train.columns.delete(0))\ncoeff_linreg.columns = ['feature']\ncoeff_linreg[\"score_linreg\"] = pd.Series(linreg.coef_)\ncoeff_linreg.sort_values(by='score_linreg', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Eli5 visualization\neli5.show_weights(linreg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# the level of importance of features is not associated with the sign\ncoeff_linreg[\"score_linreg\"] = coeff_linreg[\"score_linreg\"].abs()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"feature_score = pd.merge(feature_score, coeff_linreg, on='feature')\nfeature_score = feature_score.fillna(0)\nfeature_score = feature_score.set_index('feature')\nfeature_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 6. Comparison of the all feature importance diagrams <a class=\"anchor\" id=\"6\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"trusted":false},"cell_type":"code","source":"#Thanks to https://www.kaggle.com/nanomathias/feature-engineering-importance-testing\n# MinMax scale all importances\nfeature_score = pd.DataFrame(\n    preprocessing.MinMaxScaler().fit_transform(feature_score),\n    columns=feature_score.columns,\n    index=feature_score.index\n)\n\n# Create mean column\nfeature_score['mean'] = feature_score.mean(axis=1)\n\n# Plot the feature importances\nfeature_score.sort_values('mean', ascending=False).plot(kind='bar', figsize=(20, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"feature_score.sort_values('mean', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Create total column with different weights\nfeature_score['total'] = 0.5*feature_score['score_lgb'] + 0.3*feature_score['score_xgb'] \\\n                       + 0.1*feature_score['score_logreg'] + 0.1*feature_score['score_linreg']\n\n# Plot the feature importances\nfeature_score.sort_values('total', ascending=False).plot(kind='bar', figsize=(20, 10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"feature_score.sort_values('total', ascending=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}