{"cells":[{"metadata":{"_uuid":"1dee50e02ba20b8aabe9775e0f8b19c40e1bcff8"},"cell_type":"markdown","source":"**New Update : \nThis notebook displays the results of a search for noisy features. This search has been carried in RandomForest mode.**"},{"metadata":{"trusted":true,"_uuid":"a594abe9f66c5a090177dff966d6bedf63718a9c"},"cell_type":"code","source":"# Import the necessary libraries\nimport numpy as np\nimport pandas as pd\nimport os\nimport time\nimport warnings\nimport gc\ngc.collect()\nimport os\nfrom six.moves import urllib\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nwarnings.filterwarnings('ignore')\n%matplotlib inline\nplt.style.use('seaborn')\nfrom scipy.stats import norm, skew\nfrom sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfbbf30dcb4c354827aae6fff1aeaa632a9e874e"},"cell_type":"code","source":"\n#Add All the Models Libraries\n\n# Scalers\nfrom sklearn.utils import shuffle\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.pipeline import FeatureUnion\n\n# Models\nfrom sklearn.ensemble import RandomForestRegressor\n\nfrom sklearn.linear_model import Lasso\nfrom sklearn.metrics import mean_squared_log_error,mean_squared_error, r2_score,mean_absolute_error\n\nfrom sklearn.model_selection import train_test_split #training and testing data split\nfrom sklearn import metrics #accuracy measure\nfrom sklearn.metrics import confusion_matrix #for confusion matrix\nfrom scipy.stats import reciprocal, uniform\n\nfrom sklearn.model_selection import StratifiedKFold, RepeatedKFold\n\n# Cross-validation\nfrom sklearn.model_selection import KFold #for K-fold cross validation\nfrom sklearn.model_selection import cross_val_score #score evaluation\nfrom sklearn.model_selection import cross_val_predict #prediction\nfrom sklearn.model_selection import cross_validate\n\n# GridSearchCV\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RandomizedSearchCV\n\n#Common data processors\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nfrom sklearn import feature_selection\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.utils import check_array\nfrom scipy import sparse\nfrom sklearn.feature_selection import RFECV","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"798008f46ddb84ca5ec248990d56b64e5ec84e5a"},"cell_type":"code","source":"# to make this notebook's output stable across runs\nnp.random.seed(123)\ngc.collect()\n# To plot pretty figures\n%matplotlib inline\nplt.rcParams['axes.labelsize'] = 14\nplt.rcParams['xtick.labelsize'] = 12\nplt.rcParams['ytick.labelsize'] = 12","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47d5d544fa7e4bf9305814409c3cabc148b0169a"},"cell_type":"code","source":"#Reduce the memory usage - Inspired by Panchajanya Banerjee\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"450f05f1bb8bd470972ec8a8f80a96f4b56402a7"},"cell_type":"code","source":"train = reduce_mem_usage(pd.read_csv('../input/train.csv',parse_dates=[\"first_active_month\"]))\ntest = reduce_mem_usage(pd.read_csv('../input/test.csv', parse_dates=[\"first_active_month\"]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2352513f1d53487e789e41fbfa050199154f8916"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78407176c3c40b9d15227165028d6b61f94f3ba3"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1858899e8d6a7a6808204567ded690e4a10810d7"},"cell_type":"code","source":"plt.subplot(1, 2, 2)\nsns.distplot(train.target, kde=True, fit = norm)\nplt.xlabel('Customer Loyality (Skewed)')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce244a5a05abaa8c91ff3c73d8c5f86ba592c9a1"},"cell_type":"code","source":"# Now extract the month, year, day, weekday\ntrain[\"month\"] = train[\"first_active_month\"].dt.month\ntrain[\"year\"] = train[\"first_active_month\"].dt.year\ntrain['week'] = train[\"first_active_month\"].dt.weekofyear\ntrain['dayofweek'] = train['first_active_month'].dt.dayofweek\ntrain['days'] = (datetime.date(2018, 2, 1) - train['first_active_month'].dt.date).dt.days\ntrain['quarter'] = train['first_active_month'].dt.quarter\ntrain['is_month_start'] = train['first_active_month'].dt.is_month_start\n\n#Interaction Variables\ntrain['days_feature1'] = train['days'] * train['feature_1']\ntrain['days_feature2'] = train['days'] * train['feature_2']\ntrain['days_feature3'] = train['days'] * train['feature_3']\n\ntest[\"month\"] = test[\"first_active_month\"].dt.month\ntest[\"year\"] = test[\"first_active_month\"].dt.year\ntest['week'] = test[\"first_active_month\"].dt.weekofyear\ntest['dayofweek'] = test['first_active_month'].dt.dayofweek\ntest['days'] = (datetime.date(2018, 2, 1) - test['first_active_month'].dt.date).dt.days\ntest['quarter'] = test['first_active_month'].dt.quarter\ntest['is_month_start'] = test['first_active_month'].dt.is_month_start\n\n#Interaction Variables\ntest['days_feature1'] = test['days'] * train['feature_1']\ntest['days_feature2'] = test['days'] * train['feature_2']\ntest['days_feature3'] = test['days'] * train['feature_3']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b54460353af274062f649afe26f034c4126a6a84"},"cell_type":"markdown","source":"Now we will try to extract more features from Transactions Data"},{"metadata":{"trusted":true,"_uuid":"f9835dd6582d3e9c36954b498810929c8438d2d2"},"cell_type":"code","source":"# Taking Reference from Other Kernels\ndef aggregate_transaction_hist(trans, prefix):  \n        \n    agg_func = {\n        'purchase_date' : ['max','min'],\n        'month_diff' : ['mean', 'min', 'max', 'var'],\n        'weekend' : ['sum', 'mean'],\n        'authorized_flag': ['sum', 'mean'],\n        'category_1': ['sum','mean', 'max','min'],\n        'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n        'installments': ['sum', 'mean', 'max', 'min', 'std'],  \n        'month_lag': ['max','min','mean','var'],\n        'card_id' : ['size'],\n        'month': ['nunique'],\n        'hour': ['nunique'],\n        'weekofyear': ['nunique'],\n        'dayofweek': ['nunique'],\n        'year': ['nunique'],\n        'subsector_id': ['nunique'],\n        'merchant_category_id' : ['nunique']\n    }\n    \n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06a04f3393a83cb2687b76deba417012e18ef606","scrolled":true},"cell_type":"code","source":"transactions = reduce_mem_usage(pd.read_csv('../input/historical_transactions.csv'))\ntransactions['authorized_flag'] = transactions['authorized_flag'].map({'Y': 1, 'N': 0})\ntransactions['category_1'] = transactions['category_1'].map({'Y': 1, 'N': 0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1decf075c9422fe20823691972f055b8a1c84f50","scrolled":true},"cell_type":"code","source":"#Feature Engineering - Adding new features inspired by Chau's first kernel\ntransactions['purchase_date'] = pd.to_datetime(transactions['purchase_date'])\ntransactions['year'] = transactions['purchase_date'].dt.year\ntransactions['weekofyear'] = transactions['purchase_date'].dt.weekofyear\ntransactions['month'] = transactions['purchase_date'].dt.month\ntransactions['dayofweek'] = transactions['purchase_date'].dt.dayofweek\ntransactions['weekend'] = (transactions.purchase_date.dt.weekday >=5).astype(int)\ntransactions['hour'] = transactions['purchase_date'].dt.hour \ntransactions['quarter'] = transactions['purchase_date'].dt.quarter\ntransactions['is_month_start'] = transactions['purchase_date'].dt.is_month_start\ntransactions['month_diff'] = ((datetime.datetime.today() - transactions['purchase_date']).dt.days)//30\ntransactions['month_diff'] += transactions['month_lag']\n\n#impute missing values - This is now excluded.\ntransactions['category_2'] = transactions['category_2'].fillna(1.0,inplace=True)\ntransactions['category_3'] = transactions['category_3'].fillna('A',inplace=True)\ntransactions['merchant_id'] = transactions['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)\n\ntransactions['category_3'] = transactions['category_3'].map({'A':0, 'B':1, 'C':2})\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93a165f209a8853e4c8e1f624becd0506e6fb0f1"},"cell_type":"code","source":"agg_func = {\n        'mean': ['mean'],\n    }\nfor col in ['category_2','category_3']:\n    transactions[col+'_mean'] = transactions['purchase_amount'].groupby(transactions[col]).agg('mean')\n    transactions[col+'_max'] = transactions['purchase_amount'].groupby(transactions[col]).agg('max')\n    transactions[col+'_min'] = transactions['purchase_amount'].groupby(transactions[col]).agg('min')\n    transactions[col+'_var'] = transactions['purchase_amount'].groupby(transactions[col]).agg('var')\n    agg_func[col+'_mean'] = ['mean']\n    \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"95128e635d5c0ad26b170652971aca4fc9c41ad1"},"cell_type":"code","source":"merge_trans = aggregate_transaction_hist(transactions, prefix='hist_')\ndel transactions\ngc.collect()\ntrain = pd.merge(train, merge_trans, on='card_id',how='left')\ntest = pd.merge(test, merge_trans, on='card_id',how='left')\ndel merge_trans\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2ecbbf35630592774f720ccf88849cb8dfe9987"},"cell_type":"code","source":"train.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3e6c6e3dec6f2153d02a684fb7ddbbace4c5550"},"cell_type":"code","source":"#Feature Engineering - Adding new features inspired by Chau's first kernel\ntrain['hist_purchase_date_max'] = pd.to_datetime(train['hist_purchase_date_max'])\ntrain['hist_purchase_date_min'] = pd.to_datetime(train['hist_purchase_date_min'])\ntrain['hist_purchase_date_diff'] = (train['hist_purchase_date_max'] - train['hist_purchase_date_min']).dt.days\ntrain['hist_purchase_date_average'] = train['hist_purchase_date_diff']/train['hist_card_id_size']\ntrain['hist_purchase_date_uptonow'] = (datetime.datetime.today() - train['hist_purchase_date_max']).dt.days\ntrain['hist_purchase_date_uptomin'] = (datetime.datetime.today() - train['hist_purchase_date_min']).dt.days\ntrain['hist_first_buy'] = (train['hist_purchase_date_min'] - train['first_active_month']).dt.days\nfor feature in ['hist_purchase_date_max','hist_purchase_date_min']:\n    train[feature] = train[feature].astype(np.int64) * 1e-9\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"370757bfba37b643f463f385d9e878ff27e320bc"},"cell_type":"code","source":"#Feature Engineering - Adding new features inspired by Chau's first kernel\ntest['hist_purchase_date_max'] = pd.to_datetime(test['hist_purchase_date_max'])\ntest['hist_purchase_date_min'] = pd.to_datetime(test['hist_purchase_date_min'])\ntest['hist_purchase_date_diff'] = (test['hist_purchase_date_max'] - test['hist_purchase_date_min']).dt.days\ntest['hist_purchase_date_average'] = test['hist_purchase_date_diff']/test['hist_card_id_size']\ntest['hist_purchase_date_uptonow'] = (datetime.datetime.today() - test['hist_purchase_date_max']).dt.days\ntest['hist_purchase_date_uptomin'] = (datetime.datetime.today() - test['hist_purchase_date_min']).dt.days\n\ntest['hist_first_buy'] = (test['hist_purchase_date_min'] - test['first_active_month']).dt.days\nfor feature in ['hist_purchase_date_max','hist_purchase_date_min']:\n    test[feature] = test[feature].astype(np.int64) * 1e-9\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97235255a0829b53aca3741cc24b845fb1febdc2"},"cell_type":"code","source":"#Check fpr Missing Values after Concatination\n\nobs = train.isnull().sum().sort_values(ascending = False)\npercent = round(train.isnull().sum().sort_values(ascending = False)/len(train)*100, 2)\npd.concat([obs, percent], axis = 1,keys= ['Number of Observations', 'Percent'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b318197b47187c45962e235b6b6355970081d4a8"},"cell_type":"code","source":"#Detect Outlier\n# Remove the Outliers if any \ntrain['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\ntrain['outliers'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87d92a8281d54ba80742eacf17b8925983555b70"},"cell_type":"code","source":"target = train['target']\ndel train['target']\n\ntrain = train.drop(['card_id', 'first_active_month'], axis = 1)\ntest = test.drop(['card_id', 'first_active_month'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de5dbc4519c349552d66f192c47d73872ddf0c71","scrolled":true},"cell_type":"code","source":"# Now check the shape of Train and Test Data\n#train = train.drop(['is_month_start'],axis=1)\ntrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b5d796a748bbdf483888c96f37634f07e4c8663"},"cell_type":"code","source":"for features in ['feature_1','feature_2','feature_3']:\n    order_label = train.groupby([features])['outliers'].mean()\n    train[features] = train[features].map(order_label)\n    test[features] =  test[features].map(order_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a87782915b00c782f6f14e583e90c31b758b879"},"cell_type":"code","source":"train = train.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dac5e3fdfb1731d5b883ab2268f386cf61a869f7"},"cell_type":"markdown","source":"Apply Recursive Feature selection on the Random Forest with Stratified K Folds to select important features "},{"metadata":{"trusted":true,"_uuid":"29d413b0cf270375808c3424410e71e07f71e15e","scrolled":false},"cell_type":"code","source":"'''folds = 5\nstep = 2\n\nrfc = RandomForestRegressor(n_estimators=100, max_features='sqrt', max_depth=10, n_jobs=4)\n\nlgbmcv = RFECV(\n              estimator=rfc,\n              step=step,\n              cv=StratifiedKFold(\n                                 n_splits=folds,\n                                 shuffle=False,\n                                 random_state=42).split(train,train['outliers'].values),\n              n_jobs=1,\n              verbose=2)\n\nlgbmcv.fit(train,target)'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8b3c5b718779a1d5729e89a5341258c44de24e2","trusted":true,"scrolled":true},"cell_type":"code","source":"'''\n#Total Set of Features\nprint ('\\n Initial features: ', train.columns.tolist() )\n\n# number of Important features\nprint ('\\n Number of Shortlisted features:')\nprint (lgbmcv.n_features_)\n\nall_features = [c for c in train.columns if c not in ['target','outliers']] \n\nsel_features = [f for f, s in zip(all_features, lgbmcv.support_) if s]\nprint('\\n The selected features are {}:'.format(sel_features))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a7ca5b8ad144fc5b85222c9cfe829406aa3e352"},"cell_type":"code","source":"'''\nplt.figure(figsize=(12, 9))\nplt.xlabel('Number of features tested x 2')\nplt.ylabel('Cross-validation score')\nplt.plot(range(1, len(lgbmcv.grid_scores_) + 1), lgbmcv.grid_scores_)\nplt.savefig('ELO-lgbmcv-01.png', dpi=150)\nplt.show()'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f8ef58f45b6829247e95edbb18694ba231bc1afc"},"cell_type":"markdown","source":"Now lets look over the same implementation with Light GBM Model in the Random Forest Mode i.e. boosting : \"rf\""},{"metadata":{"trusted":true,"_uuid":"9521a3cc430262b05cd7b3e910e890db5ea71cc2"},"cell_type":"code","source":"from lightgbm import LGBMRegressor\n\nfolds = 5\nstep = 2\n\n# We will not use Param, in LGBM Regressor, Instead we call the same parameters inside LGBM\nparam = {'num_leaves': 31,\n         'min_data_in_leaf': 27, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate': 0.015,\n         \"boosting\": \"rf\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1,\n         \"nthread\": 4,\n         \"random_state\": 4950}\n\nlgbmclf = LGBMRegressor(boosting_type='rf', objective='regression', num_iteration=10000,num_leaves=31,\n                        min_data_in_leaf=27,max_depth=-1, learning_rate=0.015, feature_fraction= 0.9,\n                        bagging_freq= 1,bagging_fraction= 0.9,bagging_seed= 11,metric= 'rmse',lambda_l1=0.1,\n                        verbosity= -1,random_state= 4950)\n\nlgbmcv_2 = RFECV(\n              estimator=lgbmclf,\n              step=step,\n              cv=StratifiedKFold(\n                                 n_splits=folds,\n                                 shuffle=False,\n                                 random_state=42).split(train,train['outliers'].values),\n              n_jobs=1,\n              verbose=2)\n\nlgbmcv_2.fit(train,target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee0ac7a5c19dafa424e313843f44320fe03b03b1"},"cell_type":"code","source":"#Total Set of Features\nprint ('\\n Initial features: ', train.columns.tolist() )\n\n# number of Important features\nprint ('\\n Number of Shortlisted features:')\nprint (lgbmcv_2.n_features_)\n\nall_features_2 = [c for c in train.columns if c not in ['target','outliers']] \n\nsel_features_2 = [f for f, s in zip(all_features_2, lgbmcv_2.support_) if s]\nprint('\\n The selected features are {}:'.format(sel_features_2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea4e864fcfb70ae3f540016699b9f992615fbc86"},"cell_type":"code","source":"plt.figure(figsize=(12, 9))\nplt.xlabel('Number of features tested x 2')\nplt.ylabel('Cross-validation score')\nplt.plot(range(1, len(lgbmcv_2.grid_scores_) + 1), lgbmcv_2.grid_scores_)\nplt.savefig('ELO-lgbmcv-02.png', dpi=150)\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}