{"cells":[{"metadata":{},"cell_type":"markdown","source":"#### Payment organization **[Elo](http://)**, which is operated widely in Brazil offer discounts for card-holders. Elo wants to know that is really these discounts helpful to keep the card-holders happy.\n\n#### Based on target-score we are going to predict whether discounts helpful or not.\n\n#### I will use train & historical_transactions for training and testing purpose.\n \n#### As Dependent variable is continuous so I will use regression algorithms starting from basic Linear-Regression model to Gradient boosting models and RMSE as evaluation metric\n"},{"metadata":{},"cell_type":"markdown","source":"#### Importing required Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import *\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV\nimport lightgbm as lgb\nimport xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Importing data files"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train=pd.read_csv(\"../input/train.csv\")\ntest=pd.read_csv(\"../input/test.csv\")\nhist_transactions=pd.read_csv(\"../input/historical_transactions.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data preprocessing"},{"metadata":{"trusted":true,"_uuid":"3e9b7c4fd284a4ec62ccbc83b14f13ccf70182a0"},"cell_type":"code","source":"# looking for dimensions of data\ntrain.head()","execution_count":42,"outputs":[{"output_type":"execute_result","execution_count":42,"data":{"text/plain":" first_active_month ... first_active_year\n0 6 ... 2017\n1 1 ... 2017\n2 8 ... 2016\n3 9 ... 2017\n4 11 ... 2017\n\n[5 rows x 11 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
first_active_monthcard_idfeature_1feature_2feature_3targetmonth_laghist_transactions/cardpurchase_amtinstallmentsfirst_active_year
06C_ID_92a2005557521-0.820283-4.0260-0.6383410.02017
11C_ID_3d0044924f4100.392913-5.0350-0.6000182.02017
28C_ID_d639edf6cd2200.688056-9.043-0.6783110.02016
39C_ID_186d6a69014300.142495-3.077-0.6427451.02017
411C_ID_cdbd2c0db2130-0.159749-1.0133-0.3660731.02017
\n
"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"5df38a46fba3ad31cdf31dca29d49f6c4b07d8b4","scrolled":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab35bfa133c49d7ee4b33c4b57807e4b91df76d6"},"cell_type":"code","source":"train.shape, test.shape, hist_transactions.shape","execution_count":43,"outputs":[{"output_type":"execute_result","execution_count":43,"data":{"text/plain":"((201917, 11), (123623, 5), (29112361, 14))"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"530f4342c0244de39a434a935495dd6900acfcb9"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf7c9e64a7744cb139c247dc10161d4182bf3e81"},"cell_type":"code","source":"test.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f662ec6a789d16847380e37f09756249df35e56"},"cell_type":"code","source":"hist_transactions.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de01f506a485f5816dc98fcf1bc27d4bc39e2056"},"cell_type":"code","source":"train.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7f1e5c23c69290a75768fe9f85a81204081a8e4"},"cell_type":"code","source":"hist_transactions.describe(include='all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45363d3461e8de8cc75b7852869f8285faf72cea","scrolled":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Checking for NA values in train \ntrain.isna().sum().plot(kind='barh')\nfor i, v in enumerate(train.isna().sum()):\n plt.text( v,i, str(v))\nplt.title('missing values count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Distribution of cards used first-time \n\ntrain['first_active_month']=pd.to_datetime(train['first_active_month'])\ncount = train['first_active_month'].dt.date.value_counts()\ncount= count.sort_index()\nplt.figure(figsize=(14,6))\nsns.barplot(count.index, count.values)\nplt.xticks(rotation='vertical')\nplt.xlabel('First active month')\nplt.ylabel('Number of cards')\nplt.title(\"First active month count in train set\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking for the distributions of features using violin plot\n\n# feature 1\nplt.figure(figsize=(8,4))\nsns.violinplot(x=\"feature_1\", y='target', data=train)\nplt.xlabel('Feature 1')\nplt.ylabel('target score')\nplt.title(\"Feature 1 distribution\")\nplt.show()\n\n# feature 2\nplt.figure(figsize=(8,4))\nsns.violinplot(x=\"feature_2\", y='target', data=train)\nplt.xlabel('Feature 2')\nplt.ylabel('target score')\nplt.title(\"Feature 2 distribution\")\nplt.show()\n \n# feature 3\nplt.figure(figsize=(8,4))\nsns.violinplot(x=\"feature_3\", y='target', data=train)\nplt.xlabel('Feature 3')\nplt.ylabel('target score')\nplt.title(\"Feature 3 distribution\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Hist_transactions dataset"},{"metadata":{},"cell_type":"markdown","source":"month_lag"},{"metadata":{"trusted":true},"cell_type":"code","source":"Avg_month_lag= np.round(hist_transactions.groupby('card_id')['month_lag'].agg('mean').reset_index())\ntrain= pd.merge(train, Avg_month_lag, on=\"card_id\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Card-id"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_trans = hist_transactions.card_id.value_counts().reset_index()\nnum_trans.columns = [\"card_id\", \"hist_transactions/card\"]\ntrain= pd.merge(train, num_trans, on=\"card_id\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter('hist_transactions/card', 'target', data=train)\nplt.xlabel('Number of hist_transactions/card')\nplt.ylabel('target score')\nplt.title('Number of hist_transactions/card vs target score')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Purchase_amount"},{"metadata":{"trusted":true},"cell_type":"code","source":"pur_amt = hist_transactions.groupby(\"card_id\")\npur_amt = pur_amt[\"purchase_amount\"].agg('mean').reset_index()\npur_amt.columns = [\"card_id\", \"purchase_amt\"]\ntrain= pd.merge(train, pur_amt, on=\"card_id\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter('hist_transactions/card', 'target', data=train)\nplt.xlabel('Number of hist_transactions/card')\nplt.ylabel('target score')\nplt.title('Number of hist_transactions/card Vs target score')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Installments"},{"metadata":{"trusted":true},"cell_type":"code","source":"installments_percard = np.round(hist_transactions.groupby('card_id')['installments'].agg('mean').reset_index())\ntrain= pd.merge(train, installments_percard, on=\"card_id\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter('installments', 'target', data=train)\nplt.xlabel('no.of installments')\nplt.ylabel('target score')\nplt.title('no.of installments Vs target scre')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"first_active_month"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['first_active_month']=pd.to_datetime(train['first_active_month'])\n\ntrain['first_active_year']=train['first_active_month'].dt.year\ntrain['first_active_month']=train['first_active_month'].dt.month\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(range(train.shape[0]), np.sort(train.target))\nplt.ylabel('target Score')\nplt.title('target-score distribution')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Checking for correlatioin between variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(train.corr(), annot=True)\nplt.title('Correlation map')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Splitting dataset into train and test sets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x=train.drop(['target', 'card_id'], axis=1)\ntrain_y=train['target']","execution_count":44,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test=train_test_split(train_x, train_y, test_size=0.33)","execution_count":45,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape, x_test.shape, y_train.shape, y_test.shape","execution_count":46,"outputs":[{"output_type":"execute_result","execution_count":46,"data":{"text/plain":"((135284, 9), (66633, 9), (135284,), (66633,))"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Linear regression model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model=LinearRegression()\nmodel.fit(x_train, y_train)","execution_count":47,"outputs":[{"output_type":"execute_result","execution_count":47,"data":{"text/plain":"LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,\n normalize=False)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict=model.predict(x_test)\npredict_train=model.predict(x_train)","execution_count":48,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('RMSE test:', np.sqrt(np.mean((predict - y_test)**2)))\nprint('RMSE train:', np.sqrt(np.mean((predict_train - y_train)**2)))","execution_count":49,"outputs":[{"output_type":"stream","text":"RMSE test: 3.8402728637749637\nRMSE train: 3.851667472259535\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Randomforest Regresssor"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_rf=RandomForestRegressor()\nmodel_rf.fit(x_train, y_train)","execution_count":50,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/sklearn/ensemble/forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n","name":"stderr"},{"output_type":"execute_result","execution_count":50,"data":{"text/plain":"RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n max_features='auto', max_leaf_nodes=None,\n min_impurity_decrease=0.0, min_impurity_split=None,\n min_samples_leaf=1, min_samples_split=2,\n min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=None,\n oob_score=False, random_state=None, verbose=0, warm_start=False)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_rf=model_rf.predict(x_test)\npredict_rf_train=model_rf.predict(x_train)","execution_count":51,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Test RMSE RF:', np.sqrt(np.mean((predict_rf - y_test)**2)))\nprint('Train RMSE RF:', np.sqrt(np.mean((predict_rf_train - y_train)**2)))","execution_count":52,"outputs":[{"output_type":"stream","text":"Test RMSE RF: 4.152158818141261\nTrain RMSE RF: 1.7716493810726186\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"parameter_tuning in Randomforest Regresssor"},{"metadata":{"trusted":true,"_uuid":"b022bc7477bd634d8948179a209e3d4e9455a85e"},"cell_type":"code","source":"Random_Search_Params ={\n 'max_features':[1,2,3,4,5,6,7,8,9,10],\n \"max_depth\": list(range(1,train.shape[1])),\n 'n_estimators' : [1, 2, 4, 8, 50, 100,150, 200, 250, 300],\n \"min_samples_leaf\": [5,10,15,20,25],\n 'random_state' : [42] \n }\n\n\nrandom_search = RandomizedSearchCV(\n estimator=RandomForestRegressor(),\n param_distributions= Random_Search_Params, \n cv=3,\n refit=True,\n verbose=True)","execution_count":53,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random_search.fit(x_train, y_train)","execution_count":54,"outputs":[{"output_type":"stream","text":"Fitting 3 folds for each of 10 candidates, totalling 30 fits\n","name":"stdout"},{"output_type":"stream","text":"[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n[Parallel(n_jobs=1)]: Done 30 out of 30 | elapsed: 5.2min finished\n","name":"stderr"},{"output_type":"execute_result","execution_count":54,"data":{"text/plain":"RandomizedSearchCV(cv=3, error_score='raise-deprecating',\n estimator=RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n max_features='auto', max_leaf_nodes=None,\n min_impurity_decrease=0.0, min_impurity_split=None,\n min_samples_leaf=1, min_samples_split=2,\n min_weight_fraction_leaf=0.0, n_estimators='warn', n_jobs=None,\n oob_score=False, random_state=None, verbose=0, warm_start=False),\n fit_params=None, iid='warn', n_iter=10, n_jobs=None,\n param_distributions={'max_features': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 'n_estimators': [1, 2, 4, 8, 50, 100, 150, 200, 250, 300], 'min_samples_leaf': [5, 10, 15, 20, 25], 'random_state': [42]},\n pre_dispatch='2*n_jobs', random_state=None, refit=True,\n return_train_score='warn', scoring=None, verbose=True)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"random_search.best_params_","execution_count":55,"outputs":[{"output_type":"execute_result","execution_count":55,"data":{"text/plain":"{'random_state': 42,\n 'n_estimators': 250,\n 'min_samples_leaf': 15,\n 'max_features': 6,\n 'max_depth': 7}"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nmodel_rf_tune=RandomForestRegressor( random_state=42, \n n_estimators=250, min_samples_leaf=15,\n max_features=6, max_depth=7 )","execution_count":56,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_rf_tune.fit(x_train, y_train)","execution_count":57,"outputs":[{"output_type":"execute_result","execution_count":57,"data":{"text/plain":"RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=7,\n max_features=6, max_leaf_nodes=None, min_impurity_decrease=0.0,\n min_impurity_split=None, min_samples_leaf=15,\n min_samples_split=2, min_weight_fraction_leaf=0.0,\n n_estimators=250, n_jobs=None, oob_score=False, random_state=42,\n verbose=0, warm_start=False)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_rf_tune=model_rf_tune.predict(x_test)\n\npredict_rf_tune_train=model_rf_tune.predict(x_train)","execution_count":58,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Test RMSE RF_tune_:', np.sqrt(np.mean((predict_rf_tune - y_test)**2)))\nprint('Train RMSE RF_tune:', np.sqrt(np.mean((predict_rf_tune_train - y_train)**2)))","execution_count":59,"outputs":[{"output_type":"stream","text":"Test RMSE RF_tune_: 3.793012505695367\nTrain RMSE RF_tune: 3.8093516128689835\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28f2797d33bad3b9f2a4c0de9a2ca1ccba9aec1c"},"cell_type":"markdown","source":"lgb model"},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {'num_leaves': 30,\n 'min_data_in_leaf': 20,\n 'objective': 'regression',\n 'max_depth': 5,\n 'learning_rate': 0.01,\n \"boosting\": \"gbrt\",\n \"metric\": 'rmse'}\n\nlgb_model = lgb.LGBMRegressor(**params, n_estimators = 10000, n_jobs = -1)\nlgb_model.fit(x_train, y_train, \n eval_set=[(x_train, y_train), (x_test, y_test)], eval_metric='rmse',\n verbose=1000, early_stopping_rounds=1000)","execution_count":62,"outputs":[{"output_type":"stream","text":"Training until validation scores don't improve for 1000 rounds.\n[1000]\ttraining's rmse: 3.79487\tvalid_1's rmse: 3.79283\nEarly stopping, best iteration is:\n[527]\ttraining's rmse: 3.8078\tvalid_1's rmse: 3.7916\n","name":"stdout"},{"output_type":"execute_result","execution_count":62,"data":{"text/plain":"LGBMRegressor(boosting='gbrt', boosting_type='gbdt', class_weight=None,\n colsample_bytree=1.0, importance_type='split', learning_rate=0.01,\n max_depth=5, metric='rmse', min_child_samples=20,\n min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n n_estimators=10000, n_jobs=-1, num_leaves=30,\n objective='regression', random_state=None, reg_alpha=0.0,\n reg_lambda=0.0, silent=True, subsample=1.0,\n subsample_for_bin=200000, subsample_freq=0)"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"XGBoost model"},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_params = {'eta': 0.01,\n 'objective': 'reg:linear',\n 'max_depth': 6,\n 'min_child_weight': 3,\n 'subsample': 0.8,\n \n 'eval_metric': 'rmse',\n 'seed': 11,\n 'silent': True}\n\nmodel_xgb = xgb.XGBRegressor() \nmodel_xgb.fit(x_train, y_train)","execution_count":63,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n if getattr(data, 'base', None) is not None and \\\n","name":"stderr"},{"output_type":"execute_result","execution_count":63,"data":{"text/plain":"XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n colsample_bytree=1, gamma=0, importance_type='gain',\n learning_rate=0.1, max_delta_step=0, max_depth=3,\n min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n nthread=None, objective='reg:linear', random_state=0, reg_alpha=0,\n reg_lambda=1, scale_pos_weight=1, seed=None, silent=True,\n subsample=1)"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainPredict_xgb = model_xgb.predict(x_train)\ntestPredict_xgb = model_xgb.predict(x_test)\n\nprint(\"xgb test RMSE:\", np.sqrt(mean_squared_error(y_test, testPredict_xgb)))\nprint(\"xgb train RMSE:\", np.sqrt( mean_squared_error(y_train, trainPredict_xgb)))","execution_count":64,"outputs":[{"output_type":"stream","text":"xgb test RMSE: 3.7951851600758166\nxgb train RMSE: 3.8219666616815324\n","name":"stdout"}]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}