{"cells":[{"metadata":{"_uuid":"b843c2aadecc1279db49e25f25b036f1592eccad"},"cell_type":"markdown","source":"# References\n* https://www.kaggle.com/rooshroosh/simple-data-exploration-with-python-lb-3-762\n* https://www.kaggle.com/youhanlee/hello-elo-ensemble-will-help-you\n* https://www.kaggle.com/truocpham/feature-engineering-and-lightgbm-starter\n* https://www.kaggle.com/youhanlee/hello-elo-ensemble-will-help-you"},{"metadata":{"_uuid":"f61d94c315eaccc4fe84161b9bfe5d0fd437e788"},"cell_type":"markdown","source":"# Import libs and Load data"},{"metadata":{"trusted":true,"_uuid":"5e50e9d3a11505ac9ed8c68df983a418a801e164"},"cell_type":"code","source":"# Loading packages\nimport pandas as pd #Analysis \nimport matplotlib.pyplot as plt #Visulization\nimport seaborn as sns #Visulization\nimport numpy as np #Analysis \nfrom scipy.stats import norm #Analysis \nfrom sklearn.preprocessing import StandardScaler #Analysis \nfrom scipy import stats #Analysis \nimport warnings \nwarnings.filterwarnings('ignore')\n%matplotlib inline\nimport gc\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb\n\nimport xgboost as xgb\nfrom scipy.optimize import minimize","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"775befb73d77565afbc3ee0b2c44baa55526d2f3"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\nprint(\"shape of train : \",train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71046be24648982135a9031d9cbb4a249cf29172"},"cell_type":"code","source":"test = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])\nprint(\"shape of test : \",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import datetime\n\nfor df in [train,test]:\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['year'] = df['first_active_month'].dt.year\n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n\ntarget = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"285c2652c1068b3e1e6cb1947844ac80e5b5b702"},"cell_type":"markdown","source":"## History Transactions Processing"},{"metadata":{"trusted":true,"_uuid":"36ca3402f4bcf3f0acc5f8b7f4975f10781ac29c"},"cell_type":"code","source":"ht = pd.read_csv(\"../input/historical_transactions.csv\")\nprint(\"shape of historical_transactions : \",ht.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cd38fb19662279ff148c1a4c2c4be24fb36626c"},"cell_type":"code","source":"ht['authorized_flag'] = ht['authorized_flag'].map({'Y':1, 'N':0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06cc6e7186fe9d6594497b4724586057c7ad4b9e"},"cell_type":"code","source":"def aggregate_historical_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'merchant_id': ['nunique'],\n        'city_id': ['nunique'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std'],\n        'installments': ['sum', 'median', 'max', 'min', 'std'],\n        'purchase_date': [np.ptp],\n        'month_lag': ['min', 'max']\n        }\n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['hist_' + '_'.join(col).strip() \n                           for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='hist_transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history\n\nhistory = aggregate_historical_transactions(ht)\ndel ht\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e682c81c76b72a91701ecb501d5a725bcf2cddb9"},"cell_type":"markdown","source":"**Merge history to train and test**"},{"metadata":{"trusted":true,"_uuid":"c4276c940022e1b999ae85cd6c13487f5db3b867"},"cell_type":"code","source":"train = pd.merge(train, history, on='card_id', how='left')\ntest = pd.merge(test, history, on='card_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8bc55699381c8234732305736695fc0e009c9414"},"cell_type":"markdown","source":"## Merchants Processing"},{"metadata":{"trusted":true,"_uuid":"ce2620d239a85cf8031f522196251dc41c02296c"},"cell_type":"code","source":"merchant = pd.read_csv(\"../input/merchants.csv\")\nprint(\"shape of merchant : \",merchant.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca5f8f56a31ccc4c2f7bce80d2b54f279ad8554f"},"cell_type":"code","source":"new_merchant = pd.read_csv(\"../input/new_merchant_transactions.csv\")\nprint(\"shape of new_merchant_transactions : \",new_merchant.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2477156c60425de6b9015572d0f27226358b345a"},"cell_type":"code","source":"new_merchant['authorized_flag'] = new_merchant['authorized_flag'].map({'Y':1, 'N':0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"357dd8ff0ed795a501471898212b75836b865fd4"},"cell_type":"code","source":"def aggregate_new_transactions(new_trans):    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'merchant_id': ['nunique'],\n        'city_id': ['nunique'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std'],\n        'installments': ['sum', 'median', 'max', 'min', 'std'],\n        'month_lag': ['min', 'max']\n        }\n    agg_new_trans = new_trans.groupby(['card_id']).agg(agg_func)\n    agg_new_trans.columns = ['new_' + '_'.join(col).strip() \n                           for col in agg_new_trans.columns.values]\n    agg_new_trans.reset_index(inplace=True)\n    \n    df = (new_trans.groupby('card_id')\n          .size()\n          .reset_index(name='new_transactions_count'))\n    \n    agg_new_trans = pd.merge(df, agg_new_trans, on='card_id', how='left')\n    \n    return agg_new_trans\n\nnew_trans = aggregate_new_transactions(new_merchant)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec702ef79d7aae387cc083f8aa30b90136b534dc"},"cell_type":"code","source":"train = pd.merge(train, new_trans, on='card_id', how='left')\ntest = pd.merge(test, new_trans, on='card_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2c60815c92e123b070f528cc3e8bfcc95e261b3"},"cell_type":"markdown","source":"# Feature engineering"},{"metadata":{"trusted":true,"_uuid":"e044448d9e95e5a046dbb2cf3f2ed5fb28a72226"},"cell_type":"code","source":"use_cols = [col for col in train.columns if col not in ['card_id', 'first_active_month']]\n\ntrain = train[use_cols]\ntest = test[use_cols]\n\nfeatures = list(train[use_cols].columns)\ncategorical_feats = [col for col in features if 'feature_' in col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1aa06fc1eaf9d0605d1e68ccd1d19158b9f569cf"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfor col in categorical_feats:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n    train[col] = lbl.transform(list(train[col].values.astype('str')))\n    test[col] = lbl.transform(list(test[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb862eb90dee49e5ce903509679edfd393e4203f"},"cell_type":"code","source":"df_all = pd.concat([train, test])\ndf_all = pd.get_dummies(df_all, columns=categorical_feats)\n\nlen_train = train.shape[0]\n\ntrain = df_all[:len_train]\ntest = df_all[len_train:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c72b6f3afc6e28618c31d244d8c7ff8483664db2"},"cell_type":"markdown","source":"## LightGBM"},{"metadata":{"trusted":true,"_uuid":"267c92c42a3654b1307b82e1a0df73a07603988e"},"cell_type":"code","source":"lgb_params = {\"objective\" : \"regression\", \"metric\" : \"rmse\", \n               \"max_depth\": 9, \"min_child_samples\": 20, \n               \"reg_alpha\": 1, \"reg_lambda\": 1,\n               \"num_leaves\" : 64, \"learning_rate\" : 0.005, \n               \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \n               \"verbosity\": -1}\n\nFOLDs = KFold(n_splits=10, shuffle=True, random_state=1989)\n\noof_lgb = np.zeros(len(train))\npredictions_lgb = np.zeros(len(test))\n\nfeatures_lgb = list(train.columns)\nfeature_importance_df_lgb = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = lgb.Dataset(train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx], label=target.iloc[val_idx])\n\n    print(\"LGB \" + str(fold_) + \"-\" * 50)\n    num_round = 10000\n    clf = lgb.train(lgb_params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 50)\n    oof_lgb[val_idx] = clf.predict(train.iloc[val_idx], num_iteration=clf.best_iteration)\n\n    fold_importance_df_lgb = pd.DataFrame()\n    fold_importance_df_lgb[\"feature\"] = features_lgb\n    fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n    fold_importance_df_lgb[\"fold\"] = fold_ + 1\n    feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n    predictions_lgb += clf.predict(test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n    \nprint(np.sqrt(mean_squared_error(oof_lgb, target)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3773981487344020aa38073624f7c0f9a83e7c44"},"cell_type":"code","source":"cols = (feature_importance_df_lgb[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\n\nplt.figure(figsize=(14,14))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46e34d642ddc0001c5c44827e458b96cf09d8eba"},"cell_type":"markdown","source":"## XGBoost"},{"metadata":{"trusted":true,"_uuid":"3d0c873e8b504eb2d56a5f5738778795e14e27f5"},"cell_type":"code","source":"xgb_params = {'eta': 0.005, 'max_depth': 9, 'subsample': 0.8, 'colsample_bytree': 0.8, \n          'objective': 'reg:linear', 'eval_metric': 'rmse', 'silent': True}\n\nFOLDs = KFold(n_splits=10, shuffle=True, random_state=1989)\n\noof_xgb = np.zeros(len(train))\npredictions_xgb = np.zeros(len(test))\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = xgb.DMatrix(data=train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = xgb.DMatrix(data=train.iloc[val_idx], label=target.iloc[val_idx])\n    watchlist = [(trn_data, 'train'), (val_data, 'valid')]\n    print(\"xgb \" + str(fold_) + \"-\" * 50)\n    num_round = 10000\n    xgb_model = xgb.train(xgb_params, trn_data, num_round, watchlist, early_stopping_rounds=50, verbose_eval=1000)\n    oof_xgb[val_idx] = xgb_model.predict(xgb.DMatrix(train.iloc[val_idx]), ntree_limit=xgb_model.best_ntree_limit+50)\n\n    predictions_xgb += xgb_model.predict(xgb.DMatrix(test), ntree_limit=xgb_model.best_ntree_limit+50) / FOLDs.n_splits\n\nnp.sqrt(mean_squared_error(oof_xgb, target))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"401cb67da886b7701fed6c888b176e2b0ef2100b"},"cell_type":"markdown","source":"## Ensemble and make submission"},{"metadata":{"trusted":true,"_uuid":"e73de535c3facf976339166d895ef6d6a8b4a31e"},"cell_type":"code","source":"print('lgb', np.sqrt(mean_squared_error(oof_lgb, target)))\nprint('xgb', np.sqrt(mean_squared_error(oof_xgb, target)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c5c6082312c92d9c9c9a0d0ed12bf8c44204b6c"},"cell_type":"code","source":"def find_best_weight(preds, target):\n    def _validate_func(weights):\n        ''' scipy minimize will pass the weights as a numpy array '''\n        final_prediction = 0\n        for weight, prediction in zip(weights, preds):\n                final_prediction += weight * prediction\n        return np.sqrt(mean_squared_error(final_prediction, target))\n\n    #the algorithms need a starting value, right not we chose 0.5 for all weights\n    #its better to choose many random starting points and run minimize a few times\n    starting_values = [0.5]*len(preds)\n\n    #adding constraints and a different solver as suggested by user 16universe\n    #https://kaggle2.blob.core.windows.net/forum-message-attachments/75655/2393/otto%20model%20weights.pdf?sv=2012-02-12&se=2015-05-03T21%3A22%3A17Z&sr=b&sp=r&sig=rkeA7EJC%2BiQ%2FJ%2BcMpcA4lYQLFh6ubNqs2XAkGtFsAv0%3D\n    cons = ({'type':'eq','fun':lambda w: 1-sum(w)})\n    #our weights are bound between 0 and 1\n    bounds = [(0, 1)] * len(preds)\n    \n    res = minimize(_validate_func, starting_values, method='Nelder-Mead', bounds=bounds, constraints=cons)\n    \n    print('Ensemble Score: {best_score}'.format(best_score=(1-res['fun'])))\n    print('Best Weights: {weights}'.format(weights=res['x']))\n    \n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8c6454d4dc79d810000ada0343a6fb2906942e1"},"cell_type":"code","source":"res = find_best_weight([oof_lgb, oof_xgb], target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f63cf0c61861b73089a1b5679f45ddea47b851f"},"cell_type":"code","source":"total_sum = 0.71044189 * oof_lgb + 0.36912984 * oof_xgb\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(total_sum, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be40b3d2fc16714a9440cd83063836919a0a9818"},"cell_type":"code","source":"sub_df = pd.read_csv('../input/sample_submission.csv')\nsub_df[\"target\"] = 0.71044189 * predictions_lgb + 0.36912984 * predictions_xgb\nsub_df.to_csv(\"submission_ensemble.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}