{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Notes\n\nThis kernel just referred a very helpful kernel from Thuong Dinh. Thanks a lot!\n\n**References:**\n\n* https://www.kaggle.com/thuongdinh/ensemble-with-best-weights-finding\n* https://www.kaggle.com/truocpham/feature-engineering-and-lightgbm-starter\n\n**Update:**\n\n* Add some aggregate functions\n* Remove **year** feature\n* Do ensemble 3 models (LightGBM, CatBoost, XGBoost) with best weights finding"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Import libs"},{"metadata":{"trusted":true,"_uuid":"f8d5d300178590ee99ca10912c1327e0bc2af9b6"},"cell_type":"code","source":"import os\nimport gc\nimport datetime\n\nimport pandas as pd #Analysis \nimport matplotlib.pyplot as plt #Visulization\nimport seaborn as sns #Visulization\nimport numpy as np #Analysis \nfrom scipy.stats import norm #Analysis \nfrom sklearn.preprocessing import StandardScaler #Analysis \nfrom scipy import stats #Analysis \n%matplotlib inline\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n\nfrom scipy.optimize import minimize\n\nimport lightgbm as lgb\nimport catboost as cb\nimport xgboost as xgb\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\nprint(os.listdir('../input'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c7d7ec3e29f56f7e10c17d20b6e402ae1b0cce07"},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true,"_uuid":"cf2f4c36c52c981ad2fe8a6679e4ae271c966c33"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\nprint(\"shape of train : \",train.shape)\n\ntest = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])\nprint(\"shape of test : \",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6362b7ccf0b30861bceb4bc7112ecde0a12db0e6"},"cell_type":"markdown","source":"# Preprocessing"},{"metadata":{"_uuid":"a14c01d4c3d4547eaec65a5e18c897320624396a"},"cell_type":"markdown","source":"## Train and Test Preprocessing"},{"metadata":{"trusted":true,"_uuid":"5558500cc4bec8c66dfe26223a6b65feb8c07cf0"},"cell_type":"code","source":"def missing_impute(df):\n    for i in df.columns:\n        if df[i].dtype == \"object\":\n            df[i] = df[i].fillna(\"other\")\n        elif (df[i].dtype == \"int64\" or df[i].dtype == \"float64\"):\n            df[i] = df[i].fillna(df[i].mean())\n        else:\n            pass\n    return df\n\n\ndef datetime_extract(df, dt_col='first_active_month'):\n    df['date'] = df[dt_col].dt.date \n    df['day'] = df[dt_col].dt.day \n    df['dayofweek'] = df[dt_col].dt.dayofweek\n    df['dayofyear'] = df[dt_col].dt.dayofyear\n    df['days_in_month'] = df[dt_col].dt.days_in_month\n    df['daysinmonth'] = df[dt_col].dt.daysinmonth \n    df['month'] = df[dt_col].dt.month\n    df['week'] = df[dt_col].dt.week \n    df['weekday'] = df[dt_col].dt.weekday\n    df['weekofyear'] = df[dt_col].dt.weekofyear\n    # df['year'] = train[dt_col].dt.year\n    \n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['date']).dt.days\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7db67874e7e43f4d6c2969d3281a46aab791a27d"},"cell_type":"code","source":"# Do impute missing values for train & test\nfor df in [train, test]:\n    missing_impute(df)\n    \n# Do extract datetime values for train & test\ntrain = datetime_extract(train, dt_col='first_active_month')\ntest = datetime_extract(test, dt_col='first_active_month')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a63c6f790656b3627ee6d052b40f60fa5fe3d3b"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53608c1e1acffb862602615fd50fdfcaa52ec367"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d846a14ab7b392ea6fbe34f9626c0ce6413bc96a"},"cell_type":"markdown","source":"## Historical Transaction Preprocessing"},{"metadata":{"trusted":true,"_uuid":"450d04bec4268810b307fc8b18b364c04fd4202b"},"cell_type":"code","source":"ht = pd.read_csv(\"../input/historical_transactions.csv\")\nprint(\"shape of historical_transactions: \", ht.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a52cafd4134c2ede624f9a8b5a83d0dcf32ee1ab"},"cell_type":"code","source":"ht['authorized_flag'] = ht['authorized_flag'].map({'Y':1, 'N':0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d1c80fc888c88cc8fb85c72f34df574e75d008a"},"cell_type":"code","source":"# Do impute missing values for history\nht = missing_impute(ht)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e972bc7639616094faf7e7ceb80f41b9771b8cb"},"cell_type":"code","source":"def aggregate_historical_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'merchant_id': ['nunique'],\n        'city_id': ['nunique'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std', 'mean', 'size'],\n        'installments': ['sum', 'median', 'max', 'min', 'std', 'mean', 'size'],\n        'purchase_date': [np.ptp],\n        'month_lag': ['min', 'max']\n        }\n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['hist_' + '_'.join(col).strip() \n                           for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='hist_transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history\n\nhistory = aggregate_historical_transactions(ht)\ndel ht\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b27c768eb5df51ebfcac18a79054b0a4ae91798"},"cell_type":"code","source":"history.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a41c9995ee200f47d201d0b2ba3f3250752b16bf"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32ac1b0e87e1a9b68ed7961d4c20d9071f237d84"},"cell_type":"markdown","source":"**Merge to train and test**"},{"metadata":{"trusted":true,"_uuid":"09c7a506fe5aa912140426cb8c7d1b39544c9d5c"},"cell_type":"code","source":"train = pd.merge(train, history, on='card_id', how='left')\ntest = pd.merge(test, history, on='card_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c56bc18fcc1a02239105bfe8b9bfaa5a40dbe718"},"cell_type":"markdown","source":"## Merchants Preprocessing"},{"metadata":{"trusted":true,"_uuid":"5e6eb59464003f534a47fe00c37a85ab6ae6f7c2"},"cell_type":"code","source":"merchant = pd.read_csv(\"../input/merchants.csv\")\nprint(\"shape of merchant: \", merchant.shape)\n\nnew_merchant = pd.read_csv(\"../input/new_merchant_transactions.csv\")\nprint(\"shape of new_merchant_transactions: \", new_merchant.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af9ee0cc8b6e83331f0cec5c7260cd74c796bac8"},"cell_type":"code","source":"new_merchant['authorized_flag'] = new_merchant['authorized_flag'].map({'Y':1, 'N':0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2333029b3002e253be07e3afcf3a46f5c49cd38"},"cell_type":"code","source":"# Do impute missing values for merchant and new_merchant\nfor df in [merchant, new_merchant]:\n    missing_impute(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f68c2ac6a88c36839f1a8ebbe2ab3a2eacff25b"},"cell_type":"code","source":"def aggregate_new_transactions(new_trans):    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'merchant_id': ['nunique'],\n        'city_id': ['nunique'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std', 'mean', 'size'],\n        'installments': ['sum', 'median', 'max', 'min', 'std', 'mean', 'size'],\n        'month_lag': ['min', 'max']\n        }\n    agg_new_trans = new_trans.groupby(['card_id']).agg(agg_func)\n    agg_new_trans.columns = ['new_' + '_'.join(col).strip() \n                           for col in agg_new_trans.columns.values]\n    agg_new_trans.reset_index(inplace=True)\n    \n    df = (new_trans.groupby('card_id')\n          .size()\n          .reset_index(name='new_transactions_count'))\n    \n    agg_new_trans = pd.merge(df, agg_new_trans, on='card_id', how='left')\n    \n    return agg_new_trans\n\nnew_trans = aggregate_new_transactions(new_merchant)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0d972a767c54b2d184b8d2ec834121d545136a7"},"cell_type":"markdown","source":"**Merge to train and test**"},{"metadata":{"trusted":true,"_uuid":"eaf000ea7e25a268da71d9a37aa81faef2db6a25"},"cell_type":"code","source":"train = pd.merge(train, new_trans, on='card_id', how='left')\ntest = pd.merge(test, new_trans, on='card_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d1e8af83b210ca042589ce2d02dc4c2bc9f00a8"},"cell_type":"markdown","source":"# Target"},{"metadata":{"trusted":true,"_uuid":"78884efdadd618daf27fec7eed7a74dfd8b5d213"},"cell_type":"code","source":"# The target\ntarget = train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"440c43b51701d6a53bccaaa603da19c9342cee56"},"cell_type":"markdown","source":"# One Hot Encoding"},{"metadata":{"trusted":true,"_uuid":"9333538f49d8b41d6e7e6f74d8ecca4582094485"},"cell_type":"code","source":"excluded_features = ['first_active_month', 'card_id', 'target', 'date']\nuse_cols = [col for col in train.columns if col not in excluded_features]\n\ntrain = train[use_cols]\ntest = test[use_cols]\n\nfeatures = list(train[use_cols].columns)\ncategorical_feats = [col for col in features if 'feature_' in col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3004db447d59a76a7e3eaa5f9d548e74ea86be65"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfor col in categorical_feats:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n    train[col] = lbl.transform(list(train[col].values.astype('str')))\n    test[col] = lbl.transform(list(test[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d3b851ad0f65ea477f470577cfc11cd8a33b7a4"},"cell_type":"code","source":"df_all = pd.concat([train, test])\ndf_all = pd.get_dummies(df_all, columns=categorical_feats)\n\nlen_train = train.shape[0]\n\ntrain = df_all[:len_train]\ntest = df_all[len_train:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06c477e49aa5a17a35d943068e3a9a23b14970c6"},"cell_type":"code","source":"# Check missing again\ntrain.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e450747e460e0bd3af561d2232b33adb9fa224d"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65bf6a6abc65175d6b94427e1d85e2c88671b792"},"cell_type":"code","source":"# Final fill missing values\nfor col in train.columns:\n    for df in [train, test]:\n        if df[col].dtype == \"float64\":\n            print(col)\n            df[col] = df[col].fillna(df[col].mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba9e6e4c52fd8b272a984dec398a54284fbc61ea"},"cell_type":"markdown","source":"# Training"},{"metadata":{"_uuid":"e3445c1f8ae7d03b4856ac636ed538187f0b7032"},"cell_type":"markdown","source":"## LightGBM"},{"metadata":{"trusted":true,"_uuid":"19f4ea36993c0f3ad14e92c61b64f75fb8ae41e1"},"cell_type":"code","source":"lgb_params = {\"objective\" : \"regression\", \"metric\" : \"rmse\", \n               \"max_depth\": 9, \"min_child_samples\": 20, \n               \"reg_alpha\": 1, \"reg_lambda\": 1,\n               \"num_leaves\" : 64, \"learning_rate\" : 0.005, \n               \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \n               \"verbosity\": -1}\n\nFOLDs = KFold(n_splits=10, shuffle=True, random_state=100)\n\noof_lgb = np.zeros(len(train))\npredictions_lgb = np.zeros(len(test))\n\nfeatures_lgb = list(train.columns)\nfeature_importance_df_lgb = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = lgb.Dataset(train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx], label=target.iloc[val_idx])\n\n    print(\"LGB \" + str(fold_) + \"-\" * 50)\n    num_round = 10000\n    clf = lgb.train(lgb_params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 50)\n    oof_lgb[val_idx] = clf.predict(train.iloc[val_idx], num_iteration=clf.best_iteration)\n\n    fold_importance_df_lgb = pd.DataFrame()\n    fold_importance_df_lgb[\"feature\"] = features_lgb\n    fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n    fold_importance_df_lgb[\"fold\"] = fold_ + 1\n    feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n    predictions_lgb += clf.predict(test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n    \nprint(np.sqrt(mean_squared_error(oof_lgb, target)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df3572e7851792c0e6668619e76660ac04e0092c"},"cell_type":"code","source":"cols = (feature_importance_df_lgb[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\n\nplt.figure(figsize=(14,14))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d745b785d7295d3ed5145ebaa5b22e657ac9e75"},"cell_type":"markdown","source":"## CatBoost"},{"metadata":{"trusted":true,"_uuid":"ffd8bd4c59d03656b7779ae780eb8c9da22697b4"},"cell_type":"code","source":"FOLDs = KFold(n_splits=10, shuffle=True, random_state=100)\nX = train\ny = target\n\noof_cb = np.zeros(len(train))\npredictions_cb = np.zeros(len(test))\n\nfor n_fold, (trn_idx, val_idx) in enumerate(FOLDs.split(X, y)):\n    X_train, y_train = X.iloc[trn_idx], y.iloc[trn_idx]\n    X_valid, y_valid = X.iloc[val_idx], y.iloc[val_idx]\n    \n    # CatBoost Regressor estimator\n    model = cb.CatBoostRegressor(\n        learning_rate = 0.03,\n        iterations = 1000,\n        eval_metric = 'RMSE',\n        allow_writing_files = False,\n        od_type = 'Iter',\n        bagging_temperature = 0.2,\n        depth = 10,\n        od_wait = 20,\n        silent = True\n    )\n    \n    # Fit\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_train, y_train), (X_valid, y_valid)],\n        verbose=None,\n        early_stopping_rounds=100\n    )\n    \n    print(\"CB \" + str(n_fold) + \"-\" * 50)\n    \n    oof_cb[val_idx] = model.predict(X_valid)\n    test_preds = model.predict(test)\n    predictions_cb += test_preds / FOLDs.n_splits\n\nprint(np.sqrt(mean_squared_error(oof_cb, target)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4fb06014abd450c20f9d1519063d22718c72e32"},"cell_type":"markdown","source":"## XGBoost"},{"metadata":{"trusted":true,"_uuid":"e52c4d670c1103587cb96da9f05574760e4482ef"},"cell_type":"code","source":"xgb_params = {'eta': 0.005, 'max_depth': 9, 'subsample': 0.8, 'colsample_bytree': 0.8, \n          'objective': 'reg:linear', 'eval_metric': 'rmse', 'silent': True}\n\nFOLDs = KFold(n_splits=10, shuffle=True, random_state=100)\n\noof_xgb = np.zeros(len(train))\npredictions_xgb = np.zeros(len(test))\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = xgb.DMatrix(data=train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = xgb.DMatrix(data=train.iloc[val_idx], label=target.iloc[val_idx])\n    watchlist = [(trn_data, 'train'), (val_data, 'valid')]\n    print(\"XGB \" + str(fold_) + \"-\" * 50)\n    num_round = 10000\n    xgb_model = xgb.train(xgb_params, trn_data, num_round, watchlist, early_stopping_rounds=50, verbose_eval=1000)\n    oof_xgb[val_idx] = xgb_model.predict(xgb.DMatrix(train.iloc[val_idx]), ntree_limit=xgb_model.best_ntree_limit+50)\n\n    predictions_xgb += xgb_model.predict(xgb.DMatrix(test), ntree_limit=xgb_model.best_ntree_limit+50) / FOLDs.n_splits\n\nprint(np.sqrt(mean_squared_error(oof_xgb, target)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"585fb8b3eb3b3201fa65db3786c00bd43e281766"},"cell_type":"markdown","source":"# Ensembling"},{"metadata":{"trusted":true,"_uuid":"02d212e80c89e860be63bb03661bdffa8461cfc9"},"cell_type":"code","source":"def find_best_weight(preds, target):\n    def _validate_func(weights):\n        ''' scipy minimize will pass the weights as a numpy array '''\n        final_prediction = 0\n        for weight, prediction in zip(weights, preds):\n                final_prediction += weight * prediction\n        return np.sqrt(mean_squared_error(final_prediction, target))\n\n    #the algorithms need a starting value, right not we chose 0.5 for all weights\n    #its better to choose many random starting points and run minimize a few times\n    starting_values = [0.5]*len(preds)\n\n    #adding constraints and a different solver as suggested by user 16universe\n    #https://kaggle2.blob.core.windows.net/forum-message-attachments/75655/2393/otto%20model%20weights.pdf?sv=2012-02-12&se=2015-05-03T21%3A22%3A17Z&sr=b&sp=r&sig=rkeA7EJC%2BiQ%2FJ%2BcMpcA4lYQLFh6ubNqs2XAkGtFsAv0%3D\n    cons = ({'type':'eq','fun':lambda w: 1-sum(w)})\n    #our weights are bound between 0 and 1\n    bounds = [(0, 1)] * len(preds)\n    \n    res = minimize(_validate_func, starting_values, method='Nelder-Mead', bounds=bounds, constraints=cons)\n    \n    print('Ensemble Score: {best_score}'.format(best_score=(1-res['fun'])))\n    print('Best Weights: {weights}'.format(weights=res['x']))\n    \n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73d6f31fabe55baab2cf971576f89f09c5695d72"},"cell_type":"code","source":"print('lgb', np.sqrt(mean_squared_error(oof_lgb, target)))\nprint('xgb', np.sqrt(mean_squared_error(oof_xgb, target)))\nprint('cb', np.sqrt(mean_squared_error(oof_cb, target)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebfe900014e783cb968d8b7da165862a0b925026"},"cell_type":"code","source":"res = find_best_weight([oof_lgb, oof_cb, oof_xgb], target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99480590364393b90a9a547193442a89c3b74226"},"cell_type":"code","source":"total_sum = 0.35864667 * oof_lgb + 0.59360413 * oof_cb + 0.14343413 * oof_xgb\nprint(\"CV score: {:<8.5f}\".format(np.sqrt(mean_squared_error(total_sum, target))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34a0f1dc9286109e021b320fd30a4aa88f64cf1a"},"cell_type":"code","source":"sub_df = pd.read_csv('../input/sample_submission.csv')\nsub_df[\"target\"] = 0.35864667 * predictions_lgb + 0.59360413 * predictions_cb + 0.14343413 * predictions_xgb\nsub_df.to_csv(\"submission_ensemble.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}