{"cells":[{"metadata":{"_uuid":"1ebfdd1d0b83e3f0a56ba79ed5f56b9b9c59c90e"},"cell_type":"markdown","source":"# I referred a very helpful kernel. Thanks!\n# This is  my first attemp in kaggle, It is use in  learning and exercising.\n# A few weeks later, I will submit my own mode and result. \n# Thanks again,Best Regards.\n-https://www.kaggle.com/youhanlee/hello-elo-ensemble-will-help-you\n-https://www.kaggle.com/fabiendaniel/elo-world"},{"metadata":{"_uuid":"24b34f7ceb08c676b2e2567feed962b95864f9c3"},"cell_type":"markdown","source":"- This kernel shows \n- how to use 'new_merchant_transactions.csv' \n- how to use lgb and xgb\n- how to ensemble them."},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T18:56:52.172046Z","start_time":"2018-11-27T18:56:48.781792Z"},"trusted":true,"_uuid":"404cf8e0e81d03dae437ff45fbcad13b65481bd4","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nimport warnings\nimport time\nimport sys\nimport datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc\nimport os\nfrom sklearn.metrics import mean_squared_error\nwarnings.simplefilter(action='ignore', category=FutureWarning)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4965e9fcd356569f7031cadc8b078f39075bf164"},"cell_type":"markdown","source":"# New transactions"},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T18:56:55.549404Z","start_time":"2018-11-27T18:56:52.176437Z"},"trusted":true,"_uuid":"d3ae8e3ebefe121750ab7cbddcdf41d44541abae","_kg_hide-output":false},"cell_type":"code","source":"new_transactions = pd.read_csv('../input/new_merchant_transactions.csv')\nnew_transactions.head(5)\n# new_transactions.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a8f2cf64ab1b6dae57cf98dbe6182c5e4d9d51a"},"cell_type":"code","source":"new_transactions['authorized_flag'] = new_transactions['authorized_flag'].map({'Y':1, 'N':0})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abab1037a323ec997d93bc3804356e73bcb797bb"},"cell_type":"code","source":"def aggregate_new_transactions(new_trans):    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'merchant_id': ['nunique'],\n        'city_id': ['nunique'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std'],\n        'installments': ['sum', 'median', 'max', 'min', 'std'],\n        'month_lag': ['min', 'max']\n        }\n    agg_new_trans = new_trans.groupby(['card_id']).agg(agg_func)\n    print(agg_new_trans.columns.values)\n    agg_new_trans.columns = ['new_' + '_'.join(col).strip() \n                           for col in agg_new_trans.columns.values]\n    agg_new_trans.reset_index(inplace=True)\n    df = (new_trans.groupby('card_id').size().reset_index(name='new_transactions_count'))\n    agg_new_trans = pd.merge(df, agg_new_trans, on='card_id', how='left')    \n    return agg_new_trans\n\nnew_trans = aggregate_new_transactions(new_transactions)\nnew_trans[:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c1cffef68d8d0db830abfcd840729ff171f63f37"},"cell_type":"markdown","source":"# historical_transactions"},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T18:57:36.972521Z","start_time":"2018-11-27T18:56:55.550626Z"},"trusted":true,"_uuid":"7734c0148976a809abbb56b6265847412734e4b2"},"cell_type":"code","source":"gc.collect()\nhistorical_transactions = pd.read_csv('../input/historical_transactions.csv')\nhistorical_transactions[:5]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T18:57:39.225225Z","start_time":"2018-11-27T18:57:36.974283Z"},"trusted":true,"_uuid":"38295bc1cf5b69945edd8d9a855c42b3e3797652"},"cell_type":"code","source":"historical_transactions['authorized_flag'] = \\\n    historical_transactions['authorized_flag'].map({'Y':1, 'N':0})","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T18:58:58.888537Z","start_time":"2018-11-27T18:57:44.814944Z"},"trusted":true,"_uuid":"2cafa82dbcd8c91eab57fcc69118fa1409d48f74"},"cell_type":"code","source":"def aggregate_historical_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'merchant_id': ['nunique'],\n        'city_id': ['nunique'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std'],\n        'installments': ['sum', 'median', 'max', 'min', 'std'],\n        'purchase_date': [np.ptp],\n        'month_lag': ['min', 'max']\n        }\n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['hist_' + '_'.join(col).strip() \n                           for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='hist_transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history\n\nhistory = aggregate_historical_transactions(historical_transactions)\nhistory[:10]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T19:03:15.661311Z","start_time":"2018-11-27T19:03:14.532531Z"},"trusted":true,"_uuid":"a54b8417e8b56eb08600534a7b98a98d41ae7787"},"cell_type":"code","source":"def read_data(input_file):\n    df = pd.read_csv(input_file)\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['year'] = df['first_active_month'].dt.year\n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n    return df\ntrain = read_data('../input/train.csv')\ntest = read_data('../input/test.csv')\n\ntarget = train['target']\ndel train['target']\ntrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17eff52e6db77920e32d359627a87b7175c9ad5d"},"cell_type":"markdown","source":"# Merge historical transactions and new transactions"},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T19:03:16.055593Z","start_time":"2018-11-27T19:03:15.662818Z"},"trusted":true,"_uuid":"42e32c0e29e9e254413d0409ef621d72e983c793"},"cell_type":"code","source":"train = pd.merge(train, history, on='card_id', how='left')\ntest = pd.merge(test, history, on='card_id', how='left')\n\ntrain = pd.merge(train, new_trans, on='card_id', how='left')\ntest = pd.merge(test, new_trans, on='card_id', how='left')\ntrain.shape\ntrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf55353fac4af3e955764b9e49cf199aefe2bbda"},"cell_type":"markdown","source":"# Feature preparation"},{"metadata":{"trusted":true,"_uuid":"d2ab2d2b5490009e6964d6153c8d12fd6fe2e9d9"},"cell_type":"code","source":"use_cols = [col for col in train.columns if col not in ['card_id', 'first_active_month']]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T19:03:18.505579Z","start_time":"2018-11-27T19:03:18.459705Z"},"trusted":true,"_uuid":"184512a362b4a97e613d15cffe92047c08f3ea84"},"cell_type":"code","source":"# 选出特征列\ntrain = train[use_cols]\ntest = test[use_cols]","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T19:03:20.042059Z","start_time":"2018-11-27T19:03:20.027829Z"},"trusted":true,"_uuid":"6121ac57657156ec5af1bb6a1437aa0c37995a61"},"cell_type":"code","source":"# features = list(train[use_cols].columns)\n# features ==use_cols\ncategorical_feats = [col for col in use_cols if 'feature_' in col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88acdb59ccd3281257d76dd927da9ffa975949e4"},"cell_type":"code","source":"for col in categorical_feats:\n    print(col, 'have', train[col].value_counts().shape[0], 'categories.')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"824b524f12e5478ae3a8d6494186c08e86fcd6b6"},"cell_type":"markdown","source":"- 3 categorical features have not many categories. For now, we use one-hot encoding."},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T19:03:22.637187Z","start_time":"2018-11-27T19:03:21.131062Z"},"trusted":true,"_uuid":"f55deab3f9bcefcbe02a2164787174dd4912614a"},"cell_type":"code","source":"# make 3 categorical features to one hot encoding\nfrom sklearn.preprocessing import LabelEncoder\nfor col in categorical_feats:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n    train[col] = lbl.transform(list(train[col].values.astype('str')))\n    test[col] = lbl.transform(list(test[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T19:03:22.740006Z","start_time":"2018-11-27T19:03:22.638673Z"},"trusted":true,"_uuid":"24bcf848cccca0d6b16abec43a1d27d029a0481c"},"cell_type":"code","source":"df_all = pd.concat([train, test])\ndf_all = pd.get_dummies(df_all, columns=categorical_feats)\nlen_train = train.shape[0]\ntrain = df_all[:len_train]\ntest = df_all[len_train:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08efa8c80e1504a4f42ee9f99101b7055ae190eb"},"cell_type":"markdown","source":"# You know that ensemble is always answer."},{"metadata":{"_uuid":"aaabf65d72f4eb936c7a87774ef33202005540b5"},"cell_type":"markdown","source":"# LGB"},{"metadata":{"ExecuteTime":{"end_time":"2018-11-27T19:12:11.199094Z","start_time":"2018-11-27T19:05:17.422791Z"},"trusted":true,"scrolled":true,"_uuid":"ea646382df220eed41b31955a8f8abcc242f7915"},"cell_type":"code","source":"lgb_params = {\"objective\" : \"regression\", \"metric\" : \"rmse\", \n               \"max_depth\": 7, \"min_child_samples\": 20, \n               \"reg_alpha\": 1, \"reg_lambda\": 1,\n               \"num_leaves\" : 64, \"learning_rate\" : 0.001, \n               \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \n               \"verbosity\": -1}\n# k fold \nFOLDs = KFold(n_splits=5, shuffle=True, random_state=1989) \n\noof_lgb = np.zeros(len(train))\npredictions_lgb = np.zeros(len(test))\n\nfeatures_lgb = list(train.columns)\nfeature_importance_df_lgb = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = lgb.Dataset(train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx], label=target.iloc[val_idx])\n\n    print(\"LGB \" + str(fold_) + \"-\" * 50)\n    num_round = 2000\n    clf = lgb.train(lgb_params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=200, early_stopping_rounds = 2000)\n    oof_lgb[val_idx] = clf.predict(train.iloc[val_idx], num_iteration=clf.best_iteration)\n\n    fold_importance_df_lgb = pd.DataFrame()\n    fold_importance_df_lgb[\"feature\"] = features_lgb\n    fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n    fold_importance_df_lgb[\"fold\"] = fold_ + 1\n    feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n    predictions_lgb += clf.predict(test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n    \n\nprint(np.sqrt(mean_squared_error(oof_lgb, target)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"198a1f77fa99754ef2b53d5bf3a11f84ee855b08"},"cell_type":"markdown","source":"# XGB"},{"metadata":{"ExecuteTime":{"start_time":"2018-11-27T19:13:04.678Z"},"trusted":true,"_uuid":"48f17e6cae4bf8823bf6a18fd21f82695f106997"},"cell_type":"code","source":"import xgboost as xgb\n\nxgb_params = {'eta': 0.001, 'max_depth': 7, 'subsample': 0.8, 'colsample_bytree': 0.8, \n          'objective': 'reg:linear', 'eval_metric': 'rmse', 'silent': True}\n\n\n\nFOLDs = KFold(n_splits=2, shuffle=True, random_state=1989)\n\noof_xgb = np.zeros(len(train))\npredictions_xgb = np.zeros(len(test))\n\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = xgb.DMatrix(data=train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = xgb.DMatrix(data=train.iloc[val_idx], label=target.iloc[val_idx])\n    watchlist = [(trn_data, 'train'), (val_data, 'valid')]\n    print(\"xgb \" + str(fold_) + \"-\" * 50)\n    num_round = 2000\n    xgb_model = xgb.train(xgb_params, trn_data, num_round, watchlist, early_stopping_rounds=100, verbose_eval=200)\n    oof_xgb[val_idx] = xgb_model.predict(xgb.DMatrix(train.iloc[val_idx]), ntree_limit=xgb_model.best_ntree_limit+50)\n\n    predictions_xgb += xgb_model.predict(xgb.DMatrix(test), ntree_limit=xgb_model.best_ntree_limit+50) / FOLDs.n_splits\n\nnp.sqrt(mean_squared_error(oof_xgb, target))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b959baeb6b995adc039ad81d49437b37363e4d0"},"cell_type":"code","source":"print('lgb', np.sqrt(mean_squared_error(oof_lgb, target)))\nprint('xgb', np.sqrt(mean_squared_error(oof_xgb, target)))\ntotal_sum = 0.5 * oof_lgb + 0.5 * oof_xgb\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(total_sum, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8cf16263c0da352f0c2b514dd3b8163f650f29f"},"cell_type":"code","source":"cols = (feature_importance_df_lgb[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\n\nplt.figure(figsize=(14,14))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e2a7b9ef480b2246d4d1fd392420437b1c06183"},"cell_type":"code","source":"sub_df = pd.read_csv('../input/sample_submission.csv')\nsub_df[\"target\"] = 0.5 * predictions_lgb + 0.5 * predictions_xgb\nsub_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2481f0210e50fa05482b9777833cb05c6df4b5ba"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"df4b05483540a97d06e5b4266f391a07f64e34fc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false}},"nbformat":4,"nbformat_minor":1}