{"cells":[{"metadata":{"_uuid":"6df55b542f152882e00385a0f73198f4e3bc4316"},"cell_type":"markdown","source":"**FEEL FREE TO UPVOTE**  （＾ｖ＾）"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nimport warnings\nfrom sklearn.model_selection import GroupKFold\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\nfrom catboost import CatBoostClassifier\nfrom sklearn import tree\nimport xgboost\nfrom sklearn.metrics import mean_squared_error, roc_auc_score, log_loss\nwarnings.filterwarnings('ignore')\nnp.random.seed(4590)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"11105afdfcf5203ce25b6dbdf1c0eede5e514423","trusted":true},"cell_type":"code","source":"debug=4","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4284125ff23f6e6019c17e16923adfc265228ea4","trusted":true},"cell_type":"code","source":"if debug == 1:\n    nrows = 50000\n    print(\"本次程序运行是测试debug用\")\nelse:\n    nrows = None\n    print(\"本次程序运行是运行数据模型\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv',nrows=nrows)\ndf_test = pd.read_csv('../input/test.csv',nrows=nrows)\ndf_hist_trans = pd.read_csv('../input/historical_transactions.csv',nrows=nrows)\ndf_new_merchant_trans = pd.read_csv('../input/new_merchant_transactions.csv',nrows=nrows)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"795eff5c5de4b764a38318dc20097409a92c20a9","trusted":true},"cell_type":"code","source":"for df in [df_train,df_test,df_hist_trans,df_new_merchant_trans]:\n    print(df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"71f89a3b8a93b2f2feb2cd0a45f860cde33687be","trusted":true},"cell_type":"code","source":"for df in [df_hist_trans,df_new_merchant_trans]:\n    df['category_2'].fillna(1.0,inplace=True)\n    df['category_3'].fillna('A',inplace=True)\n    df['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dda90662d05e22310dd713df106ea07f4b8bccfc","trusted":true},"cell_type":"code","source":"def get_new_columns(name,aggs):\n    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"690ba01a38f524e9345b419200f588f937bc067a","trusted":true},"cell_type":"code","source":"for df in [df_hist_trans,df_new_merchant_trans]:\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n    df['category_1'] = df['category_1'].map({'Y':1, 'N':0}) \n    #https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73244\n    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddf1d5bb0ade2b22b0f072c208c1506ea64503ea","trusted":true},"cell_type":"code","source":"%%time\naggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\n\naggs['purchase_amount'] = ['sum','max','min','mean','var','count']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['authorized_flag'] = ['sum', 'mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    df_hist_trans[col+'_mean'] = df_hist_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']    \n\nnew_columns = get_new_columns('hist',aggs)\ndf_hist_trans_group = df_hist_trans.groupby('card_id').agg(aggs)\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\ndf_hist_trans_group['hist_purchase_date_diff'] = (df_hist_trans_group['hist_purchase_date_max'] - df_hist_trans_group['hist_purchase_date_min']).dt.days\ndf_hist_trans_group['hist_purchase_date_average'] = df_hist_trans_group['hist_purchase_date_diff']/df_hist_trans_group['hist_card_id_size']\ndf_hist_trans_group['hist_purchase_date_uptonow'] = (datetime.datetime.today() - df_hist_trans_group['hist_purchase_date_max']).dt.days\ndf_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\ndf_test = df_test.merge(df_hist_trans_group,on='card_id',how='left')\ndel df_hist_trans_group;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7f5625db40db4395374991124fb796c9decd60b","trusted":true},"cell_type":"code","source":"%%time\naggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    df_new_merchant_trans[col+'_mean'] = df_new_merchant_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']\n    \nnew_columns = get_new_columns('new_hist',aggs)\ndf_hist_trans_group = df_new_merchant_trans.groupby('card_id').agg(aggs)\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\ndf_hist_trans_group['new_hist_purchase_date_diff'] = (df_hist_trans_group['new_hist_purchase_date_max'] - df_hist_trans_group['new_hist_purchase_date_min']).dt.days\ndf_hist_trans_group['new_hist_purchase_date_average'] = df_hist_trans_group['new_hist_purchase_date_diff']/df_hist_trans_group['new_hist_card_id_size']\ndf_hist_trans_group['new_hist_purchase_date_uptonow'] = (datetime.datetime.today() - df_hist_trans_group['new_hist_purchase_date_max']).dt.days\ndf_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\ndf_test = df_test.merge(df_hist_trans_group,on='card_id',how='left')\ndel df_hist_trans_group;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a075cc90ab1322829e4fad3ff39fce307c5db93c","trusted":true},"cell_type":"code","source":"del df_hist_trans;gc.collect()\ndel df_new_merchant_trans;gc.collect()\ndf_train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6c277ac1ff5b873703f0f1a10b344bd7bd70402d","trusted":true},"cell_type":"code","source":"%%time\ndf_train['outliers'] = df_train.target.apply(lambda x: 1 if x<=-30 else 0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8735d0dd345f6ae0dd04e6233ecb8d944d00643","trusted":true},"cell_type":"code","source":"df_train['outliers'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce2082fc1fb0e3f8f7d27fc166aa7a8351b65504","trusted":true},"cell_type":"code","source":"%%time\nfor df in [df_train,df_test]:\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['dayofweek'] = df['first_active_month'].dt.dayofweek\n    df['weekofyear'] = df['first_active_month'].dt.weekofyear\n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days\n    df['new_hist_first_buy'] = (df['new_hist_purchase_date_min'] - df['first_active_month']).dt.days\n    for f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\\\n                     'new_hist_purchase_date_min']:\n        df[f] = df[f].astype(np.int64) * 1e-9\n    df['card_id_total'] = df['new_hist_card_id_size']+df['hist_card_id_size']\n    df['purchase_amount_total'] = df['new_hist_purchase_amount_sum']+df['hist_purchase_amount_sum']\n\nfor f in ['feature_1','feature_2','feature_3']:\n    order_label = df_train.groupby([f])['outliers'].mean()\n    df_train[f] = df_train[f].map(order_label)\n    df_test[f] = df_test[f].map(order_label)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54a8a7906ad6f12d1fc768d4b693eb00ed7fff18"},"cell_type":"markdown","source":"## 1、添加二分类的训练\n"},{"metadata":{"_uuid":"a68908882c1d3a5d2eff1f53c85d5c9f19f2d642","trusted":true},"cell_type":"code","source":"df_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers','outliers2']]\ny_clf = df_train['outliers']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb3f2460e309d5f2ba09634038d98a89796676ac"},"cell_type":"markdown","source":"## 1.1Bayes 优化代码"},{"metadata":{"trusted":true,"_uuid":"f5c70659dcd20321f1ef6d09886d0a9de7c0fc0d"},"cell_type":"markdown","source":"from bayes_opt import BayesianOptimization\nX = df_train[df_train_columns]\ny = y_clf"},{"metadata":{"trusted":true,"_uuid":"55c8838d77d3dbe72e2d393c06a9e00af6ab3194"},"cell_type":"markdown","source":"#1、lightgbm\ndef bayes_parameter_opt_lgb(X, y, init_round=15, opt_round=25, n_folds=5, random_seed=0, n_estimators=10000,\n                            learning_rate=0.002, output_process=True):\n    # prepare data\n    train_data = lgb.Dataset(data=X, label=y, free_raw_data=False)\n\n    # parameters\n    def lgb_eval(num_leaves, max_depth, lambda_l1, lambda_l2, min_split_gain,\n                 min_child_weight):\n        params = {'application': 'binary', 'num_iterations': n_estimators, 'learning_rate': learning_rate, \n                  'early_stopping_round': 200, 'metric': 'auc'}\n        params[\"nthread\"] = 4\n        params[\"num_leaves\"] = int(round(num_leaves))\n        #params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n        #params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n        params['max_depth'] = int(round(max_depth))\n        params['lambda_l1'] = max(lambda_l1, 0)\n        params['lambda_l2'] = max(lambda_l2, 0)\n        params['min_split_gain'] = min_split_gain\n        params['min_child_weight'] = min_child_weight\n        params[\"silent\"] = -1\n        params[\"verbose\"] = -1\n        params[\"unbalance\"] = unbalance\n        cv_result = lgb.cv(params, train_data, nfold=n_folds, seed=random_seed, verbose_eval=200,\n                           metrics=['auc'])\n        return max(cv_result['auc-mean'])\n\n    # range\n    lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (12, 36),\n                                            #'feature_fraction': (0.01, 0.5),\n                                            #'bagging_fraction': (0.9, 1),\n                                            'max_depth': (5, 45),\n                                            'lambda_l1': (0, 5),\n                                            'lambda_l2': (0, 3),\n                                            'min_split_gain': (0.001, 0.1),\n                                            'unbalance':[True,False],\n                                            'min_child_weight': (5, 50)}, random_state=0)\n    # optimize\n    lgbBO.maximize(init_points=init_round, n_iter=opt_round)\n\n    # output optimization process\n    if output_process == True: lgbBO.points_to_csv(\"bayes_opt_result_lgb.csv\")\n\n    # return best parameters\n    return lgbBO.res['max']['max_params']\n\nlgb_opt_params = bayes_parameter_opt_lgb(X, y, init_round=5, opt_round=10, n_folds=5, random_seed=0, n_estimators=10000,\n                                     learning_rate=0.02)\nlgb_opt_params"},{"metadata":{"trusted":true,"_uuid":"560d7369c7c1a2557e5a9fa6c106e7b0b781b578"},"cell_type":"markdown","source":"#2、xgboost\ndef bayes_parameter_opt_xgb(X, y, init_round=15, opt_round=25, n_folds=5, random_seed=0, n_estimators=10000,\n                            learning_rate=0.004, output_process=True):\n    # prepare data\n    train_data = xgb.Dataset(data=X, label=y, free_raw_data=False)\n\n    # parameters\n    def xgb_eval(num_leaves, max_depth, lambda_l1, lambda_l2, min_split_gain,\n                 min_child_weight):\n        params = {'application': 'binary:logistic', 'num_iterations': n_estimators, 'learning_rate': learning_rate, \n                  'early_stopping_round': 200, 'metric': 'auc'}\n        params[\"nthread\"] = 4\n        params[\"num_leaves\"] = int(round(num_leaves))\n        params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n        params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n        params['max_depth'] = int(round(max_depth))\n        params['lambda_l1'] = max(lambda_l1, 0)\n        params['lambda_l2'] = max(lambda_l2, 0)\n        params['min_split_gain'] = min_split_gain\n        params['min_child_weight'] = min_child_weight\n        params[\"silent\"] = -1\n        params[\"verbose\"] = -1\n        params[\"unbalance\"] = unbalance\n        cv_result = xgb.cv(params, train_data, nfold=n_folds, seed=random_seed, verbose_eval=200,\n                           metrics=['rmse'])\n        return max(cv_result['rmse-mean'])\n\n    # range\n    xgbBO = BayesianOptimization(xgb_eval, {'num_leaves': (24, 45),\n                                            'feature_fraction': (0.1, 0.9),\n                                            'bagging_fraction': (0.8, 1),\n                                            'max_depth': (5, 45),\n                                            'lambda_l1': (0, 5),\n                                            'lambda_l2': (0, 3),\n                                            'min_split_gain': (0.001, 0.1),\n                                            'unbalance':[True,False],\n                                            'min_child_weight': (5, 50)}, random_state=0)\n    # optimize\n    xgbBO.maximize(init_points=init_round, n_iter=opt_round)\n\n    # output optimization process\n    if output_process == True: xgbBO.points_to_csv(\"bayes_opt_result_xgb.csv\")\n\n    # return best parameters\n    return xgbBO.res['max']['max_params']\n\nxgb_opt_params = bayes_parameter_opt_xgb(X, y, init_round=5, opt_round=10, n_folds=5, random_seed=0, n_estimators=10000,\n                                     learning_rate=0.02)\nxgb_opt_params"},{"metadata":{"trusted":true,"_uuid":"974ecbfe3c0045c62ba4b3e53c2ff1e394206f51"},"cell_type":"markdown","source":"#3、catboost\ndef bayes_parameter_opt_cat(X, y, init_round=15, opt_round=25, n_folds=5, random_seed=0, n_estimators=10000,\n                            learning_rate=0.1, output_process=True):\n    # prepare data\n    train_data = cat.Dataset(data=X, label=y, free_raw_data=False)\n\n    # parameters\n    def cat_eval(num_leaves, max_depth, lambda_l1, lambda_l2, min_split_gain,\n                 min_child_weight):\n        params = {'application': 'auc', 'num_iterations': n_estimators, 'learning_rate': learning_rate, \n                  'early_stopping_round': 200, 'metric': 'AUC'}\n        params[\"nthread\"] = 4\n        params[\"num_leaves\"] = int(round(num_leaves))\n        params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n        params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n        params['max_depth'] = int(round(max_depth))\n        params['lambda_l1'] = max(lambda_l1, 0)\n        params['lambda_l2'] = max(lambda_l2, 0)\n        params['min_split_gain'] = min_split_gain\n        params['min_child_weight'] = min_child_weight\n        params[\"silent\"] = -1\n        params[\"verbose\"] = -1\n        cv_result = cat.cv(params, train_data, nfold=n_folds, seed=random_seed, verbose_eval=200,\n                           metrics=['rmse'])\n        return max(cv_result['rmse-mean'])\n\n    # range\n    catBO = BayesianOptimization(cat_eval, {'num_leaves': (24, 45),\n                                            'feature_fraction': (0.1, 0.9),\n                                            'bagging_fraction': (0.8, 1),\n                                            'max_depth': (5, 45),\n                                            'lambda_l1': (0, 5),\n                                            'lambda_l2': (0, 3),\n                                            'min_split_gain': (0.001, 0.1),\n                                            'min_child_weight': (5, 50)}, random_state=0)\n    # optimize\n    catBO.maximize(init_points=init_round, n_iter=opt_round)\n\n    # output optimization process\n    if output_process == True: catBO.points_to_csv(\"bayes_opt_result_cat.csv\")\n\n    # return best parameters\n    return catBO.res['max']['max_params']\n\ncat_opt_params = bayes_parameter_opt_cat(X, y, init_round=5, opt_round=10, n_folds=5, random_seed=0, n_estimators=10000,\n                                     learning_rate=0.02)\ncat_opt_params"},{"metadata":{"_uuid":"df2a005370fb834bf40377285b5ffa738932ccd5","trusted":true,"scrolled":true},"cell_type":"code","source":"%%time\n\nfolds = GroupKFold(n_splits=3)\n#train_features = [_f for _f in train.columns if _f not in excluded_clf_features]\ntrain_features = df_train_columns\ntrain = df_train\ntest = df_test\n\nprint(\"train_features数量：\",len(train_features))\nimportances = pd.DataFrame()\noof_clf_preds1 = np.zeros(train.shape[0])\noof_clf_preds11 = np.zeros(train.shape[0])\noof_clf_preds21 = np.zeros(train.shape[0])\nsub_clf_preds1 = np.zeros(test.shape[0])\nsub_clf_preds11 = np.zeros(test.shape[0])\nsub_clf_preds21 = np.zeros(test.shape[0])\nfor fold_, (trn_, val_) in enumerate(folds.split(df_train, y_clf, groups=train['card_id'])):\n    print(\"folder----------\",fold_)\n    trn_x, trn_y = train[train_features].iloc[trn_], y_clf.iloc[trn_]\n    val_x, val_y = train[train_features].iloc[val_], y_clf.iloc[val_]\n    \n    clf = lgb.LGBMClassifier(\n        objective='binary',\n        learning_rate=0.004,\n        n_estimators=50000,\n        max_depth=6,\n        eval_metric='auc',\n        lambda_l1=1.852,\n        lambda_l2=1.548,\n        child_weight=5.243,\n        split_gain=0.04221,\n        num_leaves=13,\n        random_state=1\n    )\n    print(\"-\"* 10 + \"LightGBM Training\" + \"-\"* 10)\n    clf.fit(\n        trn_x, trn_y,\n        eval_set=[(val_x, val_y)],\n        early_stopping_rounds=200,\n        verbose=200\n    )\n\n    xgb_params = {\n        'objective': 'binary:logistic',\n        'booster': 'gbtree',\n        'eta': 0.004,\n        'lambda_l1':0.4472,\n        'lambda_l2':0.08835,\n        'max_depth':44,\n        'child_weight':5.331,\n        'split_gain':0.0454,\n        'num_leaves':23,\n        'is_unbalance':True,\n        'random_state': 27\n    }\n    \n    print(\"-\"* 10 + \"Xgboost Training\" + \"-\"* 10)\n    xg = XGBClassifier(**xgb_params, n_estimators=200000)\n    xg.fit(\n        trn_x, trn_y,\n        eval_set=[(val_x, val_y)],\n        early_stopping_rounds=200,\n        verbose=200\n    )\n    print(\"-\"* 10 + \"Catboost Training\" + \"-\"* 10)\n    cat = CatBoostClassifier(iterations=20000,\n                             learning_rate=0.02,\n                             eval_metric='AUC',\n                             random_seed = 42,\n                             metric_period = 50,\n                             od_wait=20)\n    cat.fit(\n        trn_x, trn_y,\n        eval_set=[(val_x, val_y)],\n        use_best_model=True,\n        early_stopping_rounds=200,\n        verbose=200)\n    \n    ###此处仅显示lgb模型训练下的importance\n    imp_lgb_df = pd.DataFrame()\n    imp_lgb_df['feature3'] = train_features\n    imp_lgb_df['gain3'] = clf.booster_.feature_importance(importance_type='gain')\n    imp_lgb_df['fold3'] = fold_ + 1\n    importances = pd.concat([importances, imp_lgb_df], axis=0, sort=False)\n    \n    oof_clf_preds1[val_] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]\n    oof_clf_preds11[val_] = xg.predict_proba(val_x)[:, 1]\n    oof_clf_preds21[val_] = cat.predict_proba(val_x)[:, 1]\n    \n    \n    print(roc_auc_score(val_y, oof_clf_preds1[val_]))\n    print(roc_auc_score(val_y, oof_clf_preds11[val_]))\n    sub_clf_preds1 += clf.predict_proba(test[train_features], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits\n    sub_clf_preds11 += xg.predict_proba(test[train_features])[:, 1] / folds.n_splits\n    sub_clf_preds21 += cat.predict_proba(test[train_features])[:, 1] / folds.n_splits\n    \nprint(\"LGB AUC模型分数：\",roc_auc_score(y_clf, oof_clf_preds1))\nprint(\"XGB AUC模型分数：\",roc_auc_score(y_clf, oof_clf_preds11))\nprint(\"CAT AUC模型分数：\",roc_auc_score(y_clf, oof_clf_preds21))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9a7439980e51510761e0b7dbb41962c50ba2fe4"},"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore', FutureWarning)\nmean_gain = importances[['gain3', 'feature3']].groupby('feature3').mean()\nimportances['mean_gain3'] = importances['feature3'].map(mean_gain['gain3'])\nplt.figure(figsize=(10, 16))\nsns.barplot(x='gain3', y='feature3', data=importances.sort_values('mean_gain3', ascending=False).iloc[:350])\nplt.savefig('lgbm_clf_importances13.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a429a99782148fb8ea2df6e78abb3394f5b7e90","trusted":true},"cell_type":"code","source":"df_train['probal_lgb'] = oof_clf_preds1\ndf_train['probal_xgb'] = oof_clf_preds11\ndf_train['probal_cat'] = oof_clf_preds21\n\ndf_test['probal_lgb'] = sub_clf_preds1\ndf_test['probal_xgb'] = sub_clf_preds11\ndf_test['probal_cat'] = sub_clf_preds21\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48227e73841c9ccb632e26990b01523b4efca804"},"cell_type":"markdown","source":"### 归一化"},{"metadata":{"trusted":true,"_uuid":"5f076b9ee906324454f9b96f57a3d2c8fae0d65d"},"cell_type":"code","source":"lgb_min = np.min(df_train['probal_lgb'].values)\nlgb_max = np.max(df_train['probal_lgb'].values)\nxgb_min = np.min(df_train['probal_xgb'].values)\nxgb_max = np.max(df_train['probal_xgb'].values)\ncat_min = np.min(df_train['probal_cat'].values)\ncat_max = np.max(df_train['probal_cat'].values)\nprint(lgb_min)\nprint(lgb_max)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"092b9734f94b9bcaa0f8b842b8867a9f0e597e86"},"cell_type":"code","source":"df_train['probal_lgb'] = df_train.probal_lgb.apply(lambda x:(x-lgb_min)/(lgb_max-lgb_min))\ndf_train['probal_xgb'] = df_train.probal_xgb.apply(lambda x:(x-xgb_min)/(xgb_max-xgb_min))\ndf_train['probal_cat'] = df_train.probal_cat.apply(lambda x:(x-cat_min)/(cat_max-cat_min))\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55bd1d2621373d0c69b845488b67704654db5af8"},"cell_type":"code","source":"df_train['lgb_xgb'] = oof_clf_preds1 + oof_clf_preds11\ndf_train['lgb_cat'] = oof_clf_preds1 + oof_clf_preds21\ndf_train['xgb_cat'] = oof_clf_preds11 + oof_clf_preds21\ndf_train['probal_combine'] = oof_clf_preds1 + oof_clf_preds11 + oof_clf_preds21\ndf_test['lgb_xgb'] = sub_clf_preds1 + sub_clf_preds11\ndf_test['lgb_cat'] = sub_clf_preds1 + sub_clf_preds21\ndf_test['xgb_cat'] = sub_clf_preds11 + sub_clf_preds21\ndf_test['probal_combine'] = sub_clf_preds1 + sub_clf_preds11 + sub_clf_preds21","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89f9c3ad7c3e7623458d9e1d4122dfaeb6b62b28"},"cell_type":"code","source":"df_train['probal_max'] = df_train[['probal_lgb','probal_xgb','probal_cat']].max(axis=1)\ndf_train['probal_min'] = df_train[['probal_lgb','probal_xgb','probal_cat']].min(axis=1)\ndf_test['probal_max'] = df_test[['probal_lgb','probal_xgb','probal_cat']].max(axis=1)\ndf_test['probal_min'] = df_test[['probal_lgb','probal_xgb','probal_cat']].min(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf4662101356509dad9791b8c12828c380020e26"},"cell_type":"code","source":"df_train['larger2'] = df_train['probal_lgb'] + df_train['probal_xgb'] + df_train['probal_cat'] - df_train['probal_min']\ndf_train['less2'] = df_train['probal_lgb'] + df_train['probal_xgb'] + df_train['probal_cat'] - df_train['probal_max']\ndf_test['larger2'] = df_test['probal_lgb'] + df_test['probal_xgb'] + df_test['probal_cat'] - df_test['probal_min']\ndf_test['less2'] = df_test['probal_lgb'] + df_test['probal_xgb'] + df_test['probal_cat'] - df_test['probal_max']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98e2205a6f13c77036f19532dce842a354940c60"},"cell_type":"code","source":"for col in ['probal_lgb','probal_xgb','probal_cat','probal_max','probal_min',\n            'lgb_xgb','lgb_cat','xgb_cat','probal_combine','larger2','less2']:\n    df_train[col + \"_multiplication\"] = -33.219281 * df_train[col]\n    df_test[col + \"_multiplication\"] = -33.219281 * df_test[col]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"988e7451646dd61c137f6c9fa1abaf85edd59442","trusted":true},"cell_type":"code","source":"df_train.head(6)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a960ec811e31130511c07d601ef40aeaab94c21b","trusted":true},"cell_type":"code","source":"x=df_train[df_train['target']<=-11]\nx=x.sort_values(by ='target',ascending=False)\nx=x[['target','outliers','probal_lgb','probal_xgb','probal_cat']]\nx","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9bf92b804e84444d5b4970ef706df3d858dddaca"},"cell_type":"markdown","source":"## 2、进行回归训练"},{"metadata":{"_uuid":"c4f20f27679889542acfd60d1f1ac381b201ac43","trusted":true},"cell_type":"code","source":"df_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers','outliers2']]\ntarget = df_train['target']\n#del df_train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9bbc95244978b519d94131907b547c2b6c94191","trusted":true},"cell_type":"code","source":"%%time\nparam = {'num_leaves': 31,\n         'min_data_in_leaf': 30, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate': 0.01,\n         \"min_child_samples\": 20,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1,\n         \"nthread\": 4,\n         \"random_state\": 4590}\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=4590)\noof = np.zeros(len(df_train))\npredictions = np.zeros(len(df_test))\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(df_train,df_train['outliers'].values)):\n    print(\"fold {}\".format(fold_))\n    trn_data = lgb.Dataset(df_train.iloc[trn_idx][df_train_columns], label=target.iloc[trn_idx])#, categorical_feature=categorical_feats)\n    val_data = lgb.Dataset(df_train.iloc[val_idx][df_train_columns], label=target.iloc[val_idx])#, categorical_feature=categorical_feats)\n\n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=200, early_stopping_rounds = 200)\n    oof[val_idx] = clf.predict(df_train.iloc[val_idx][df_train_columns], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = df_train_columns\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(df_test[df_train_columns], num_iteration=clf.best_iteration) / folds.n_splits\n\nnp.sqrt(mean_squared_error(oof, target))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40b64481054fa71e692829c7039eccceb31b77fe","trusted":true},"cell_type":"code","source":"cols = (feature_importance_df[[\"Feature\", \"importance\"]]\n        .groupby(\"Feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.Feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"Feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"355e9c24949b8e5d677fe5a2f117228c3310dab6","trusted":true},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":df_test[\"card_id\"].values})\nsub_df[\"target\"] = predictions\nif debug == 1:\n    sub_df.to_csv(\"submission_debug.csv\", index=False)\n    print(\"本次程序运行是测试debug用\")\nelse:\n    sub_df.to_csv(\"submission_model.csv\", index=False)\n    print(\"本次程序运行是运行数据模型\")\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c948f1d5cb568fe60dfb8f93a711d37864385335","trusted":false},"cell_type":"markdown","source":"## 3、删除outliers为1的样本后进行训练"},{"metadata":{"trusted":true,"_uuid":"ef081649d82d0010d8dd164fdab8524f83dc1e6a"},"cell_type":"code","source":"df_train2 = df_train[df_train['outliers'] == 0]\ntarget = df_train2['target']\ndel df_train2['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6658c8ecd193fc2d58d3f0667cb7222eca840768"},"cell_type":"code","source":"%%time\nparam = {'num_leaves': 31,\n         'min_data_in_leaf': 30, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate': 0.01,\n         \"min_child_samples\": 20,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1,\n         \"nthread\": 4,\n         \"random_state\": 4590}\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=4590)\noof = np.zeros(len(df_train2))\npredictions = np.zeros(len(df_test))\nfeature_importance_df2 = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(df_train2,df_train2['outliers'].values)):\n    print(\"fold {}\".format(fold_))\n    trn_data = lgb.Dataset(df_train2.iloc[trn_idx][df_train_columns], label=target.iloc[trn_idx])#, categorical_feature=categorical_feats)\n    val_data = lgb.Dataset(df_train2.iloc[val_idx][df_train_columns], label=target.iloc[val_idx])#, categorical_feature=categorical_feats)\n\n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=200, early_stopping_rounds = 200)\n    oof[val_idx] = clf.predict(df_train2.iloc[val_idx][df_train_columns], num_iteration=clf.best_iteration)\n    \n    fold_importance_df2 = pd.DataFrame()\n    fold_importance_df2[\"Feature2\"] = df_train_columns\n    fold_importance_df2[\"importance2\"] = clf.feature_importance()\n    fold_importance_df2[\"fold2\"] = fold_ + 1\n    feature_importance_df2 = pd.concat([feature_importance_df2, fold_importance_df2], axis=0)\n    \n    predictions += clf.predict(df_test[df_train_columns], num_iteration=clf.best_iteration) / folds.n_splits\n\nnp.sqrt(mean_squared_error(oof, target))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d3e276412360c1c5e91b8b0d8dd918f7fe73458"},"cell_type":"code","source":"cols = (feature_importance_df2[[\"Feature2\", \"importance2\"]]\n        .groupby(\"Feature2\")\n        .mean()\n        .sort_values(by=\"importance2\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df2.loc[feature_importance_df2.Feature2.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance2\",\n            y=\"Feature2\",\n            data=best_features.sort_values(by=\"importance2\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances2.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df44d7985e2f24705cb6c61f315c75f971b62e5e"},"cell_type":"code","source":"sub_df2 = pd.DataFrame({\"card_id\":df_test[\"card_id\"].values})\nsub_df2[\"target\"] = predictions\nif debug == 1:\n    sub_df2.to_csv(\"submission_debug2_without_outliers.csv\", index=False)\n    print(\"本次程序运行是测试debug用\")\nelse:\n    sub_df2.to_csv(\"submission_model2_without_outliers.csv\", index=False)\n    print(\"本次程序运行是运行数据模型\")\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"58c9a5445698e42dfbd9548695290487a2ce171a"},"cell_type":"markdown","source":"**To be continued ...**"},{"metadata":{"_uuid":"4911c52487dbf7ba90288818d3eb8761c8903be8","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}