{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom lightgbm import LGBMClassifier\nfrom sklearn.model_selection import KFold,StratifiedKFold, RepeatedKFold\nfrom sklearn.ensemble import RandomForestClassifier\nimport warnings\nimport time\nimport sys\nimport datetime\nfrom datetime import timedelta\nimport sklearn\nfrom sklearn.metrics import mean_squared_error,auc,roc_auc_score\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.preprocessing import QuantileTransformer\nfrom sklearn.preprocessing import PowerTransformer\nfrom scipy.stats.mstats import mode\nimport gc\nwarnings.simplefilter(action='ignore', category=FutureWarning)\npd.set_option('display.max_columns', 500)\n\nimport os\nprint(os.listdir(\"../input\"))\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/elo-merchant-category-recommendation/train.csv\")\ntest = pd.read_csv(\"../input/elo-merchant-category-recommendation/test.csv\")\ntrain_df = pd.read_csv(\"../input/fork-of-stacked-lgbm-reduced-models/trainconsol.csv\")\ntest_df = pd.read_csv(\"../input/fork-of-stacked-lgbm-reduced-models/testconsol.csv\")\ntrain_rfm = pd.read_csv(\"../input/customer-loyalty-based-on-rfm-analysis/trainrfm.csv\")\ntest_rfm = pd.read_csv(\"../input/customer-loyalty-based-on-rfm-analysis/testrfm.csv\")\nfeat_imp = pd.read_csv(\"../input/feature-importance-selection/scores_df.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"710c33afe39949a4d4dd9e9d8ab088edc6755b0a"},"cell_type":"code","source":"lessimp_feature =feat_imp.loc[feat_imp.gain_score <=0,'feature'].tolist()\nlessimp_feature.extend(['card_id','target','outliers','RFMIndex','RFMnewIndex','month_diff_mean','month_diff_min','month_diff_max','month_diff_var','month_diffnew_mean','month_diffnew_var'])\n# lessimp_feature = ['feature_1', 'year', 'weekday', 'cat2_2.0_sum', 'cat2_2.0_mean', 'cat2_3.0_sum',\n#                     'cat2_3.0_mean', 'cat2_5.0_sum', 'cat2_5.0_mean', 'cat3_A_mean', 'cattrans_1.0_sum',\n#                      'month_lag_max', 'month_diff_min', 'time_diff_min', 'city_idnew_nunique',\n#                      'state_idnew_nunique', 'merchant_category_idnew_nunique', 'cat2_1.0new_mean',\n#                      'cat2_2.0new_sum', 'cat2_2.0new_mean', 'cat2_3.0new_sum', 'cat2_3.0new_mean',\n#                      'cat2_4.0new_sum', 'cat3_Anew_sum', 'cat3_Cnew_sum', 'cat3_Cnew_mean',\n#                      'month_diffnew_min', 'month_diffnew_max', 'time_diffnew_min',\n#                      'weekendnew_sum', 'dayofweeknew_mode', 'mon_change_ratio_card_std',\n#                      'f_quantile', 'v_quantile', 'fnew_quantile', 'RFMnewScore', 'card_id',\n#                      'target', 'outliers', 'RFMIndex', 'RFMnewIndex', 'month_diff_mean', 'month_diff_min',\n#                      'month_diff_max', 'month_diff_var', 'month_diffnew_mean', 'month_diffnew_var']\nlessimp_feature","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcc8662a9c71a9e0615000c6c587abe80f14b220"},"cell_type":"code","source":"# lessimp_feature =['year','weekday','cat2_2.0_sum','cat2_2.0_mean', 'cat2_3.0_sum', 'cat2_3.0_mean',\n#  'cat2_5.0_sum','cat2_5.0_mean', 'cat3_A_sum', 'cat3_A_mean', 'cattrans_1.0_sum', 'installments_min',\n#  'month_lag_max', 'time_diff_min', 'city_idnew_nunique', 'cat2_1.0new_mean', 'cat2_2.0new_sum',\n#  'cat2_2.0new_mean', 'cat2_3.0new_sum', 'cat2_3.0new_mean', 'cat2_4.0new_sum', 'cat2_5.0new_sum',\n#  'cat2_5.0new_mean','cat3_Anew_sum','cat3_Cnew_sum','installmentsnew_min', 'installmentsnew_max',\n#  'weekendnew_sum', 'weekendnew_mean', 'mon_change_ratio_card_std', 'merchant_id_rep_nunique',\n#  'rep_card_month_freq_rep_max', 'merchant_id_repmer_nunique', 'v_quantile', 'fnew_quantile', 'RFMnewScore']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7e0f2034cd8d9515ad1e4e7f450aa59ba5b3d38"},"cell_type":"code","source":"train_rfm.drop(columns=['first_active_month', 'feature_1', 'feature_2', 'feature_3',\n       'target', 'year', 'month', 'weekday',  'frequency',\n       'value', 'recency',  'frequency_new', 'value_new', 'recency_new',\n        'reference_date', 'category_month_lag','elapsedtime'],inplace= True )\ntest_rfm.drop(columns=['first_active_month', 'feature_1', 'feature_2', 'feature_3',\n        'year', 'month', 'weekday',  'frequency',\n       'value', 'recency',  'frequency_new', 'value_new', 'recency_new',\n        'reference_date', 'category_month_lag','elapsedtime'],inplace= True )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d27e2f9a39c7012440071732f693e4435bc8a70"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4070e3e12c4d94ce997879f43be33e3f2bf63e9b"},"cell_type":"code","source":"train_df= pd.merge(train_df,train_rfm,on='card_id',how='left')\ntest_df= pd.merge(test_df,test_rfm,on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ecb537dc35ad83d97cedf2401cf60f4aad0ed69"},"cell_type":"code","source":"# for df in [train_df,test_df]:\n#     df['days_feature1'] = df['elapsedtime'] * df['feature_1']\n#     df['days_feature2'] = df['elapsedtime'] * df['feature_2']\n#     df['days_feature3'] = df['elapsedtime'] * df['feature_3']\n#     df['days_feature1_ratio'] = df['feature_1'] / df['elapsedtime']\n#     df['days_feature2_ratio'] = df['feature_2'] / df['elapsedtime']\n#     df['days_feature3_ratio'] = df['feature_3'] / df['elapsedtime']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2063df81a9da15f1ad313257a10da3c5665ca9cf"},"cell_type":"code","source":"target = train_df.target\ntrain_df= train_df.drop(['card_id','target'],axis=1)\ncard_id = test_df['card_id']\ntest_df= test_df.drop(['card_id'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f398d5fc7ad2df98d53e6813b142b6886d066fd3"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a36206ce795078c1cdd2fd572d0e3fa88ebdd46"},"cell_type":"code","source":"features = [c for c in train_df.columns if c not in lessimp_feature ]\ncategorical_feats = ['feature_2','feature_1','feature_comb_index', 'month','category_month_lag','f_quantile','r_quantile','RFMScore','rnew_quantile','vnew_quantile' ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1554ab458895b64dbbd608358d6eb7b23660683"},"cell_type":"code","source":"train_df[features].info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efcb5f4231721ea05d22fca5e0d1d7fe4f6043ef"},"cell_type":"code","source":"param = {\n    'objective':'regression',\n    'learning_rate': 0.01,\n    'boosting': 'gbdt',\n    'num_leaves': 31,\n    'colsample_bytree': 0.5178284844630764,\n    'subsample': 0.6701600397051216,\n    'max_depth': 10,\n    'reg_alpha': 2.5056142297682493,\n    'reg_lambda': 2.4364948978876253,\n    'min_split_gain': 4.947802532689812,\n    'min_child_weight': 43.41313044141597,\n    'min_data_in_leaf': 40,\n      \"metric\": 'rmse',\n    'bagging_seed':11,\n      \"verbosity\": -1\n}\n\nparam1= {'num_leaves': 50,\n         'min_data_in_leaf': 30, \n         'objective':'regression',\n         'max_depth': 10,\n         'learning_rate': 0.005,\n         \"min_child_samples\": 100,\n         \"boosting\": \"gbdt\",\n         \"metric\": 'rmse',\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 11,\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1\n           }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c25d911594359f9325d87cd0114fd6635a05d386"},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)\noof = np.zeros(len(train_df))\npredictions = np.zeros(len(test_df))\nstart = time.time()\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_df.values,train_df['feature_comb_index'].values)):\n     print(\"fold n°{}\".format(fold_))\n     trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)\n     val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)\n\n     num_round = 10000\n     params= param\n     clf = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)\n     oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)\n    \n     fold_importance_df = pd.DataFrame()\n     fold_importance_df[\"feature\"] = features\n     fold_importance_df[\"importance\"] = clf.feature_importance()\n     fold_importance_df[\"fold\"] = fold_ + 1\n     feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n     predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration) / folds.n_splits\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d33f664e3fb35fafb1fd2dd66399a24892e26fb1"},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n         .groupby(\"feature\")\n         .mean()\n         .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n             y=\"feature\",\n             data=best_features.sort_values(by=\"importance\",\n                                            ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f36569a9a4f212f566a61f1b8a22684b13ef1ac8"},"cell_type":"code","source":"print(min(predictions))\nprint(max(predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2de62603d2a76077f716163f0da57221fa24539"},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(30,12)})\nsns.scatterplot(x=target,y=oof)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d114e6837a3360362482dcaaf38aac2b7bf1568"},"cell_type":"code","source":"submit = pd.DataFrame({'card_id':card_id,'target':predictions})\nsubmit.to_csv(\"featurecombLGBM1.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6dd6f25a87c3ddf0a662e2d4665d547c1c74db12"},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)\noof1 = np.zeros(len(train_df))\npredictions1 = np.zeros(len(test_df))\nstart = time.time()\nfeature_importance_df1 = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_df.values,train_df['category_month_lag'].values)):\n     print(\"fold n°{}\".format(fold_))\n     trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)\n     val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)\n\n     num_round = 10000\n     params= param\n#      params ={\n#                  'task': 'train',\n#                  'boosting': 'goss',\n#                  'objective': 'regression',\n#                  'metric': 'rmse',\n#                  'learning_rate': 0.01,\n#                  'subsample': 0.9855232997390695,\n#                  'max_depth': 7,\n#                  'top_rate': 0.9064148448434349,\n#                  'num_leaves': 63,\n#                  'min_child_weight': 41.9612869171337,\n#                  'other_rate': 0.0721768246018207,\n#                  'reg_alpha': 9.677537745007898,\n#                  'colsample_bytree': 0.5665320670155495,\n#                  'min_split_gain': 9.820197773625843,\n#                  'reg_lambda': 8.2532317400459,\n#                  'min_data_in_leaf': 21,\n#                  'verbose': -1,\n#                  'seed':int(2**fold_),\n#                  'bagging_seed':int(2**fold_),\n#                  'drop_seed':int(2**fold_)}\n     \n    \n     clf1 = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)\n     oof1[val_idx] = clf1.predict(train_df.iloc[val_idx][features], num_iteration=clf1.best_iteration)\n    \n     fold_importance_df1 = pd.DataFrame()\n     fold_importance_df1[\"feature\"] = features\n     fold_importance_df1[\"importance\"] = clf1.feature_importance()\n     fold_importance_df1[\"fold\"] = fold_ + 1\n     feature_importance_df1 = pd.concat([feature_importance_df1, fold_importance_df1], axis=0)\n    \n     predictions1 += clf1.predict(test_df[features], num_iteration=clf1.best_iteration) / folds.n_splits\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof1, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a417f9b279de87b0f0b27c94b956282601455356"},"cell_type":"code","source":" cols = (feature_importance_df1[[\"feature\", \"importance\"]]\n         .groupby(\"feature\")\n         .mean()\n         .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\n best_features1 = feature_importance_df1.loc[feature_importance_df1.feature.isin(cols)]\n\n plt.figure(figsize=(14,25))\n sns.barplot(x=\"importance\",\n             y=\"feature\",\n             data=best_features1.sort_values(by=\"importance\",\n                                            ascending=False))\n plt.title('LightGBM Features (avg over folds)')\n plt.tight_layout()\n plt.savefig('lgbm_importances1.png')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f90c0fecce4dadcce5bc934b87945c50f256cec"},"cell_type":"code","source":"print(min(predictions1))\nprint(max(predictions1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ff1c7ffd9204d7035c562b2403512da6442d3d6"},"cell_type":"code","source":"submissioncatmonth = pd.read_csv('../input/elo-merchant-category-recommendation/sample_submission.csv')\nsubmissioncatmonth['target'] = predictions1\nsubmissioncatmonth.to_csv('LGBMcategorymonthlag1.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"721b5764ecf6f6eaafac2031b164c9ca8fb2c5fb"},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(30,12)})\nsns.scatterplot(x=target,y=oof1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"602fc942cc8dd32f1e35703f27fcb279d4764716"},"cell_type":"code","source":"folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)\noof2 = np.zeros(len(train_df))\npredictions2 = np.zeros(len(test_df))\nstart = time.time()\nfeature_importance_df2 = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_df.values,train_df['RFMScore'].values)):\n     print(\"fold n°{}\".format(fold_))\n     trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)\n     val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)\n\n     num_round = 10000\n     params= param\n#      params ={\n#                  'task': 'train',\n#                  'boosting': 'goss',\n#                  'objective': 'regression',\n#                  'metric': 'rmse',\n#                  'learning_rate': 0.01,\n#                  'subsample': 0.9855232997390695,\n#                  'max_depth': 7,\n#                  'top_rate': 0.9064148448434349,\n#                  'num_leaves': 63,\n#                  'min_child_weight': 41.9612869171337,\n#                  'other_rate': 0.0721768246018207,\n#                  'reg_alpha': 9.677537745007898,\n#                  'colsample_bytree': 0.5665320670155495,\n#                  'min_split_gain': 9.820197773625843,\n#                  'reg_lambda': 8.2532317400459,\n#                  'min_data_in_leaf': 21,\n#                  'verbose': -1,\n#                  'seed':int(2**fold_),\n#                  'bagging_seed':int(2**fold_),\n#                  'drop_seed':int(2**fold_)}\n     \n    \n     clf2 = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)\n     oof2[val_idx] = clf2.predict(train_df.iloc[val_idx][features], num_iteration=clf2.best_iteration)\n    \n     fold_importance_df2 = pd.DataFrame()\n     fold_importance_df2[\"feature\"] = features\n     fold_importance_df2[\"importance\"] = clf2.feature_importance()\n     fold_importance_df2[\"fold\"] = fold_ + 1\n     feature_importance_df2 = pd.concat([feature_importance_df2, fold_importance_df2], axis=0)\n    \n     predictions2 += clf2.predict(test_df[features], num_iteration=clf2.best_iteration) / folds.n_splits\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof2, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57a01e87f94459afe5e20e5750288c72ea01f338"},"cell_type":"code","source":" cols = (feature_importance_df2[[\"feature\", \"importance\"]]\n         .groupby(\"feature\")\n         .mean()\n         .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\n best_features2 = feature_importance_df2.loc[feature_importance_df2.feature.isin(cols)]\n\n plt.figure(figsize=(14,25))\n sns.barplot(x=\"importance\",\n             y=\"feature\",\n             data=best_features2.sort_values(by=\"importance\",\n                                            ascending=False))\n plt.title('LightGBM Features (avg over folds)')\n plt.tight_layout()\n plt.savefig('lgbm_importances2.png')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31450f810d010f4550a210f54de6ab7c088719f9"},"cell_type":"code","source":"print(min(predictions2))\nprint(max(predictions2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8aba149046b3841a1a6c5fe84971927695b2410d"},"cell_type":"code","source":"submissionrfm = pd.read_csv('../input/elo-merchant-category-recommendation/sample_submission.csv')\nsubmissionrfm['target'] = predictions2\nsubmissionrfm.to_csv('LGBMrfmscore1.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5343fe00c79ef0d9c0303a427eae7d7cf00b115d"},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(30,12)})\nsns.scatterplot(x=target,y=oof2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b98cb2d5293c3c1fedf6e882f68c9ec7f7b6aa86"},"cell_type":"code","source":"folds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=15)\noof3 = np.zeros(len(train_df))\npredictions3 = np.zeros(len(test_df))\nstart = time.time()\nfeature_importance_df3 = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_df.values,target.values)):\n     print(\"fold n°{}\".format(fold_))\n     trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=categorical_feats)\n     val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=categorical_feats)\n\n     num_round = 10000\n     params= param\n#      params ={\n#                  'task': 'train',\n#                  'boosting': 'goss',\n#                  'objective': 'regression',\n#                  'metric': 'rmse',\n#                  'learning_rate': 0.01,\n#                  'subsample': 0.9855232997390695,\n#                  'max_depth': 7,\n#                  'top_rate': 0.9064148448434349,\n#                  'num_leaves': 63,\n#                  'min_child_weight': 41.9612869171337,\n#                  'other_rate': 0.0721768246018207,\n#                  'reg_alpha': 9.677537745007898,\n#                  'colsample_bytree': 0.5665320670155495,\n#                  'min_split_gain': 9.820197773625843,\n#                  'reg_lambda': 8.2532317400459,\n#                  'min_data_in_leaf': 21,\n#                  'verbose': -1,\n#                  'seed':int(2**fold_),\n#                  'bagging_seed':int(2**fold_),\n#                  'drop_seed':int(2**fold_)}\n     \n    \n     clf3 = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)\n     oof3[val_idx] = clf3.predict(train_df.iloc[val_idx][features], num_iteration=clf3.best_iteration)\n    \n     fold_importance_df3 = pd.DataFrame()\n     fold_importance_df3[\"feature\"] = features\n     fold_importance_df3[\"importance\"] = clf3.feature_importance()\n     fold_importance_df3[\"fold\"] = fold_ + 1\n     feature_importance_df3 = pd.concat([feature_importance_df3, fold_importance_df3], axis=0)\n    \n     predictions3 += clf3.predict(test_df[features], num_iteration=clf3.best_iteration) / (5 *  2)\n\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof3, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0c51ae98ef3b53632ca37a3f81a5f88d265d871"},"cell_type":"code","source":" cols = (feature_importance_df3[[\"feature\", \"importance\"]]\n         .groupby(\"feature\")\n         .mean()\n         .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\n best_features3 = feature_importance_df3.loc[feature_importance_df3.feature.isin(cols)]\n\n plt.figure(figsize=(14,25))\n sns.barplot(x=\"importance\",\n             y=\"feature\",\n             data=best_features3.sort_values(by=\"importance\",\n                                            ascending=False))\n plt.title('LightGBM Features (avg over folds)')\n plt.tight_layout()\n plt.savefig('lgbm_importances3.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"576e07d5ecb60747617e11e3086a800dcdd6af86"},"cell_type":"code","source":"submissiontarget = pd.read_csv('../input/elo-merchant-category-recommendation/sample_submission.csv')\nsubmissiontarget['target'] = predictions3\nsubmissiontarget.to_csv('LGBMfinal1.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2c3e678d1dea05da86ead32c44dcd13aa4bd3d0"},"cell_type":"code","source":"print(min(predictions3))\nprint(max(predictions3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc898c5bfff59382affeefc5370050b9bb42468c"},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(30,12)})\nsns.scatterplot(x=target,y=oof3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c304829cdb7c2c1aa401e797fe400304338b36a"},"cell_type":"code","source":"oof.to_csv(\"oof.csv\",index=False)\noof1.to_csv(\"oof1.csv\",index=False)\noof2.to_csv(\"oof2.csv\",index=False)\noof3.to_csv(\"oof3.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6842e5961c52c3bd55ecfac10bafaa25a7f4096f"},"cell_type":"code","source":"from sklearn.linear_model import BayesianRidge\n\ntrain_stack = np.vstack([oof,oof1,oof2,oof3]).transpose()\ntest_stack = np.vstack([predictions, predictions1, predictions2,predictions3]).transpose()\n\nfolds_stack = RepeatedKFold(n_splits=5, n_repeats=1, random_state=4590)\noof_stack = np.zeros(train_stack.shape[0])\npredictions4 = np.zeros(test_stack.shape[0])\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds_stack.split(train_stack,target)):\n    print(\"fold {}\".format(fold_))\n    trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values\n    val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values\n    \n    clf4 = BayesianRidge()\n    clf4.fit(trn_data, trn_y)\n    \n    oof_stack[val_idx] = clf4.predict(val_data)\n    predictions4 += clf4.predict(test_stack) / 5\n    \nnp.sqrt(mean_squared_error(target.values, oof_stack))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c46ee174e19cec1b03e10284f54238654e0bec4d"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/elo-merchant-category-recommendation/sample_submission.csv')\nsample_submission['target'] = predictions4\nsample_submission.to_csv('LGBMstacksubmissionfinal1.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d25b434bcd5cf5d92f77f39f0078c6ede9b99683"},"cell_type":"code","source":"print(min(predictions4))\nprint(max(predictions4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b1b5846c8f268ae3b6e0097cf22d2fa05ffac66"},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(30,12)})\nsns.scatterplot(x=target,y=oof_stack)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}