{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nimport gc\nimport time\nfrom pandas.core.common import SettingWithCopyWarning\nimport warnings\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold, GridSearchCV\n\n# I don't like SettingWithCopyWarnings ...\nwarnings.simplefilter('error', SettingWithCopyWarning)\ngc.enable()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8c6c0d5d8cf00737b8ae536b0d528b32fac9166"},"cell_type":"markdown","source":"### 1.Get the extracted data"},{"metadata":{"trusted":true,"_uuid":"6fe04db087c3d25b8b4a5873ee6f104e427525f0"},"cell_type":"code","source":"train = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', \n                    dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', \n                   dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b64b336b70ccb5478d33f9e042eeeabee549c919"},"cell_type":"markdown","source":"### 2.Data Fold using GroupKfold"},{"metadata":{"trusted":true,"_uuid":"cd54722ebf9ae8b68eddb2116c0559625e7d6a48"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18b2fb7729077132ef249956a1bbb072baee17db"},"cell_type":"markdown","source":"### 3.Target Define transaction"},{"metadata":{"trusted":true,"_uuid":"c539c76d9a5305066ea084ca32016132da41b35b"},"cell_type":"code","source":"y_reg = train['totals.transactionRevenue'].fillna(0)\ndel train['totals.transactionRevenue']\n\nif 'totals.transactionRevenue' in test.columns:\n    del test['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d9af86b1cf94e192d704bfd419eed4490e1debe"},"cell_type":"markdown","source":"### 4.Add date features\n\nOnly add the one I think can ganeralize"},{"metadata":{"trusted":true,"_uuid":"fbf93fda18940aa29105fa2d66f3514311d4fc68"},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b04391e12fdca0199ff22ab7febd9b0b53d6cbb3"},"cell_type":"code","source":"for df in [train, test]:\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['sess_date_dow'] = df['date'].dt.dayofweek\n    df['sess_date_hours'] = df['date'].dt.hour\n    df['sess_date_dom'] = df['date'].dt.day","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c46393e2a0eb5b96972fc099e36ece5f90d7dcdd"},"cell_type":"code","source":"# https://www.kaggle.com/prashantkikani/teach-lightgbm-to-sum-predictions-fe\ndef browser_mapping(x):\n    browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']\n    if x in browsers:\n        return x.lower()\n    elif  ('android' in x) or ('samsung' in x) or ('mini' in x) or ('iphone' in x) or ('in-app' in x) or ('playstation' in x):\n        return 'mobile browser'\n    elif  ('mozilla' in x) or ('chrome' in x) or ('blackberry' in x) or ('nokia' in x) or ('browser' in x) or ('amazon' in x):\n        return 'mobile browser'\n    elif  ('lunascape' in x) or ('netscape' in x) or ('blackberry' in x) or ('konqueror' in x) or ('puffin' in x) or ('amazon' in x):\n        return 'mobile browser'\n    elif '(not set)' in x:\n        return x\n    else:\n        return 'others'\n    \n    \ndef adcontents_mapping(x):\n    if  ('google' in x):\n        return 'google'\n    elif  ('placement' in x) | ('placememnt' in x):\n        return 'placement'\n    elif '(not set)' in x or 'nan' in x:\n        return x\n    elif 'ad' in x:\n        return 'ad'\n    else:\n        return 'others'\n    \ndef source_mapping(x):\n    if  ('google' in x):\n        return 'google'\n    elif  ('youtube' in x):\n        return 'youtube'\n    elif '(not set)' in x or 'nan' in x:\n        return x\n    elif 'yahoo' in x:\n        return 'yahoo'\n    elif 'facebook' in x:\n        return 'facebook'\n    elif 'reddit' in x:\n        return 'reddit'\n    elif 'bing' in x:\n        return 'bing'\n    elif 'quora' in x:\n        return 'quora'\n    elif 'outlook' in x:\n        return 'outlook'\n    elif 'linkedin' in x:\n        return 'linkedin'\n    elif 'pinterest' in x:\n        return 'pinterest'\n    elif 'ask' in x:\n        return 'ask'\n    elif 'siliconvalley' in x:\n        return 'siliconvalley'\n    elif 'lunametrics' in x:\n        return 'lunametrics'\n    elif 'amazon' in x:\n        return 'amazon'\n    elif 'mysearch' in x:\n        return 'mysearch'\n    elif 'qiita' in x:\n        return 'qiita'\n    elif 'messenger' in x:\n        return 'messenger'\n    elif 'twitter' in x:\n        return 'twitter'\n    elif 't.co' in x:\n        return 't.co'\n    elif 'vk.com' in x:\n        return 'vk.com'\n    elif 'search' in x:\n        return 'search'\n    elif 'edu' in x:\n        return 'edu'\n    elif 'mail' in x:\n        return 'mail'\n    elif 'ad' in x:\n        return 'ad'\n    elif 'golang' in x:\n        return 'golang'\n    elif 'direct' in x:\n        return 'direct'\n    elif 'dealspotr' in x:\n        return 'dealspotr'\n    elif 'sashihara' in x:\n        return 'sashihara'\n    elif 'phandroid' in x:\n        return 'phandroid'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'mdn' in x:\n        return 'mdn'\n    elif 'duckduckgo' in x:\n        return 'duckduckgo'\n    elif 'seroundtable' in x:\n        return 'seroundtable'\n    elif 'metrics' in x:\n        return 'metrics'\n    elif 'sogou' in x:\n        return 'sogou'\n    elif 'businessinsider' in x:\n        return 'businessinsider'\n    elif 'github' in x:\n        return 'github'\n    elif 'gophergala' in x:\n        return 'gophergala'\n    elif 'yandex' in x:\n        return 'yandex'\n    elif 'msn' in x:\n        return 'msn'\n    elif 'dfa' in x:\n        return 'dfa'\n    elif '(not set)' in x:\n        return '(not set)'\n    elif 'feedly' in x:\n        return 'feedly'\n    elif 'arstechnica' in x:\n        return 'arstechnica'\n    elif 'squishable' in x:\n        return 'squishable'\n    elif 'flipboard' in x:\n        return 'flipboard'\n    elif 't-online.de' in x:\n        return 't-online.de'\n    elif 'sm.cn' in x:\n        return 'sm.cn'\n    elif 'wow' in x:\n        return 'wow'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'partners' in x:\n        return 'partners'\n    else:\n        return 'others'\n\ntrain['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')\n\ntest['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntest['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntest['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')\n\ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['source.country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']\n    data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '_' + data_df['trafficSource.medium']\n    data_df['browser.category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']\n    data_df['browser.os'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']\n    return data_df\n\ntrain = process_device(train)\ntest = process_device(test)\n\ndef custom(data):\n    print('custom..')\n    data['device_deviceCategory_channelGrouping'] = data['device.deviceCategory'] + \"_\" + data['channelGrouping']\n    data['channelGrouping_browser'] = data['device.browser'] + \"_\" + data['channelGrouping']\n    data['channelGrouping_OS'] = data['device.operatingSystem'] + \"_\" + data['channelGrouping']\n    \n    for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:\n        for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:\n            data[i + \"_\" + j] = data[i] + \"_\" + data[j]\n    \n    data['content.source'] = data['trafficSource.adContent'] + \"_\" + data['source.country']\n    data['medium.source'] = data['trafficSource.medium'] + \"_\" + data['source.country']\n    return data\n\ntrain = custom(train)\ntest = custom(test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59e7c9bcf14d3f430489d15d22757b91dd9d5ad1"},"cell_type":"markdown","source":"### 5.Create features list"},{"metadata":{"trusted":true,"_uuid":"e09ff3124721ad70eab65b924093b5fd0c12f9c6"},"cell_type":"code","source":"excluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime'\n]\n\ncategorical_features = [\n    _f for _f in train.columns\n    if (_f not in excluded_features) & (train[_f].dtype == 'object')\n]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d207a80600a44e758b6011f24c5808f9b164866"},"cell_type":"markdown","source":"### 6.Factorize categoricals"},{"metadata":{"trusted":true,"_uuid":"4f00eb6b6166cd5be25ebad22aa9dc91a8a3ded2"},"cell_type":"code","source":"for f in categorical_features:\n    train[f], indexer = pd.factorize(train[f])\n    test[f] = indexer.get_indexer(test[f])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49c931caccd24198208667b9a6bb6cc5ccd99695"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1da57eaaf55fda457e392d2dcdb73ca6487cdb47"},"cell_type":"markdown","source":"### 7.Cross Validation for Hyperparameter Tuning"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"a6605f3d75474f1b48e5c01f9965b783830f433c"},"cell_type":"code","source":"params={'learning_rate': 0.03,\n        'objective':'regression',\n        'metric':'rmse',\n        'num_leaves': 31,\n        'verbose': 1,\n        \"subsample\": 0.99,\n        \"colsample_bytree\": 0.99,\n        \"random_state\":42,\n        'max_depth': 15,\n        'lambda_l2': 0.02085548700474218,\n        'lambda_l1': 0.004107624022751344,\n        'bagging_fraction': 0.7934712636944741,\n        'feature_fraction': 0.686612409641711,\n        'min_child_samples': 21\n       }\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7517b473bd776f51925ab0243cf1e3b6592ce0e"},"cell_type":"markdown","source":"### 8.Model Training with Kfold Validation LightGBM"},{"metadata":{"trusted":true,"_uuid":"e9c155197545dc43ec2f3e23e25955532f7ad25d"},"cell_type":"code","source":"folds = get_folds(df=train, n_splits=5)\n\ntrain_features = [_f for _f in train.columns if _f not in excluded_features]\n# print(train_features)\n\nimportances = pd.DataFrame()\noof_reg_preds = np.zeros(train.shape[0])\nsub_reg_preds = np.zeros(test.shape[0])\nfor fold_, (trn_, val_) in enumerate(folds):\n    print(\"Fold:\",fold_)\n    trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]\n    val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]\n    reg = lgb.LGBMRegressor(**params,\n         n_estimators=1000\n    )\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        verbose=100,\n        eval_metric='rmse'\n    )\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    _preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_reg_preds += np.expm1(_preds) / len(folds)\n    \nmean_squared_error(np.log1p(y_reg), oof_reg_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b23cdb7ee1e35c6deba9edd6ffd87a4cdc72920"},"cell_type":"markdown","source":"### 9.Display feature importances"},{"metadata":{"trusted":true,"_uuid":"61d4aacc57c2c7c4d9ce0b1cab7221ca7c69138f"},"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nimportances['gain_log'] = np.log1p(importances['gain'])\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c64447b5858dc899b6b78d26a4e68a8582995051"},"cell_type":"markdown","source":"### 10.Create user level predictions"},{"metadata":{"trusted":true,"_uuid":"1fc92504b40407cff766489db1966e729d0466df"},"cell_type":"code","source":"train['predictions'] = np.expm1(oof_reg_preds)\ntest['predictions'] = sub_reg_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"515d3d14483844c93710fb6b6f9e39a6a2a1d384"},"cell_type":"code","source":"# Aggregate data at User level\ntrn_data = train[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31d76001e77e3ee2e10e67a5bc899a83c4cfeb8d"},"cell_type":"code","source":"%%time\n# Create a list of predictions for each Visitor\ntrn_pred_list = train[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8f1bac1ef32a58f70aa7bcca79acc7119ddcfb5"},"cell_type":"code","source":"# Create a DataFrame with VisitorId as index\n# trn_pred_list contains dict \n# so creating a dataframe from it will expand dict values into columns\ntrn_all_predictions = pd.DataFrame(list(trn_pred_list.values), index=trn_data.index)\ntrn_feats = trn_all_predictions.columns\ntrn_all_predictions['t_mean'] = np.log1p(trn_all_predictions[trn_feats].mean(axis=1))\ntrn_all_predictions['t_median'] = np.log1p(trn_all_predictions[trn_feats].median(axis=1))\ntrn_all_predictions['t_sum_log'] = np.log1p(trn_all_predictions[trn_feats]).sum(axis=1)\ntrn_all_predictions['t_sum_act'] = np.log1p(trn_all_predictions[trn_feats].fillna(0).sum(axis=1))\ntrn_all_predictions['t_nb_sess'] = trn_all_predictions[trn_feats].isnull().sum(axis=1)\n#trn_all_predictions.to_csv('trn_all_predictions.csv', index=False)\nfull_data = pd.concat([trn_data, trn_all_predictions], axis=1)\ndel trn_data, trn_all_predictions\ngc.collect()\nfull_data.shape\nfull_data.to_csv('full_data.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12a86e1f61f819417bede61f3746366b5028222f"},"cell_type":"code","source":"%%time\nsub_pred_list = test[['fullVisitorId', 'predictions']].groupby('fullVisitorId')\\\n    .apply(lambda df: list(df.predictions))\\\n    .apply(lambda x: {'pred_'+str(i): pred for i, pred in enumerate(x)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"611f2a86ffff5947b6863818538577b58f099535"},"cell_type":"code","source":"sub_data = test[train_features + ['fullVisitorId']].groupby('fullVisitorId').mean()\nsub_all_predictions = pd.DataFrame(list(sub_pred_list.values), index=sub_data.index)\nfor f in trn_feats:\n    if f not in sub_all_predictions.columns:\n        sub_all_predictions[f] = np.nan\nsub_all_predictions['t_mean'] = np.log1p(sub_all_predictions[trn_feats].mean(axis=1))\nsub_all_predictions['t_median'] = np.log1p(sub_all_predictions[trn_feats].median(axis=1))\nsub_all_predictions['t_sum_log'] = np.log1p(sub_all_predictions[trn_feats]).sum(axis=1)\nsub_all_predictions['t_sum_act'] = np.log1p(sub_all_predictions[trn_feats].fillna(0).sum(axis=1))\nsub_all_predictions['t_nb_sess'] = sub_all_predictions[trn_feats].isnull().sum(axis=1)\n#sub_all_predictions.to_csv('sub_all_predictions.csv',index = False)\nsub_full_data = pd.concat([sub_data, sub_all_predictions], axis=1)\ndel sub_data, sub_all_predictions\ngc.collect()\nsub_full_data.shape\nsub_full_data.to_csv('sub_full_data.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c901512b911c731113c0515e97ce80ddf7b7f118"},"cell_type":"markdown","source":"### 11.Create target and Cross Validation"},{"metadata":{"trusted":true,"_uuid":"bf3b05a18f83821d12d05500f8e0e52f0603d6c5"},"cell_type":"code","source":"train['target'] = y_reg\ntrn_user_target = train[['fullVisitorId', 'target']].groupby('fullVisitorId').sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41948378780ac6abfec82f4c25a93542f8d278d3"},"cell_type":"code","source":"params={'learning_rate': 0.03,\n        'objective':'regression',\n        'metric':'rmse',\n        'num_leaves': 31,\n        'verbose': 1,\n        \"subsample\": 0.99,\n        \"colsample_bytree\": 0.99,\n        \"random_state\":42,\n        'max_depth': 15,\n        'lambda_l2': 0.02085548700474218,\n        'lambda_l1': 0.004107624022751344,\n        'bagging_fraction': 0.7934712636944741,\n        'feature_fraction': 0.686612409641711,\n        'min_child_samples': 21\n       }\n\nxgb_params = {\n        'objective': 'reg:linear',\n        'booster': 'gbtree',\n        'learning_rate': 0.02,\n        'max_depth': 22,\n        'min_child_weight': 57,\n        'gamma' : 1.45,\n        'alpha': 0.0,\n        'lambda': 0.0,\n        'subsample': 0.67,\n        'colsample_bytree': 0.054,\n        'colsample_bylevel': 0.50,\n        'n_jobs': -1,\n        'random_state': 456\n    }\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73a0db5cfa89fa95a120d6c02b768272f30bca2b"},"cell_type":"markdown","source":"### 12.Train a model at Visitor level"},{"metadata":{"trusted":true,"_uuid":"236f5546dfad2fee431a28534ef57cf8fec249ec"},"cell_type":"code","source":"from xgboost import XGBRegressor\nfolds = get_folds(df=full_data[['totals.pageviews']].reset_index(), n_splits=5)\n\noof_preds = np.zeros(full_data.shape[0])\noof_preds1 = np.zeros(full_data.shape[0])\nboth_oof = np.zeros(full_data.shape[0])\nsub_preds = np.zeros(sub_full_data.shape[0])\nvis_importances = pd.DataFrame()\n\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = full_data.iloc[trn_], trn_user_target['target'].iloc[trn_]\n    val_x, val_y = full_data.iloc[val_], trn_user_target['target'].iloc[val_]\n    \n    xg = XGBRegressor(**xgb_params, n_estimators=1000)\n    \n    reg = lgb.LGBMRegressor(**params,\n        n_estimators=1500,\n    )\n    print('XGB' + \"-\" * 50)\n    xg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    \n    print('LGB' + \"-\" * 50)\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(trn_x, np.log1p(trn_y)), (val_x, np.log1p(val_y))],\n        eval_names=['TRAIN', 'VALID'],\n        early_stopping_rounds=50,\n        eval_metric='rmse',\n        verbose=100\n    )\n    \n    imp_df = pd.DataFrame()\n    imp_df['feature'] = trn_x.columns\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    vis_importances = pd.concat([vis_importances, imp_df], axis=0, sort=False)\n    \n    oof_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_preds1[val_] = xg.predict(val_x)\n    \n    oof_preds[oof_preds < 0] = 0\n    oof_preds1[oof_preds1 < 0] = 0\n    \n    both_oof[val_] = oof_preds[val_] * 0.6 + oof_preds1[val_] * 0.4\n    \n    # Make sure features are in the same order\n    _preds = reg.predict(sub_full_data[full_data.columns], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    \n    pre = xg.predict(sub_full_data[full_data.columns])\n    pre[pre<0]=0\n    \n    sub_preds += (_preds / len(folds)) * 0.6 + (pre / len(folds)) * 0.4\n    \nprint(\"LGB  \", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds) ** .5)\nprint(\"XGB  \", mean_squared_error(np.log1p(trn_user_target['target']), oof_preds1) ** .5)\nprint(\"Combine  \", mean_squared_error(np.log1p(trn_user_target['target']), both_oof) ** .5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a334da6d1b615108f7d7200a8268bd2aff52d65"},"cell_type":"markdown","source":"### 13.Display feature importances"},{"metadata":{"trusted":true,"_uuid":"c9722bdf108cf510b6f59db3d99e406790c99194"},"cell_type":"code","source":"vis_importances['gain_log'] = np.log1p(vis_importances['gain'])\nmean_gain = vis_importances[['gain', 'feature']].groupby('feature').mean()\nvis_importances['mean_gain'] = vis_importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 25))\nsns.barplot(x='gain_log', y='feature', data=vis_importances.sort_values('mean_gain', ascending=False).iloc[:300])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbd81ac5df63f2c94c00e678b00dd7993d06df6f"},"cell_type":"markdown","source":"### 14.Save Result"},{"metadata":{"trusted":true,"_uuid":"feb757b51ac10132118cc0db22f7adda70f0913f"},"cell_type":"code","source":"sub_full_data['PredictedLogRevenue'] = sub_preds\nsub_full_data[['PredictedLogRevenue']].to_csv('LGB_XGB.csv', index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}