{"cells":[{"metadata":{"_uuid":"3d981a603152fefdc26802952cd859bf887103ee"},"cell_type":"markdown","source":"# Bayesian Optimization for LightGBM  \n- prashantkikani kenel (https://www.kaggle.com/prashantkikani/rstudio-lgb-single-model-lb1-6607) \n- Bayesian Optimization kernel (https://www.kaggle.com/sz8416/simple-bayesian-optimization-for-lightgbm)\n- stratified kernel (https://www.kaggle.com/youhanlee/stratified-sampling-for-regression-lb-1-6595)"},{"metadata":{"_uuid":"e31aa3e6f20cc9eacac8cd6d90905617f865fa5b"},"cell_type":"markdown","source":"prashantkikani (https://www.kaggle.com/prashantkikani/rstudio-lgb-single-model-lb1-6607) feature engineering code using."},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0f23a70127f63b3d7de1f3b2540056031971a986","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"'''\nAdding some new features..\nCombining categorical features.\n'''\n\nimport os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport time\nfrom datetime import datetime\nimport gc\nimport psutil\nfrom sklearn.preprocessing import LabelEncoder\n\nPATH=\"../input/\"\nNUM_ROUNDS = 20000\nVERBOSE_EVAL = 500\nSTOP_ROUNDS = 100\nN_SPLITS = 10\n\n #the columns that will be parsed to extract the fields from the jsons\ncols_to_parse = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\ndef read_parse_dataframe(file_name):\n    #full path for the data file\n    path = PATH + file_name\n    #read the data file, convert the columns in the list of columns to parse using json loader,\n    #convert the `fullVisitorId` field as a string\n    data_df = pd.read_csv(path, \n        converters={column: json.loads for column in cols_to_parse}, \n        dtype={'fullVisitorId': 'str'})\n    #parse the json-type columns\n    for col in cols_to_parse:\n        #each column became a dataset, with the columns the fields of the Json type object\n        json_col_df = json_normalize(data_df[col])\n        json_col_df.columns = [f\"{col}_{sub_col}\" for sub_col in json_col_df.columns]\n        #we drop the object column processed and we add the columns created from the json fields\n        data_df = data_df.drop(col, axis=1).merge(json_col_df, right_index=True, left_index=True)\n    return data_df\n    \ndef process_date_time(data_df):\n    print(\"process date time ...\")\n    data_df['date'] = data_df['date'].astype(str)\n    data_df[\"date\"] = data_df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n    data_df[\"date\"] = pd.to_datetime(data_df[\"date\"])   \n    data_df[\"year\"] = data_df['date'].dt.year\n    data_df[\"month\"] = data_df['date'].dt.month\n    data_df[\"day\"] = data_df['date'].dt.day\n    data_df[\"weekday\"] = data_df['date'].dt.weekday\n    data_df['weekofyear'] = data_df['date'].dt.weekofyear\n    data_df['month_unique_user_count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day_unique_user_count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday_unique_user_count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n\n    return data_df\n\ndef process_format(data_df):\n    print(\"process format ...\")\n    for col in ['visitNumber', 'totals_hits', 'totals_pageviews']:\n        data_df[col] = data_df[col].astype(float)\n    data_df['trafficSource_adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\n    data_df['trafficSource_isTrueDirect'].fillna(False, inplace=True)\n    return data_df\n    \ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['browser_category'] = data_df['device_browser'] + '_' + data_df['device_deviceCategory']\n    data_df['browser_os'] = data_df['device_browser'] + '_' + data_df['device_operatingSystem']\n    return data_df\n\ndef process_totals(data_df):\n    print(\"process totals ...\")\n    data_df['visitNumber'] = np.log1p(data_df['visitNumber'])\n    data_df['totals_hits'] = np.log1p(data_df['totals_hits'])\n    data_df['totals_pageviews'] = np.log1p(data_df['totals_pageviews'].fillna(0))\n    data_df['mean_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('mean')\n    data_df['sum_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('sum')\n    data_df['max_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('max')\n    data_df['min_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('min')\n    data_df['var_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('var')\n    data_df['mean_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('mean')\n    data_df['sum_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('sum')\n    data_df['max_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('max')\n    data_df['min_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('min')\n    data_df['max_pageviews_WoY'] = data_df.groupby(['weekofyear'])['totals_pageviews'].transform('max')\n    data_df['min_pageviews_WoY'] = data_df.groupby(['weekofyear'])['totals_pageviews'].transform('min')\n    data_df['hit_vs_views'] = data_df['totals_pageviews'] / data_df['totals_hits']\n    data_df['hit_vs_views_r'] = data_df['totals_hits'] / data_df['totals_pageviews']\n    return data_df\n\ndef process_geo_network(data_df):\n    print(\"process geo network ...\")\n    data_df['sum_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')\n    data_df['count_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')\n    data_df['mean_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('mean')\n    data_df['sum_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')\n    data_df['count_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')\n    data_df['mean_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('mean')\n    return data_df\n\ndef process_traffic_source(data_df):\n    print(\"process traffic source ...\")\n    data_df['source_country'] = data_df['trafficSource_source'] + '_' + data_df['geoNetwork_country']\n    data_df['campaign_medium'] = data_df['trafficSource_campaign'] + '_' + data_df['trafficSource_medium']\n    data_df['medium_hits_mean'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('mean')\n    data_df['medium_hits_max'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('max')\n    data_df['medium_hits_min'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('min')\n    data_df['medium_hits_sum'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('sum')\n    return data_df\n    \ndef custom(data):\n    print('custom..')\n    for i in ['geoNetwork_city', 'geoNetwork_continent', 'geoNetwork_country','geoNetwork_metro', 'geoNetwork_networkDomain', 'geoNetwork_region','geoNetwork_subContinent']:\n        for j in ['device_browser','device_deviceCategory', 'device_operatingSystem', 'trafficSource_source']:\n            data[i + \"_\" + j] = data[i] + \"_\" + data[j]\n\n    return data\n\n#Feature processing\n## Load data\nprint('reading train')\ntrain_df = read_parse_dataframe('train.csv')\ntrn_len = train_df.shape[0]\ntrain_df = process_date_time(train_df)\nprint('reading test')\ntest_df = read_parse_dataframe('test.csv')\ntest_df = process_date_time(test_df)\n\n## Drop columns\ncols_to_drop = [col for col in train_df.columns if train_df[col].nunique(dropna=False) == 1]\ntrain_df.drop(cols_to_drop, axis=1, inplace=True)\ntest_df.drop([col for col in cols_to_drop if col in test_df.columns], axis=1, inplace=True)\n\n###only one not null value\ntrain_df.drop(['trafficSource_campaignCode'], axis=1, inplace=True)\n\n###converting columns format\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].astype(float)\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].fillna(0)\ntrain_df['totals_transactionRevenue'] = np.log1p(train_df['totals_transactionRevenue'])\n\n\n## Features engineering\ntrain_df = process_format(train_df)\ntrain_df = process_device(train_df)\ntrain_df = process_totals(train_df)\ntrain_df = process_geo_network(train_df)\ntrain_df = process_traffic_source(train_df)\ntrain_df = custom(train_df)\n\ntest_df = process_format(test_df)\ntest_df = process_device(test_df)\ntest_df = process_totals(test_df)\ntest_df = process_geo_network(test_df)\ntest_df = process_traffic_source(test_df)\ntest_df = custom(test_df)\n\n## Categorical columns\nprint(\"process categorical columns ...\")\nnum_cols = ['month_unique_user_count', 'day_unique_user_count', 'weekday_unique_user_count',\n            'visitNumber', 'totals_hits', 'totals_pageviews', \n            'mean_hits_per_day', 'sum_hits_per_day', 'min_hits_per_day', 'max_hits_per_day', 'var_hits_per_day',\n            'mean_pageviews_per_day', 'sum_pageviews_per_day', 'min_pageviews_per_day', 'max_pageviews_per_day',\n            'sum_pageviews_per_network_domain', 'count_pageviews_per_network_domain', 'mean_pageviews_per_network_domain',\n            'sum_hits_per_network_domain', 'count_hits_per_network_domain', 'mean_hits_per_network_domain',\n            'medium_hits_mean','medium_hits_min','medium_hits_max','medium_hits_sum','hit_vs_views','hit_vs_views_r',\n            'max_pageviews_WoY','min_pageviews_WoY']\n            \nnot_used_cols = [\"visitNumber\", \"date\", \"fullVisitorId\", \"sessionId\", \n        \"visitId\", \"visitStartTime\", 'totals_transactionRevenue', 'trafficSource_referralPath']\ncat_cols = [col for col in train_df.columns if col not in num_cols and col not in not_used_cols]\n\nfor col in cat_cols:\n    #print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))\n\nprint('FINAL train shape : ', train_df.shape, ' test shape : ', test_df.shape)\n#print(train_df.columns)\ntrain_df = train_df.sort_values('date')\nX = train_df.drop(not_used_cols, axis=1)\ny = train_df['totals_transactionRevenue']\nX_test = test_df.drop([col for col in not_used_cols if col in test_df.columns], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0531a3a0a3c1243a7de1063f42a9031be0cac6d9"},"cell_type":"markdown","source":"## Step 1: parameters to be tuned"},{"metadata":{"trusted":true,"_uuid":"51836900b4bde0582405bcfff2a9cd3c2308f558"},"cell_type":"code","source":"import warnings\nimport time\nwarnings.filterwarnings(\"ignore\")\nimport lightgbm as lgb\nfrom bayes_opt import BayesianOptimization\nfrom sklearn.metrics import roc_auc_score, mean_squared_error\nfrom sklearn.cross_validation import KFold, cross_val_score\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fe0b510638e210f0673ce0996af327b70a03905"},"cell_type":"code","source":"params = {\n            \"objective\" : \"regression\", \"bagging_fraction\" : 0.8, \"bagging_freq\": 1,\n            \"min_child_samples\": 20, \"reg_alpha\": 1, \"reg_lambda\": 1,\"boosting\": \"rf\",\n            \"learning_rate\" : 0.01, \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \"verbosity\": -1, \"metric\" : 'rmse'\n        }\ntrain_data = lgb.Dataset(data=X, label=y, categorical_feature = list(X.columns),free_raw_data=False)\n    \ncv_result = lgb.cv(params, train_data, nfold=5, seed=0, verbose_eval =200,stratified=False,shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e16134b4907210c94936dc5e70576e25277d7b2"},"cell_type":"code","source":"-1.0 * np.array(cv_result['rmse-mean'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4400f761ef9118acbb7552144aad99e9bd7e51c6"},"cell_type":"code","source":"(-1.0 * np.array(cv_result['rmse-mean'])).max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ade594a014ebf4edfb8b538079d541c6aee3c74"},"cell_type":"code","source":"def lgb_eval(num_leaves, feature_fraction, max_depth , min_split_gain, min_child_weight):\n    params = {\n            \"objective\" : \"regression\", \"bagging_fraction\" : 0.8, \"bagging_freq\": 1,\n            \"min_child_samples\": 20, \"reg_alpha\": 1, \"reg_lambda\": 1,\"boosting\": \"rf\",\n            \"learning_rate\" : 0.01, \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \"verbosity\": -1, \"metric\" : 'rmse'\n        }\n    params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n    params['max_depth'] = int(round(max_depth))\n    params['num_leaves'] = int(round(num_leaves))\n    params['min_split_gain'] = min_split_gain\n    params['min_child_weight'] = min_child_weight\n    cv_result = lgb.cv(params, train_data, nfold=5, seed=0, verbose_eval =200,stratified=False)\n    return (-1.0 * np.array(cv_result['rmse-mean'])).max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"073605bb2e74b067fb275642902e9b2dbf830a6a"},"cell_type":"code","source":"lgbBO = BayesianOptimization(lgb_eval, {'feature_fraction': (0.1, 0.9),\n                                            'max_depth': (5, 9),\n                                            'num_leaves' : (200,300),\n                                            'min_split_gain': (0.001, 0.1),\n                                            'min_child_weight': (5, 50)}, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43aed27a056bcace6c9670a3022085bb680ed1c6"},"cell_type":"markdown","source":"## Step 2: minimize rmse"},{"metadata":{"trusted":true,"_uuid":"e1a91bd4796bf62adb6d36b049b07e462e045381"},"cell_type":"code","source":"lgbBO.maximize(init_points=5, n_iter=5,acq='ei')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9bd49ed272e215ed780cab58e2a503ebaaed4683"},"cell_type":"markdown","source":"## Step 3: minimize rmse"},{"metadata":{"trusted":true,"_uuid":"f58976af0a8922bd5bdc1cd924ba9c2828c6a6f5","scrolled":true},"cell_type":"code","source":"def bayes_parameter_opt_lgb(X, y, init_round=15, opt_round=25, n_folds=5, random_seed=0, n_estimators=10000, learning_rate=0.05, output_process=False):\n    # prepare data\n    train_data = lgb.Dataset(data=X, label=y, categorical_feature = list(X.columns),free_raw_data=False)\n    # parameters\n\n    def lgb_eval(num_leaves, feature_fraction, max_depth , min_split_gain, min_child_weight):\n        params = {\n            \"objective\" : \"regression\", \"bagging_fraction\" : 0.8, \"bagging_freq\": 1,\n            \"min_child_samples\": 20, \"reg_alpha\": 1, \"reg_lambda\": 1,\"boosting\": \"rf\",\n            \"learning_rate\" : 0.01, \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \"verbosity\": -1, \"metric\" : 'rmse'\n        }\n        params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n        params['max_depth'] = int(round(max_depth))\n        params['num_leaves'] = int(round(num_leaves))\n        params['min_split_gain'] = min_split_gain\n        params['min_child_weight'] = min_child_weight\n        cv_result = lgb.cv(params, train_data, nfold=n_folds, seed=random_seed, verbose_eval =200,stratified=False)\n        return (-1.0 * np.array(cv_result['rmse-mean'])).max()\n    \n        # range \n    lgbBO = BayesianOptimization(lgb_eval, {'feature_fraction': (0.1, 0.9),\n                                            'max_depth': (5, 9),\n                                            'num_leaves' : (200,300),\n                                            'min_split_gain': (0.001, 0.1),\n                                            'min_child_weight': (5, 50)}, random_state=0)\n        # optimize\n    lgbBO.maximize(init_points=init_round, n_iter=opt_round,acq='ei')\n\n        # output optimization process\n    if output_process==True: lgbBO.points_to_csv(\"bayes_opt_result.csv\")\n\n        # return best parameters\n    return lgbBO.res['max']['max_params']\n\nopt_params = bayes_parameter_opt_lgb(X, y, init_round=10, opt_round=10, n_folds=5, random_seed=0, n_estimators=1000, learning_rate=0.01)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfc5167ea391d4044bbafd0b7aa9fe0f7755bc48"},"cell_type":"code","source":"print(opt_params)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d04926f73d5716dbe9c3643783c10cd4e8bfa577"},"cell_type":"markdown","source":"## STEP 3: Check Result1\nprashantkikani kenel (https://www.kaggle.com/prashantkikani/rstudio-lgb-single-model-lb1-6607) "},{"metadata":{"trusted":true,"_uuid":"d5ab3e9ffcc4472c1bfe763879cda2eac52b0df2"},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn import model_selection, preprocessing, metrics\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nlgb_params = {'num_leaves': 201,\n             'min_data_in_leaf': 30, \n             'objective':'regression',\n             'max_depth': 9,\n             'learning_rate': 0.01,\n             \"min_child_samples\": 50,\n             \"boosting\": \"rf\",\n             \"feature_fraction\": 0.7941654378759639,\n             \"min_split_gain\" : 0.025580954346285312,\n             \"min_child_weight\" : 49.94864050157583,\n             \"bagging_freq\": 1,\n             \"bagging_fraction\": 0.8,\n             \"bagging_seed\": 11,\n             \"metric\": 'rmse',\n             \"lambda_l1\": 1,\n             \"verbosity\": -1\n             }\n             \nlgb_params1 = {\"objective\" : \"regression\", \"metric\" : \"rmse\", \n               \"feature_fraction\": 0.7941654378759639,\n               \"min_split_gain\" : 0.025580954346285312,\n               \"min_child_weight\" : 49.94864050157583,\n               \"max_depth\": 9, \"min_child_samples\": 20, \"reg_alpha\": 1, \"reg_lambda\": 1,\n               \"num_leaves\" : 201, \"learning_rate\" : 0.01, \"subsample\" : 0.8, \n               \"colsample_bytree\" : 0.8, \"verbosity\": -1}\n\nrun_lgb = True\nprint('LGB : ', run_lgb)\n# modeling\n#--------------------------------------------------------------------------\nif run_lgb:\n    import lightgbm as lgb\n    def kfold_lgb_xgb():\n        folds = KFold(n_splits=5, shuffle=True, random_state=7)\n        \n        oof_lgb = np.zeros(len(train_df))\n        predictions_lgb = np.zeros(len(test_df))\n\n        features_lgb = list(X.columns)\n        feature_importance_df_lgb = pd.DataFrame()\n\n        for fold_, (trn_idx, val_idx) in enumerate(folds.split(X)):\n            trn_data = lgb.Dataset(X.iloc[trn_idx], label=y.iloc[trn_idx])\n            val_data = lgb.Dataset(X.iloc[val_idx], label=y.iloc[val_idx])\n            \n            print(\"LGB \" + str(fold_) + \"-\" * 50)\n            num_round = 20000\n            clf = lgb.train(lgb_params1, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)\n            oof_lgb[val_idx] = clf.predict(X.iloc[val_idx], num_iteration=clf.best_iteration)\n            \n            fold_importance_df_lgb = pd.DataFrame()\n            fold_importance_df_lgb[\"feature\"] = features_lgb\n            fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n            fold_importance_df_lgb[\"fold\"] = fold_ + 1\n            feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n            predictions_lgb += clf.predict(X_test, num_iteration=clf.best_iteration) / folds.n_splits\n        \n        #lgb.plot_importance(clf, max_num_features=30)    \n        cols = feature_importance_df_lgb[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:50].index\n        best_features_lgb = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\n        plt.figure(figsize=(14,10))\n        sns.barplot(x=\"importance\", y=\"feature\", data=best_features_lgb.sort_values(by=\"importance\", ascending=False))\n        plt.title('LightGBM Features (avg over folds)')\n        plt.tight_layout()\n        plt.savefig('lgbm_importances.png')\n        x = []\n        for i in oof_lgb:\n            if i < 0:\n                x.append(0.0)\n            else:\n                x.append(i)\n        cv_lgb = mean_squared_error(x, y)**0.5\n        cv_lgb = str(cv_lgb)\n        cv_lgb = cv_lgb[:10]\n        \n        pd.DataFrame({'preds': x}).to_csv('lgb_oof_' + cv_lgb + '.csv', index = False)\n        \n        print(\"CV_LGB : \", cv_lgb)\n        return cv_lgb, predictions_lgb\n        \n    cv_lgb, lgb_ans = kfold_lgb_xgb()\n    x = []\n    for i in lgb_ans:\n        if i < 0:\n            x.append(0.0)\n        else:\n            x.append(i)\n    np.save('lgb_ans.npy', x)\n    submission = test_df[['fullVisitorId']].copy()\n    submission.loc[:, 'PredictedLogRevenue'] = x\n    submission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].apply(lambda x : 0.0 if x < 0 else x)\n    submission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].fillna(0.0)\n    grouped_test = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum().reset_index()\n    grouped_test.to_csv('lgb_' + cv_lgb + '.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a3fc0eccdbb4a4e010f92f151a9094cb8bc32f1"},"cell_type":"markdown","source":"# STEP 3 : Result check2  \nkernel : Stratified sampling for regression LB: 1.6595  \nhttps://www.kaggle.com/youhanlee/stratified-sampling-for-regression-lb-1-6595  "},{"metadata":{"trusted":true,"_uuid":"cf959d1ff4b42cfa93e3cef3d5bea12f61cd58d5"},"cell_type":"code","source":"def categorize_target(x):\n    if x < 2:\n        return 0\n    elif x < 4:\n        return 1\n    elif x < 6:\n        return 2\n    elif x < 8:\n        return 3\n    elif x < 10:\n        return 4\n    elif x < 12:\n        return 5\n    elif x < 14:\n        return 6\n    elif x < 16:\n        return 7\n    elif x < 18:\n        return 8\n    elif x < 20:\n        return 9\n    elif x < 22:\n        return 10\n    else:\n        return 11\n    \nfrom sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb\ny_categorized = y.apply(categorize_target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b09a4ce602616bce735afc5e42a8727e22e6abdb"},"cell_type":"code","source":"import lightgbm as lgb\ndef kfold_lgb_xgb():\n    FOLDs = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)\n\n    oof_lgb = np.zeros(len(train_df))\n    predictions_lgb = np.zeros(len(test_df))\n\n    features_lgb = list(X.columns)\n    feature_importance_df_lgb = pd.DataFrame()\n\n    for fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(X, y_categorized)):\n        trn_data = lgb.Dataset(X.iloc[trn_idx], label=y.iloc[trn_idx])\n        val_data = lgb.Dataset(X.iloc[val_idx], label=y.iloc[val_idx])\n\n        print(\"LGB \" + str(fold_) + \"-\" * 50)\n        num_round = 20000\n        clf = lgb.train(lgb_params1, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)\n        oof_lgb[val_idx] = clf.predict(X.iloc[val_idx], num_iteration=clf.best_iteration)\n\n        fold_importance_df_lgb = pd.DataFrame()\n        fold_importance_df_lgb[\"feature\"] = features_lgb\n        fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n        fold_importance_df_lgb[\"fold\"] = fold_ + 1\n        feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n        predictions_lgb += clf.predict(X_test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n\n    #lgb.plot_importance(clf, max_num_features=30)    \n    cols = feature_importance_df_lgb[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:50].index\n    best_features_lgb = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\n    plt.figure(figsize=(14,10))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features_lgb.sort_values(by=\"importance\", ascending=False))\n    plt.title('LightGBM Features (avg over folds)')\n    plt.tight_layout()\n    plt.savefig('lgbm_importances.png')\n    x = []\n    for i in oof_lgb:\n        if i < 0:\n            x.append(0.0)\n        else:\n            x.append(i)\n    cv_lgb = mean_squared_error(x, y)**0.5\n    cv_lgb = str(cv_lgb)\n    cv_lgb = cv_lgb[:10]\n\n    pd.DataFrame({'preds': x}).to_csv('lgb_oof_' + cv_lgb + '.csv', index = False)\n\n    print(\"CV_LGB : \", cv_lgb)\n    return cv_lgb, predictions_lgb\n\ncv_lgb, lgb_ans = kfold_lgb_xgb()\nx = []\nfor i in lgb_ans:\n    if i < 0:\n        x.append(0.0)\n    else:\n        x.append(i)\nnp.save('lgb_ans.npy', x)\nsubmission = test_df[['fullVisitorId']].copy()\nsubmission.loc[:, 'PredictedLogRevenue'] = x\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].apply(lambda x : 0.0 if x < 0 else x)\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].fillna(0.0)\ngrouped_test = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum().reset_index()\ngrouped_test.to_csv('Stratified_lgb_' + cv_lgb + '.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3eb0994abaa75fb4fe752c3ee172ce384d43b4d2"},"cell_type":"markdown","source":"## Result\n\n- https://www.kaggle.com/prashantkikani/rstudio-lgb-single-model-lb1-6607 is LB: 1.6607 **==>**\n\n- https://www.kaggle.com/youhanlee/stratified-sampling-for-regression-lb-1-6595 is LB : 1.6595. **==>**"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}