{"cells":[{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0f23a70127f63b3d7de1f3b2540056031971a986"},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport time\nfrom datetime import datetime\nimport gc\nimport psutil\nfrom sklearn.preprocessing import LabelEncoder\n\nPATH=\"../input/\"\nNUM_ROUNDS = 20000\nVERBOSE_EVAL = 500\nSTOP_ROUNDS = 100\nN_SPLITS = 10\n\n #the columns that will be parsed to extract the fields from the jsons\ncols_to_parse = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\ndef read_parse_dataframe(file_name):\n    #full path for the data file\n    path = PATH + file_name\n    #read the data file, convert the columns in the list of columns to parse using json loader,\n    #convert the `fullVisitorId` field as a string\n    data_df = pd.read_csv(path, \n        converters={column: json.loads for column in cols_to_parse}, \n        dtype={'fullVisitorId': 'str'})\n    #parse the json-type columns\n    for col in cols_to_parse:\n        #each column became a dataset, with the columns the fields of the Json type object\n        json_col_df = json_normalize(data_df[col])\n        json_col_df.columns = [f\"{col}_{sub_col}\" for sub_col in json_col_df.columns]\n        #we drop the object column processed and we add the columns created from the json fields\n        data_df = data_df.drop(col, axis=1).merge(json_col_df, right_index=True, left_index=True)\n    return data_df\n    \ndef process_date_time(data_df):\n    print(\"process date time ...\")\n    data_df['date'] = data_df['date'].astype(str)\n    data_df[\"date\"] = data_df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n    data_df[\"date\"] = pd.to_datetime(data_df[\"date\"])   \n    data_df[\"year\"] = data_df['date'].dt.year\n    data_df[\"month\"] = data_df['date'].dt.month\n    data_df[\"day\"] = data_df['date'].dt.day\n    data_df[\"weekday\"] = data_df['date'].dt.weekday\n    data_df['weekofyear'] = data_df['date'].dt.weekofyear\n    data_df['month_unique_user_count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day_unique_user_count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday_unique_user_count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n\n    return data_df\n\ndef process_format(data_df):\n    print(\"process format ...\")\n    for col in ['visitNumber', 'totals_hits', 'totals_pageviews']:\n        data_df[col] = data_df[col].astype(float)\n    data_df['trafficSource_adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\n    data_df['trafficSource_isTrueDirect'].fillna(False, inplace=True)\n    return data_df\n    \ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['browser_category'] = data_df['device_browser'] + '_' + data_df['device_deviceCategory']\n    data_df['browser_os'] = data_df['device_browser'] + '_' + data_df['device_operatingSystem']\n    return data_df\n\ndef process_totals(data_df):\n    print(\"process totals ...\")\n    data_df['visitNumber'] = np.log1p(data_df['visitNumber'])\n    data_df['totals_hits'] = np.log1p(data_df['totals_hits'])\n    data_df['totals_pageviews'] = np.log1p(data_df['totals_pageviews'].fillna(0))\n    data_df['mean_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('mean')\n    data_df['sum_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('sum')\n    data_df['max_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('max')\n    data_df['min_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('min')\n    data_df['var_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('var')\n    data_df['mean_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('mean')\n    data_df['sum_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('sum')\n    data_df['max_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('max')\n    data_df['min_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('min')    \n    return data_df\n\ndef process_geo_network(data_df):\n    print(\"process geo network ...\")\n    data_df['sum_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')\n    data_df['count_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')\n    data_df['mean_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('mean')\n    data_df['sum_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')\n    data_df['count_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')\n    data_df['mean_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('mean')\n    return data_df\n\ndef process_traffic_source(data_df):\n    print(\"process traffic source ...\")\n    data_df['source_country'] = data_df['trafficSource_source'] + '_' + data_df['geoNetwork_country']\n    data_df['campaign_medium'] = data_df['trafficSource_campaign'] + '_' + data_df['trafficSource_medium']\n    data_df['medium_hits_mean'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('mean')\n    data_df['medium_hits_max'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('max')\n    data_df['medium_hits_min'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('min')\n    data_df['medium_hits_sum'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('sum')\n    return data_df\n\n#Feature processing\n## Load data\nprint('reading train')\ntrain_df = read_parse_dataframe('train.csv')\ntrn_len = train_df.shape[0]\ntrain_df = process_date_time(train_df)\nprint('reading test')\ntest_df = read_parse_dataframe('test.csv')\ntest_df = process_date_time(test_df)\n\n## Drop columns\ncols_to_drop = [col for col in train_df.columns if train_df[col].nunique(dropna=False) == 1]\ntrain_df.drop(cols_to_drop, axis=1, inplace=True)\ntest_df.drop([col for col in cols_to_drop if col in test_df.columns], axis=1, inplace=True)\n\n###only one not null value\ntrain_df.drop(['trafficSource_campaignCode'], axis=1, inplace=True)\n\n###converting columns format\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].astype(float)\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].fillna(0)\ntrain_df['totals_transactionRevenue'] = np.log1p(train_df['totals_transactionRevenue'])\n\n\n## Features engineering\ntrain_df = process_format(train_df)\ntrain_df = process_device(train_df)\ntrain_df = process_totals(train_df)\ntrain_df = process_geo_network(train_df)\ntrain_df = process_traffic_source(train_df)\n\ntest_df = process_format(test_df)\ntest_df = process_device(test_df)\ntest_df = process_totals(test_df)\ntest_df = process_geo_network(test_df)\ntest_df = process_traffic_source(test_df)\n\n## Categorical columns\nprint(\"process categorical columns ...\")\nnum_cols = ['month_unique_user_count', 'day_unique_user_count', 'weekday_unique_user_count',\n            'visitNumber', 'totals_hits', 'totals_pageviews', \n            'mean_hits_per_day', 'sum_hits_per_day', 'min_hits_per_day', 'max_hits_per_day', 'var_hits_per_day',\n            'mean_pageviews_per_day', 'sum_pageviews_per_day', 'min_pageviews_per_day', 'max_pageviews_per_day',\n            'sum_pageviews_per_network_domain', 'count_pageviews_per_network_domain', 'mean_pageviews_per_network_domain',\n            'sum_hits_per_network_domain', 'count_hits_per_network_domain', 'mean_hits_per_network_domain',\n            'medium_hits_mean','medium_hits_min','medium_hits_max','medium_hits_sum']\n            \nnot_used_cols = [\"visitNumber\", \"date\", \"fullVisitorId\", \"sessionId\", \n        \"visitId\", \"visitStartTime\", 'totals_transactionRevenue', 'trafficSource_referralPath']\ncat_cols = [col for col in train_df.columns if col not in num_cols and col not in not_used_cols]\n\nmerged_df = pd.concat([train_df, test_df])\nprint('Cat columns : ', len(cat_cols))\nohe_cols = []\nfor i in cat_cols:\n    if len(set(merged_df[i].values)) < 100:\n        ohe_cols.append(i)\n\nprint('ohe_cols : ', ohe_cols)\nprint(len(ohe_cols))\nmerged_df = pd.get_dummies(merged_df, columns = ohe_cols)\ntrain_df = merged_df[:trn_len]\ntest_df = merged_df[trn_len:]\ndel merged_df\ngc.collect()\n\nfor col in cat_cols:\n    if col in ohe_cols:\n        continue\n    #print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))\n\nprint('FINAL train shape : ', train_df.shape, ' test shape : ', test_df.shape)\n#print(train_df.columns)\ntrain_df = train_df.sort_values('date')\nX = train_df.drop(not_used_cols, axis=1)\ny = train_df['totals_transactionRevenue']\nX_test = test_df.drop([col for col in not_used_cols if col in test_df.columns], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"630c6c166024eba28579e42db8582f1e2a7b533a"},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn import model_selection, preprocessing, metrics\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b8eec5a8a9691e46adfa23fb3cd585047a83884c"},"cell_type":"markdown","source":"# Stratified sampling"},{"metadata":{"trusted":true,"_uuid":"b01b3916d2a7007b4a42d67cdecd4377a17cadb7"},"cell_type":"code","source":"def categorize_target(x):\n    if x < 2:\n        return 0\n    elif x < 4:\n        return 1\n    elif x < 6:\n        return 2\n    elif x < 8:\n        return 3\n    elif x < 10:\n        return 4\n    elif x < 12:\n        return 5\n    elif x < 14:\n        return 6\n    elif x < 16:\n        return 7\n    elif x < 18:\n        return 8\n    elif x < 20:\n        return 9\n    elif x < 22:\n        return 10\n    else:\n        return 11","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b091a85718e820a964f5d847250a812d9832a223"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\ny_categorized = y.apply(categorize_target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff2001e389e33f0b403c80d002984e4de9ca3b85"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train,X_valid, y_train, y_valid = train_test_split(X,y,test_size=0.15, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7de4c916dd133d396ec8c56aba8a06d48a04d548"},"cell_type":"code","source":"print(f\"X_train shape {X_train.shape}\")\nprint(f\"y_train shape {y_train.shape}\")\nprint(f\"X_valid shape {X_valid.shape}\")\nprint(f\"y_valid shape {y_valid.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a92bfb0b25d7170b8bbb0eb0da3a8ddc17e3660"},"cell_type":"markdown","source":"# Bayesian Optimization"},{"metadata":{"trusted":true,"_uuid":"bb2abb78ea69d01087302758dd97e6207982c8b6"},"cell_type":"code","source":"from skopt.space import Real, Integer\nfrom skopt.utils import use_named_args\nimport itertools\nfrom math import sqrt\nfrom sklearn.metrics import mean_squared_error\nfrom skopt import gp_minimize\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0547d497bcea2b268457adfca02499bce23e006"},"cell_type":"code","source":"space  = [Integer(3, 10, name='max_depth'),\n          Integer(6, 50, name='num_leaves'),\n          Integer(50, 200, name='min_child_samples'),\n          Real(1, 400,  name='scale_pos_weight'),\n          Real(0.6, 0.9, name='subsample'),\n          Real(0.6, 0.9, name='colsample_bytree'),\n          Real(10**-3, 10**0, \"log-uniform\", name='learning_rate'),\n          Integer(100, 256, name='max_bin'),\n          Real(0.1,0.9, name='feature_fraction'),\n          Real(0.5, 1, name='bagging_fraction'),\n          Integer(20,100, name='min_data_in_leaf'),\n          Integer(0, 6, name='bagging_freq')\n         ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5a6adc9edfe97df3136f8479f5fe3397638cb23"},"cell_type":"code","source":"def objective(values):\n    \n    params = {'max_depth': values[0], \n          'num_leaves': values[1], \n          'min_child_samples': values[2], \n          'scale_pos_weight': values[3],\n            'subsample': values[4],\n            'colsample_bytree': values[5],\n            'learning_rate':values[6],\n            'max_bin': values[7],\n            'feature_fraction':values[8],\n            'bagging_fraction':values[9],\n            'min_data_in_leaf':values[10],\n            'bagging_freq':values[11],\n             'metric':'rmse',\n             'nthread': 8,\n             'boosting_type': 'gbdt',\n             'objective': 'regression',\n             'min_child_weight': 0,\n             'min_split_gain': 0,\n             'subsample_freq': 1}\n\n    print('\\nNext set of params.....',params)\n\n    \n    early_stopping_rounds = 50\n    num_boost_round       = 2000\n    xgtrain = lgb.Dataset(X_train, label=y_train)\n    xgvalid = lgb.Dataset(X_valid, label=y_valid)\n    \n    evals_results = {}\n    model_lgb     = lgb.train(params,xgtrain,valid_sets=[xgtrain, xgvalid], \n                              valid_names=['train','valid'], \n                               evals_result=evals_results, \n                               num_boost_round=num_boost_round,\n                                early_stopping_rounds=early_stopping_rounds,\n                               verbose_eval=None, feval=None)\n    \n    rmse = sqrt(mean_squared_error(y_valid,model_lgb.predict(X_valid)))\n    \n    print('\\nRMSE.....',rmse,\".....iter.....\", model_lgb.current_iteration())\n    \n    gc.collect()\n    \n    return  rmse\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d256949778cc017e0974c6789112bde4790b0637"},"cell_type":"code","source":"res_gp = gp_minimize(objective, space, n_calls=50,\n                     random_state=0,n_random_starts=10)\n\nprint(\"Best score=%.4f\" % res_gp.fun)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"372724db9ccb10113f84136891f9dc988b76f83f"},"cell_type":"code","source":"print(\"\"\"Best parameters:\n- max_depth=%d\n- num_leaves=%d\n- min_child_samples=%d\n- scale_pos_weight=%.4f\n- subsample=%.4f\n- colsample_bytree=%.4f\n- learning_rate=%.4f\n- max_bin=%d\n- feature_fraction=%.2f\n- bagging_fraction=%.2f\n- min_data_in_leaf=%d\n- bagging_freq=%d\n    \"\"\" % (res_gp.x[0], res_gp.x[1], res_gp.x[2], res_gp.x[3], \n          res_gp.x[4], res_gp.x[5], res_gp.x[6], res_gp.x[7],\n          res_gp.x[8], res_gp.x[9], res_gp.x[10], res_gp.x[11]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4df995aefa5ac2cd73e51e9895b972d4c201c570"},"cell_type":"code","source":"\"Best score=%.4f\" % res_gp.fun","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e86958a9fa397fd3b81f21a35a2bc06347bd9cad"},"cell_type":"code","source":"lgb_params1 = {\"objective\" : \"regression\", \n               \"metric\" : \"rmse\", \n                'max_depth':8, \n               'num_leaves':50,\n               'min_child_samples':50,\n              'scale_pos_weight':400.0000,\n              'subsample':0.6000,\n              'colsample_bytree':0.9000,\n             'learning_rate':0.0146,\n             'max_bin':100,\n             'feature_fraction':0.55,\n             'bagging_fraction':0.50,\n             'min_data_in_leaf':100,\n             'bagging_freq':0}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cf3fe29791d41a40f1966df5c7ea702f991bd15"},"cell_type":"code","source":"def kfold_lgb_xgb():\n    FOLDs = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)\n\n    oof_lgb = np.zeros(len(train_df))\n    predictions_lgb = np.zeros(len(test_df))\n\n    features_lgb = list(X.columns)\n    feature_importance_df_lgb = pd.DataFrame()\n\n    for fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(X, y_categorized)):\n        trn_data = lgb.Dataset(X.iloc[trn_idx], label=y.iloc[trn_idx])\n        val_data = lgb.Dataset(X.iloc[val_idx], label=y.iloc[val_idx])\n\n        print(\"LGB \" + str(fold_) + \"-\" * 50)\n        num_round = 20000\n        clf = lgb.train(lgb_params1, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)\n        oof_lgb[val_idx] = clf.predict(X.iloc[val_idx], num_iteration=clf.best_iteration)\n\n        fold_importance_df_lgb = pd.DataFrame()\n        fold_importance_df_lgb[\"feature\"] = features_lgb\n        fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n        fold_importance_df_lgb[\"fold\"] = fold_ + 1\n        feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n        predictions_lgb += clf.predict(X_test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n\n    #lgb.plot_importance(clf, max_num_features=30)    \n    cols = feature_importance_df_lgb[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:50].index\n    best_features_lgb = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\n    plt.figure(figsize=(14,10))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features_lgb.sort_values(by=\"importance\", ascending=False))\n    plt.title('LightGBM Features (avg over folds)')\n    plt.tight_layout()\n    plt.savefig('lgbm_importances.png')\n    x = []\n    for i in oof_lgb:\n        if i < 0:\n            x.append(0.0)\n        else:\n            x.append(i)\n    cv_lgb = mean_squared_error(x, y)**0.5\n    cv_lgb = str(cv_lgb)\n    cv_lgb = cv_lgb[:10]\n\n    pd.DataFrame({'preds': x}).to_csv('lgb_oof_' + cv_lgb + '.csv', index = False)\n\n    print(\"CV_LGB : \", cv_lgb)\n    return cv_lgb, predictions_lgb\n\ncv_lgb, lgb_ans = kfold_lgb_xgb()\nx = []\nfor i in lgb_ans:\n    if i < 0:\n        x.append(0.0)\n    else:\n        x.append(i)\nnp.save('lgb_ans.npy', x)\nsubmission = test_df[['fullVisitorId']].copy()\nsubmission.loc[:, 'PredictedLogRevenue'] = x\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].apply(lambda x : 0.0 if x < 0 else x)\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].fillna(0.0)\ngrouped_test = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum().reset_index()\ngrouped_test.to_csv('lgb_' + cv_lgb + '.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a62cc5f912cbdcda5e68aea06cb13fba55ec6a2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}