{"cells":[{"metadata":{"_uuid":"3d981a603152fefdc26802952cd859bf887103ee"},"cell_type":"markdown","source":"###### Background\n- As you know that, the non-zero value is very few in target in this competition. It is like imbalance of target in classfication problem.\n- For solving the imbalance in classfication problem, we commonly use the \"stratifed sampling\".\n- For this cometition, we can simply apply the stratified sampling to get more well-distributed sampling for continuous target.\n- To compare the effect of this strategy, I forked the good kernel(https://www.kaggle.com/prashantkikani/rstudio-lgb-single-model-lb1-6607) and used same parameters, same random seeds.\n- I just change the sampling strategy. Ok, Let's see whether it works."},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0f23a70127f63b3d7de1f3b2540056031971a986"},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport time\nfrom datetime import datetime\nimport gc\nimport psutil\nfrom sklearn.preprocessing import LabelEncoder\n\nPATH=\"../input/\"\nNUM_ROUNDS = 20000\nVERBOSE_EVAL = 500\nSTOP_ROUNDS = 100\nN_SPLITS = 10\n\n #the columns that will be parsed to extract the fields from the jsons\ncols_to_parse = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\ndef read_parse_dataframe(file_name):\n    #full path for the data file\n    path = PATH + file_name\n    #read the data file, convert the columns in the list of columns to parse using json loader,\n    #convert the `fullVisitorId` field as a string\n    data_df = pd.read_csv(path, \n        converters={column: json.loads for column in cols_to_parse}, \n        dtype={'fullVisitorId': 'str'})\n    #parse the json-type columns\n    for col in cols_to_parse:\n        #each column became a dataset, with the columns the fields of the Json type object\n        json_col_df = json_normalize(data_df[col])\n        json_col_df.columns = [f\"{col}_{sub_col}\" for sub_col in json_col_df.columns]\n        #we drop the object column processed and we add the columns created from the json fields\n        data_df = data_df.drop(col, axis=1).merge(json_col_df, right_index=True, left_index=True)\n    return data_df\n    \ndef process_date_time(data_df):\n    print(\"process date time ...\")\n    data_df['date'] = data_df['date'].astype(str)\n    data_df[\"date\"] = data_df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n    data_df[\"date\"] = pd.to_datetime(data_df[\"date\"])   \n    data_df[\"year\"] = data_df['date'].dt.year\n    data_df[\"month\"] = data_df['date'].dt.month\n    data_df[\"day\"] = data_df['date'].dt.day\n    data_df[\"weekday\"] = data_df['date'].dt.weekday\n    data_df['weekofyear'] = data_df['date'].dt.weekofyear\n    data_df['month_unique_user_count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day_unique_user_count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday_unique_user_count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n\n    return data_df\n\ndef process_format(data_df):\n    print(\"process format ...\")\n    for col in ['visitNumber', 'totals_hits', 'totals_pageviews']:\n        data_df[col] = data_df[col].astype(float)\n    data_df['trafficSource_adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\n    data_df['trafficSource_isTrueDirect'].fillna(False, inplace=True)\n    return data_df\n    \ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['browser_category'] = data_df['device_browser'] + '_' + data_df['device_deviceCategory']\n    data_df['browser_os'] = data_df['device_browser'] + '_' + data_df['device_operatingSystem']\n    return data_df\n\ndef process_totals(data_df):\n    print(\"process totals ...\")\n    data_df['visitNumber'] = np.log1p(data_df['visitNumber'])\n    data_df['totals_hits'] = np.log1p(data_df['totals_hits'])\n    data_df['totals_pageviews'] = np.log1p(data_df['totals_pageviews'].fillna(0))\n    data_df['mean_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('mean')\n    data_df['sum_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('sum')\n    data_df['max_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('max')\n    data_df['min_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('min')\n    data_df['var_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('var')\n    data_df['mean_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('mean')\n    data_df['sum_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('sum')\n    data_df['max_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('max')\n    data_df['min_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('min')    \n    return data_df\n\ndef process_geo_network(data_df):\n    print(\"process geo network ...\")\n    data_df['sum_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')\n    data_df['count_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')\n    data_df['mean_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('mean')\n    data_df['sum_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')\n    data_df['count_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')\n    data_df['mean_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('mean')\n    return data_df\n\ndef process_traffic_source(data_df):\n    print(\"process traffic source ...\")\n    data_df['source_country'] = data_df['trafficSource_source'] + '_' + data_df['geoNetwork_country']\n    data_df['campaign_medium'] = data_df['trafficSource_campaign'] + '_' + data_df['trafficSource_medium']\n    data_df['medium_hits_mean'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('mean')\n    data_df['medium_hits_max'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('max')\n    data_df['medium_hits_min'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('min')\n    data_df['medium_hits_sum'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('sum')\n    return data_df\n\n#Feature processing\n## Load data\nprint('reading train')\ntrain_df = read_parse_dataframe('train.csv')\ntrn_len = train_df.shape[0]\ntrain_df = process_date_time(train_df)\nprint('reading test')\ntest_df = read_parse_dataframe('test.csv')\ntest_df = process_date_time(test_df)\n\n## Drop columns\ncols_to_drop = [col for col in train_df.columns if train_df[col].nunique(dropna=False) == 1]\ntrain_df.drop(cols_to_drop, axis=1, inplace=True)\ntest_df.drop([col for col in cols_to_drop if col in test_df.columns], axis=1, inplace=True)\n\n###only one not null value\ntrain_df.drop(['trafficSource_campaignCode'], axis=1, inplace=True)\n\n###converting columns format\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].astype(float)\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].fillna(0)\n# train_df['totals_transactionRevenue'] = np.log1p(train_df['totals_transactionRevenue'])\n\n\n## Features engineering\ntrain_df = process_format(train_df)\ntrain_df = process_device(train_df)\ntrain_df = process_totals(train_df)\ntrain_df = process_geo_network(train_df)\ntrain_df = process_traffic_source(train_df)\n\ntest_df = process_format(test_df)\ntest_df = process_device(test_df)\ntest_df = process_totals(test_df)\ntest_df = process_geo_network(test_df)\ntest_df = process_traffic_source(test_df)\n\n## Categorical columns\nprint(\"process categorical columns ...\")\nnum_cols = ['month_unique_user_count', 'day_unique_user_count', 'weekday_unique_user_count',\n            'visitNumber', 'totals_hits', 'totals_pageviews', \n            'mean_hits_per_day', 'sum_hits_per_day', 'min_hits_per_day', 'max_hits_per_day', 'var_hits_per_day',\n            'mean_pageviews_per_day', 'sum_pageviews_per_day', 'min_pageviews_per_day', 'max_pageviews_per_day',\n            'sum_pageviews_per_network_domain', 'count_pageviews_per_network_domain', 'mean_pageviews_per_network_domain',\n            'sum_hits_per_network_domain', 'count_hits_per_network_domain', 'mean_hits_per_network_domain',\n            'medium_hits_mean','medium_hits_min','medium_hits_max','medium_hits_sum']\n            \nnot_used_cols = [\"visitNumber\", \"date\", \"fullVisitorId\", \"sessionId\", \n        \"visitId\", \"visitStartTime\", 'totals_transactionRevenue', 'trafficSource_referralPath']\ncat_cols = [col for col in train_df.columns if col not in num_cols and col not in not_used_cols]\n\nmerged_df = pd.concat([train_df, test_df])\nprint('Cat columns : ', len(cat_cols))\nohe_cols = []\nfor i in cat_cols:\n    if len(set(merged_df[i].values)) < 100:\n        ohe_cols.append(i)\n\nprint('ohe_cols : ', ohe_cols)\nprint(len(ohe_cols))\nmerged_df = pd.get_dummies(merged_df, columns = ohe_cols)\ntrain_df = merged_df[:trn_len]\ntest_df = merged_df[trn_len:]\ndel merged_df\ngc.collect()\n\nfor col in cat_cols:\n    if col in ohe_cols:\n        continue\n    #print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))\n\nprint('FINAL train shape : ', train_df.shape, ' test shape : ', test_df.shape)\n#print(train_df.columns)\ntrain_df = train_df.sort_values('date')\nX = train_df.drop(not_used_cols, axis=1)\ny = train_df['totals_transactionRevenue']\nX_test = test_df.drop([col for col in not_used_cols if col in test_df.columns], axis=1)\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn import model_selection, preprocessing, metrics\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n             \nlgb_params1 = {\"objective\" : \"regression\", \"metric\" : \"rmse\", \n               \"max_depth\": 8, \"min_child_samples\": 20, \n               \"reg_alpha\": 1, \"reg_lambda\": 1,\n               \"num_leaves\" : 257, \"learning_rate\" : 0.01, \n               \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \n               \"verbosity\": -1}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2d62248f690b08c77a1ddbcfdfbf03513bad7f4"},"cell_type":"markdown","source":"# Stratified sampling"},{"metadata":{"_uuid":"f0c32f82633dd6287349ddd725abae02158603c3"},"cell_type":"markdown","source":"- Before stratified samling, we need to pseudo-label for continous target.\n- In this case, I categorize the continous target into 12 class using range of 2."},{"metadata":{"trusted":true,"_uuid":"8dfccb015381cf6828b10ef7e2c48d2152fc3831"},"cell_type":"code","source":"# def categorize_target(x):\n#     if x < 2:\n#         return 0\n#     elif x < 4:\n#         return 1\n#     elif x < 6:\n#         return 2\n#     elif x < 8:\n#         return 3\n#     elif x < 10:\n#         return 4\n#     elif x < 12:\n#         return 5\n#     elif x < 14:\n#         return 6\n#     elif x < 16:\n#         return 7\n#     elif x < 18:\n#         return 8\n#     elif x < 20:\n#         return 9\n#     elif x < 22:\n#         return 10\n#     else:\n#         return 11","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c98ea0d0db32157d2fad609c31d3e47d60259bd"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d97d0a2c727b0bae59d90381eb29d394441d3286"},"cell_type":"markdown","source":"## Target, prediction process\n\n- 1st log1p to target\n- 2nd exmp1 predictions\n- 3rd sum predictions\n- 4th log1p to sum"},{"metadata":{"trusted":true,"_uuid":"f304e06349f97c07ce89422d8ac8387b79322e4c"},"cell_type":"code","source":"# y_categorized = y.apply(categorize_target)\ny_log = np.log1p(y)\n\ny_categorized= pd.cut(y_log, bins=range(0,25,3), include_lowest=True,right=False, labels=range(0,24,3)) # Thanks to Vitaly Portnoy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"021e01170966134e1da95bd893f64b8eb537cc7f"},"cell_type":"code","source":"FOLDs = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)\n\noof_lgb = np.zeros(len(train_df))\npredictions_lgb = np.zeros(len(test_df))\n\nfeatures_lgb = list(X.columns)\nfeature_importance_df_lgb = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(X, y_categorized)):\n    trn_data = lgb.Dataset(X.iloc[trn_idx], label=y_log.iloc[trn_idx])\n    val_data = lgb.Dataset(X.iloc[val_idx], label=y_log.iloc[val_idx])\n\n    print(\"LGB \" + str(fold_) + \"-\" * 50)\n    num_round = 20000\n    clf = lgb.train(lgb_params1, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)\n    oof_lgb[val_idx] = clf.predict(X.iloc[val_idx], num_iteration=clf.best_iteration)\n\n    fold_importance_df_lgb = pd.DataFrame()\n    fold_importance_df_lgb[\"feature\"] = features_lgb\n    fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n    fold_importance_df_lgb[\"fold\"] = fold_ + 1\n    feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n    predictions_lgb += clf.predict(X_test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n    \n#lgb.plot_importance(clf, max_num_features=30)    \ncols = feature_importance_df_lgb[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:50].index\nbest_features_lgb = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\nplt.figure(figsize=(14,10))\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features_lgb.sort_values(by=\"importance\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')\nx = []\nfor i in oof_lgb:\n    if i < 0:\n        x.append(0.0)\n    else:\n        x.append(i)\ncv_lgb = mean_squared_error(x, y_log)**0.5\ncv_lgb = str(cv_lgb)\ncv_lgb = cv_lgb[:10]\n\npd.DataFrame({'preds': x}).to_csv('lgb_oof_' + cv_lgb + '.csv', index = False)\n\nprint(\"CV_LGB : \", cv_lgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6a709f8c26bd74c44de0fedd4e949299f87fe01"},"cell_type":"code","source":"sub_df = test_df[['fullVisitorId']].copy()\npredictions_lgb[predictions_lgb<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(predictions_lgb)\n\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\n\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ee47937e75c9dedecf19ecc4e90be73e64b9c6c9"},"cell_type":"markdown","source":"* - My result is LB : 1.4627"},{"metadata":{"_uuid":"e30e889d018e31e4e51ae12949a4d6f38f940121"},"cell_type":"markdown","source":"# Conclusion"},{"metadata":{"_uuid":"5d633c359e47a084d1e4eec6ce002fcd9478269a"},"cell_type":"markdown","source":"- The improvement seems to be small, but you know that the small result can change  the medal winner.\n- This strategy would be improved using more category, etc.\n- How about using it? "}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}