{"cells":[{"metadata":{"trusted":true,"_uuid":"79ee72e47d2440d6f5d620cba83e3a482d3d1362"},"cell_type":"markdown","source":"So happy can share you all.\nThanks to previous kernel\nhttps://www.kaggle.com/ashishpatel26/future-is-here\nhttps://www.kaggle.com/youhanlee/stratified-sampling-for-regression-lb-1-4627\n\nI just modify the next_session feature and added to next kernel."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir('../input'))\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport time\nfrom datetime import datetime\nimport gc\nimport psutil\nfrom sklearn.preprocessing import LabelEncoder\n\nPATH=\"../input/ga-customer-revenue-prediction/\"\nNUM_ROUNDS = 20000\nVERBOSE_EVAL = 500\nSTOP_ROUNDS = 100\nN_SPLITS = 10\n\n #the columns that will be parsed to extract the fields from the jsons\ncols_to_parse = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\ndef read_parse_dataframe(file_name):\n    #full path for the data file\n    path = PATH + file_name\n    #read the data file, convert the columns in the list of columns to parse using json loader,\n    #convert the `fullVisitorId` field as a string\n    data_df = pd.read_csv(path, \n        converters={column: json.loads for column in cols_to_parse}, \n        dtype={'fullVisitorId': 'str'}, nrows=None)\n    #parse the json-type columns\n    for col in cols_to_parse:\n        #each column became a dataset, with the columns the fields of the Json type object\n        json_col_df = json_normalize(data_df[col])\n        json_col_df.columns = [f\"{col}_{sub_col}\" for sub_col in json_col_df.columns]\n        #we drop the object column processed and we add the columns created from the json fields\n        data_df = data_df.drop(col, axis=1).merge(json_col_df, right_index=True, left_index=True)\n    return data_df\n    \ndef process_date_time(data_df):\n    print(\"process date time ...\")\n    data_df['date'] = data_df['date'].astype(str)\n    data_df[\"date\"] = data_df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n    data_df[\"date\"] = pd.to_datetime(data_df[\"date\"])   \n    data_df[\"year\"] = data_df['date'].dt.year\n    data_df[\"month\"] = data_df['date'].dt.month\n    data_df[\"day\"] = data_df['date'].dt.day\n    data_df[\"weekday\"] = data_df['date'].dt.weekday\n    data_df['weekofyear'] = data_df['date'].dt.weekofyear\n    data_df['month_unique_user_count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day_unique_user_count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday_unique_user_count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n    \n    data_df['date'] = pd.to_datetime(data_df['visitStartTime'], unit='s')\n    data_df = data_df.sort_values('visitStartTime')\n    data_df['next_session_1'] = (data_df['visitStartTime'] - data_df[['fullVisitorId', 'visitStartTime']].groupby('fullVisitorId')['visitStartTime'].shift(1)\n    ).astype(np.float32)\n    data_df['next_session_1'] = data_df['next_session_1'].fillna(data_df['next_session_1'].max())\n    data_df['next_session_1'] = (data_df['next_session_1'] - data_df['next_session_1'].mean()) / (data_df['next_session_1'].max() - data_df['next_session_1'].min())\n    data_df['next_session_2'] = ( data_df['visitStartTime'] - data_df[['fullVisitorId', 'visitStartTime']].groupby('fullVisitorId')['visitStartTime'].shift(-1)\n    ).astype(np.float32)\n    data_df['next_session_2'] = data_df['next_session_2'].fillna(data_df['next_session_2'].min())\n    data_df['next_session_2'] = (data_df['next_session_2'] - data_df['next_session_2'].mean()) / (data_df['next_session_2'].max() - data_df['next_session_2'].min())\n#     print('next session 2 = ',data_df['next_session_2'])\n    return data_df\n\ndef process_format(data_df):\n    print(\"process format ...\")\n    for col in ['visitNumber', 'totals_hits', 'totals_pageviews']:\n        data_df[col] = data_df[col].astype(float)\n    data_df['trafficSource_adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\n    data_df['trafficSource_isTrueDirect'].fillna(False, inplace=True)\n    return data_df\n    \ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['browser_category'] = data_df['device_browser'] + '_' + data_df['device_deviceCategory']\n    data_df['browser_os'] = data_df['device_browser'] + '_' + data_df['device_operatingSystem']\n    return data_df\n\ndef process_totals(data_df):\n    print(\"process totals ...\")\n    data_df['visitNumber'] = np.log1p(data_df['visitNumber'])\n    data_df['totals_hits'] = np.log1p(data_df['totals_hits'])\n    data_df['totals_pageviews'] = np.log1p(data_df['totals_pageviews'].fillna(0))\n    data_df['mean_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('mean')\n    data_df['sum_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('sum')\n    data_df['max_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('max')\n    data_df['min_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('min')\n    data_df['var_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('var')\n    data_df['mean_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('mean')\n    data_df['sum_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('sum')\n    data_df['max_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('max')\n    data_df['min_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('min')    \n    return data_df\n\ndef process_geo_network(data_df):\n    print(\"process geo network ...\")\n    data_df['sum_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('sum')\n    data_df['count_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('count')\n    data_df['mean_pageviews_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_pageviews'].transform('mean')\n    data_df['sum_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('sum')\n    data_df['count_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('count')\n    data_df['mean_hits_per_network_domain'] = data_df.groupby('geoNetwork_networkDomain')['totals_hits'].transform('mean')\n    return data_df\n\ndef process_traffic_source(data_df):\n    print(\"process traffic source ...\")\n    data_df['source_country'] = data_df['trafficSource_source'] + '_' + data_df['geoNetwork_country']\n    data_df['campaign_medium'] = data_df['trafficSource_campaign'] + '_' + data_df['trafficSource_medium']\n    data_df['medium_hits_mean'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('mean')\n    data_df['medium_hits_max'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('max')\n    data_df['medium_hits_min'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('min')\n    data_df['medium_hits_sum'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('sum')\n    return data_df\n\n#Feature processing\n## Load data\nprint('reading train')\ntrain_df = read_parse_dataframe('train.csv')\ntrn_len = train_df.shape[0]\ntrain_df = process_date_time(train_df)\nprint('reading test')\ntest_df = read_parse_dataframe('test.csv')\ntest_df = process_date_time(test_df)\n\n\n## Drop columns\ncols_to_drop = [col for col in train_df.columns if train_df[col].nunique(dropna=False) == 1]\ntrain_df.drop(cols_to_drop, axis=1, inplace=True)\ntest_df.drop([col for col in cols_to_drop if col in test_df.columns], axis=1, inplace=True)\n\n###only one not null value\ntrain_df.drop(['trafficSource_campaignCode'], axis=1, inplace=True)\n\n###converting columns format\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].astype(float)\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].fillna(0)\n# train_df['totals_transactionRevenue'] = np.log1p(train_df['totals_transactionRevenue'])\n\n\n## Features engineering\ntrain_df = process_format(train_df)\ntrain_df = process_device(train_df)\ntrain_df = process_totals(train_df)\ntrain_df = process_geo_network(train_df)\ntrain_df = process_traffic_source(train_df)\n\ntest_df = process_format(test_df)\ntest_df = process_device(test_df)\ntest_df = process_totals(test_df)\ntest_df = process_geo_network(test_df)\ntest_df = process_traffic_source(test_df)\n\n## Categorical columns\nprint(\"process categorical columns ...\")\nnum_cols = ['month_unique_user_count', 'day_unique_user_count', 'weekday_unique_user_count',\n            'visitNumber', 'totals_hits', 'totals_pageviews', \n            'mean_hits_per_day', 'sum_hits_per_day', 'min_hits_per_day', 'max_hits_per_day', 'var_hits_per_day',\n            'mean_pageviews_per_day', 'sum_pageviews_per_day', 'min_pageviews_per_day', 'max_pageviews_per_day',\n            'sum_pageviews_per_network_domain', 'count_pageviews_per_network_domain', 'mean_pageviews_per_network_domain',\n            'sum_hits_per_network_domain', 'count_hits_per_network_domain', 'mean_hits_per_network_domain',\n            'medium_hits_mean','medium_hits_min','medium_hits_max','medium_hits_sum']\n            \nnot_used_cols = [\"visitNumber\", \"date\", \"fullVisitorId\", \"sessionId\", \n        \"visitId\", \"visitStartTime\", 'totals_transactionRevenue', 'trafficSource_referralPath']\ncat_cols = [col for col in train_df.columns if col not in num_cols and col not in not_used_cols]\n\nmerged_df = pd.concat([train_df, test_df])\nprint('Cat columns : ', len(cat_cols))\nohe_cols = []\nfor i in cat_cols:\n    if len(set(merged_df[i].values)) < 100:\n        ohe_cols.append(i)\n\nprint('ohe_cols : ', ohe_cols)\nprint(len(ohe_cols))\nmerged_df = pd.get_dummies(merged_df, columns = ohe_cols)\ntrain_df = merged_df[:trn_len]\ntest_df = merged_df[trn_len:]\ndel merged_df\ngc.collect()\n\nfor col in cat_cols:\n    if col in ohe_cols:\n        continue\n    #print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))\n\nprint('FINAL train shape : ', train_df.shape, ' test shape : ', test_df.shape)\n#print(train_df.columns)\ntrain_df = train_df.sort_values('date')\nX = train_df.drop(not_used_cols, axis=1)\ny = train_df['totals_transactionRevenue']\nX_test = test_df.drop([col for col in not_used_cols if col in test_df.columns], axis=1)\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn import model_selection, preprocessing, metrics\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb\n             \nlgb_params1 = {\"objective\" : \"regression\", \"metric\" : \"rmse\", \n               \"max_depth\": 8, \"min_child_samples\": 20, \n               \"reg_alpha\": 1, \"reg_lambda\": 1,\n               \"num_leaves\" : 257, \"learning_rate\" : 0.01, \n               \"subsample\" : 0.8, \"colsample_bytree\" : 0.8, \n               \"verbosity\": -1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6dccdae3e2b32275a2c7d0e6aac0a47cebd2e39"},"cell_type":"code","source":"y_log = np.log1p(y)\n\ny_categorized= pd.cut(y_log, bins=range(0,25,3), include_lowest=True,right=False, labels=range(0,24,3)) # Thanks to Vitaly Portnoy\nFOLDs = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)\n\noof_lgb = np.zeros(len(train_df))\npredictions_lgb = np.zeros(len(test_df))\n\nfeatures_lgb = list(X.columns)\nfeature_importance_df_lgb = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(X, y_categorized)):\n    trn_data = lgb.Dataset(X.iloc[trn_idx], label=y_log.iloc[trn_idx])\n    val_data = lgb.Dataset(X.iloc[val_idx], label=y_log.iloc[val_idx])\n\n    print(\"LGB \" + str(fold_) + \"-\" * 50)\n    num_round = 20000\n    clf = lgb.train(lgb_params1, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 100)\n    oof_lgb[val_idx] = clf.predict(X.iloc[val_idx], num_iteration=clf.best_iteration)\n\n    fold_importance_df_lgb = pd.DataFrame()\n    fold_importance_df_lgb[\"feature\"] = features_lgb\n    fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n    fold_importance_df_lgb[\"fold\"] = fold_ + 1\n    feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n    predictions_lgb += clf.predict(X_test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n    \n#lgb.plot_importance(clf, max_num_features=30)    \ncols = feature_importance_df_lgb[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:50].index\nbest_features_lgb = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\nplt.figure(figsize=(14,10))\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features_lgb.sort_values(by=\"importance\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')\nx = []\nfor i in oof_lgb:\n    if i < 0:\n        x.append(0.0)\n    else:\n        x.append(i)\ncv_lgb = mean_squared_error(x, y_log)**0.5\ncv_lgb = str(cv_lgb)\ncv_lgb = cv_lgb[:10]\n\npd.DataFrame({'preds': x}).to_csv('lgb_oof_' + cv_lgb + '.csv', index = False)\n\nprint(\"CV_LGB : \", cv_lgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d333891c3bc6a7ced55440c2766c73861dbbbc63"},"cell_type":"code","source":"sub_df = test_df[['fullVisitorId']].copy()\noof_lgb[oof_lgb<0] = 0\npredictions_lgb[predictions_lgb<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(predictions_lgb)\n\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\n\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"897c4a0d49909d2ea1cb32c3de3fb9b22693b52d"},"cell_type":"code","source":"train_df['predictions'] = np.expm1(oof_lgb)\ntest_df['predictions'] = np.expm1(predictions_lgb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db2a4088bb89e2bf236b05c8fd053f7032e87c8a"},"cell_type":"code","source":"train_df.to_csv(\"temp_train.csv\", index=False)\ntest_df.to_csv(\"temp_test.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e57341329ab1e05d3cf86abe2b21c37109e7e4b"},"cell_type":"code","source":"train_df[['fullVisitorId','predictions']].to_csv(\"temp_train_predictions.csv\", index=False)\ntest_df[['fullVisitorId','predictions']].to_csv(\"temp_test_predictions.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1df090e2054f4c29a1a63df9542f77afedaf6e3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afe128105117dd6b3c67def4d9ce9d28cf1f4977"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}