{"cells":[{"metadata":{"trusted":true,"_uuid":"d4c04e473bf95f4defe389197b3bb76f3bfda39c"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport h2o\nimport seaborn as sns\nimport json\n%matplotlib inline\nfrom pandas.io.json import json_normalize\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split, KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b0db29fde417a6c9ef493629b506ee68afe7946"},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\ndef rmse(y_true, y_pred):\n    return round(np.sqrt(mean_squared_error(y_true, y_pred)), 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3797c29c68aca841b2296f5cb0e7dd7bb997b23c"},"cell_type":"code","source":"def load_df(filename):\n    json_cols = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    df = pd.read_csv(filename, converters={column: json.loads for column in json_cols}, \n                     dtype={'fullVisitorId': 'str'})\n    \n    for column in json_cols:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b77656dcaf50eca32e176a527e2dee24159710c"},"cell_type":"code","source":"%%time\ntrain_df = load_df('../input/train.csv')\ntest_df = load_df('../input/test.csv')\ntrain_ind = train_df['fullVisitorId'].copy()\ntest_ind = test_df['fullVisitorId'].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7251d4fc001035e3238b780f855d26f64566b5fc"},"cell_type":"code","source":"def feature_design(df):\n    df['date'] = pd.to_datetime(df['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))\n    for col in ['visitNumber', 'totals.hits', 'totals.pageviews']:\n        df[col] = df[col].astype(float)\n        \n    df['month'] = df['date'].dt.month\n    df['day'] = df['date'].dt.day\n    df['weekday'] = df['date'].dt.weekday\n    df['weekofyear'] = df['date'].dt.weekofyear\n    \n    df['month_unique_user_count'] = df.groupby('month')['fullVisitorId'].transform('nunique')\n    df['day_unique_user_count'] = df.groupby('day')['fullVisitorId'].transform('nunique')\n    df['weekday_unique_user_count'] = df.groupby('weekday')['fullVisitorId'].transform('nunique')\n    df['weekofyear_unique_user_count'] = df.groupby('weekofyear')['fullVisitorId'].transform('nunique')\n    \n    df['browser_category'] = df['device.browser'] + '_' + df['device.deviceCategory']\n    df['browser_operatingSystem'] = df['device.browser'] + '_' + df['device.operatingSystem']\n    df['source_country'] = df['trafficSource.source'] + '_' + df['geoNetwork.country']\n    \n    df['visitNumber'] = np.log1p(df['visitNumber'])\n    df['totals.hits'] = np.log1p(df['totals.hits'])\n    df['totals.pageviews'] = np.log1p(df['totals.pageviews'].fillna(0))\n    \n    df['sum_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('sum')\n    df['count_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('count')\n    df['mean_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('mean')\n    df['sum_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('sum')\n    df['count_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('count')\n    df['mean_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('mean')\n    \n    df['mean_hits_per_day'] = df.groupby(['day'])['totals.hits'].transform('mean')\n    df['sum_hits_per_day'] = df.groupby(['day'])['totals.hits'].transform('median')\n\n    df['sum_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('sum')\n    df['count_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('median')\n    df['mean_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('mean')\n\n    df['sum_pageviews_per_region'] = df.groupby('geoNetwork.region')['totals.pageviews'].transform('sum')\n    df['count_pageviews_per_region'] = df.groupby('geoNetwork.region')['totals.pageviews'].transform('median')\n    df['mean_pageviews_per_region'] = df.groupby('geoNetwork.region')['totals.pageviews'].transform('mean')\n\n    df['sum_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('sum')\n    df['count_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('median')\n    df['mean_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('mean')\n\n    df['sum_hits_per_region'] = df.groupby('geoNetwork.region')['totals.hits'].transform('sum')\n    df['count_hits_per_region'] = df.groupby('geoNetwork.region')['totals.hits'].transform('median')\n    df['mean_hits_per_region'] = df.groupby('geoNetwork.region')['totals.hits'].transform('mean')\n\n    df['sum_hits_per_country'] = df.groupby('geoNetwork.country')['totals.hits'].transform('sum')\n    df['count_hits_per_country'] = df.groupby('geoNetwork.country')['totals.hits'].transform('median')\n    df['mean_hits_per_country'] = df.groupby('geoNetwork.country')['totals.hits'].transform('mean')\n\n    df['user_pageviews_sum'] = df.groupby('fullVisitorId')['totals.pageviews'].transform('sum')\n    df['user_hits_sum'] = df.groupby('fullVisitorId')['totals.hits'].transform('sum')\n\n    df['user_pageviews_count'] = df.groupby('fullVisitorId')['totals.pageviews'].transform('count')\n    df['user_hits_count'] = df.groupby('fullVisitorId')['totals.hits'].transform('count')\n\n    df['user_pageviews_sum_to_mean'] = df['user_pageviews_sum'] / df['user_pageviews_sum'].mean()\n    df['user_hits_sum_to_mean'] = df['user_hits_sum'] / df['user_hits_sum'].mean()\n\n    df['user_pageviews_to_region'] = df['user_pageviews_sum'] / df['mean_pageviews_per_region']\n    df['user_hits_to_region'] = df['user_hits_sum'] / df['mean_hits_per_region']\n    \n    useless_columns = ['sessionId', 'visitId', 'fullVisitorId', 'date', 'visitStartTime','user_pageviews_sum', 'user_hits_sum',\n                      'user_pageviews_count', 'user_hits_count']\n    df = df.drop(useless_columns, axis = 1)\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"88086dd0f15523825fbc2c591cafce3d286b237d"},"cell_type":"code","source":"target = train_df['totals.transactionRevenue'].fillna(0).astype(float)\ntarget = target.apply(lambda x: np.log(x) if x > 0 else x)\ndel train_df['totals.transactionRevenue']\n\ncolumns = [col for col in train_df.columns if train_df[col].nunique() > 1]\n\ntrain_df = train_df[columns].copy()\ntest_df = test_df[columns].copy()\n\ntrain_df =  feature_design(train_df)\ntest_df =  feature_design(test_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3efdfb639a98320a488a6c11a6c3b01e51da46e1"},"cell_type":"markdown","source":"Categorical columns"},{"metadata":{"trusted":false,"_uuid":"383dbf95cc327ac661b998c2b5aed48e36ec8d4d"},"cell_type":"code","source":"cat_cols = train_df.select_dtypes(exclude=['float64', 'int64']).columns\n\nfor col in cat_cols:\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"685fa39a42cdf07f2a7896652869413597a73961"},"cell_type":"markdown","source":"## LGBM"},{"metadata":{"trusted":false,"_uuid":"dcab74568bbda795533166581aa625b0f122810b"},"cell_type":"code","source":"import lightgbm","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7a59614a1a7d8c229c487f3b8d9492c4c192a7e5"},"cell_type":"code","source":"def lgb_train(X, y, test_X, params):\n    kf = KFold(n_splits=5, shuffle=True, random_state=2017)\n    pred_test = 0\n    pred_train = 0\n    for dev_index, val_index in kf.split(X):\n        train_x, valid_x = X.iloc[dev_index,:], X.iloc[val_index,:]\n        train_y, valid_y = y[dev_index], y[val_index]\n        lgtrain = lightgbm.Dataset(train_x, train_y,categorical_feature=list(cat_cols))\n        lgvalid = lightgbm.Dataset(valid_x, valid_y,categorical_feature=list(cat_cols))\n        model = lightgbm.train(params, lgtrain, 2000, valid_sets=[lgvalid], early_stopping_rounds=100, verbose_eval=100)\n        pred_test_iter = model.predict(test_X, num_iteration=model.best_iteration)\n        pred_test_iter[pred_test_iter<0]=0\n        pred_test+=pred_test_iter\n        pred_train_iter = model.predict(X, num_iteration=model.best_iteration)\n        pred_train_iter[pred_train_iter<0]=0\n        pred_train+=pred_train_iter\n    pred_test /= 5.\n    pred_train  /= 5.\n    return pred_test, pred_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4828609d21c85d0ed8952e80a4c08336eae6b037"},"cell_type":"code","source":"params_lgb = {'objective': 'regression', \n          'metric': 'rmse', \n          'num_leaves': 49, \n          'max_depth': 14, \n          'lambda_l2': 0.01931081461346337, \n          'lambda_l1': 0.007163878762237125, \n          'num_threads': 4, \n          'min_child_samples': 40, \n          'learning_rate': 0.01, \n          'bagging_fraction': 0.7910460446769023, \n          'feature_fraction': 0.5046791892199741, \n          'subsample_freq': 5, \n          'bagging_seed': 42, \n          'verbosity': -1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"01d2e61b2e08a89d6a09228e991f4320cdb677f9"},"cell_type":"code","source":"sub_lgb_test, sub_lgb_train = lgb_train(train_df, target, test_df, params_lgb)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"16acaac92afe8bbe85fcc0a18b0591d54e19261f"},"cell_type":"markdown","source":"## XGBoost"},{"metadata":{"trusted":false,"_uuid":"fb9c7c7b2c2cd9a47e06a3c356dfba57c043db8a"},"cell_type":"code","source":"import xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b7c9ecfc8495b8c5b504945bca816b1a1e7c7861"},"cell_type":"code","source":"def xgb_train(X, y, test_X, params):\n    kf = KFold(n_splits=5, shuffle=True, random_state=2017)\n    pred_test_xgb = 0\n    pred_train_xgb = 0\n    for dev_index, val_index in kf.split(train_df):\n        train_x, valid_x = X.loc[dev_index,:], X.loc[val_index,:]\n        train_y, valid_y = y[dev_index], y[val_index]\n        xgb_train_data = xgb.DMatrix(train_x, train_y)\n        xgb_val_data = xgb.DMatrix(valid_x, valid_y)\n        xgb_submit_data = xgb.DMatrix(test_X)\n        xgb_submit_data_train = xgb.DMatrix(X)\n        xgb_model = xgb.train(params, xgb_train_data, \n                          num_boost_round=2000, \n                          evals= [(xgb_train_data, 'train'), (xgb_val_data, 'valid')],\n                          early_stopping_rounds=100, \n                          verbose_eval=500\n                         )\n        pred_test = xgb_model.predict(xgb_submit_data, ntree_limit=xgb_model.best_ntree_limit)\n        pred_train = xgb_model.predict(xgb_submit_data_train, ntree_limit=xgb_model.best_ntree_limit)\n        pred_test[pred_test<0]=0\n        pred_train[pred_train<0]=0\n        pred_test_xgb += pred_test\n        pred_train_xgb += pred_train\n    pred_test_xgb /= 5.\n    pred_train_xgb /= 5.\n    return pred_test_xgb, pred_train_xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f94413dbdf004e589472455a28cbc71506705869"},"cell_type":"code","source":"params_xgb = {\n            'objective': 'reg:linear',\n            'eval_metric': 'rmse',\n            'eta': 0.001,\n            'max_depth': 7,\n            'gamma': 1.3250360141843498, \n            'min_child_weight': 13.0958516960316, \n            'max_delta_step': 8.88492863796954, \n            'subsample': 0.9864199446951019, \n            'colsample_bytree': 0.8376539278239742,\n            'subsample': 0.6,\n            'colsample_bytree': 0.8,\n            'alpha':0.001,\n            \"num_leaves\" : 40,\n            'random_state': 42,\n            'silent': True,\n            }","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ce5086ed3f2d2ce0ca419dd4ca8ae29277963915"},"cell_type":"code","source":"sub_xgb_test, sub_xgb_train = xgb_train(train_df, target, test_df, params_xgb)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a02d474767d6a360b08689a2126067e32d16078"},"cell_type":"markdown","source":"## CatBoost"},{"metadata":{"trusted":false,"_uuid":"0b1a8c4b29a21437d11777fb384d01e88c3da1c8"},"cell_type":"code","source":"from catboost import CatBoostRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bf227244ffc88d8ab37617b095e14acac54095e7"},"cell_type":"code","source":"def cat_train(X, y, test_X):\n    kf = KFold(n_splits=5, shuffle=True, random_state=2017)\n    pred_test_cat = 0\n    pred_train_cat = 0\n    for dev_index, val_index in kf.split(train_df):\n        train_x, valid_x = X.loc[dev_index,:], X.loc[val_index,:]\n        train_y, valid_y = y[dev_index], y[val_index]\n        model = CatBoostRegressor(iterations=1000,\n                             learning_rate=0.05,\n                             depth=10,\n                             eval_metric='RMSE',\n                             random_seed = 42,\n                             bagging_temperature = 0.2,\n                             od_type='Iter',\n                             metric_period = 50,\n                             od_wait=20)\n        model.fit(train_x, train_y, eval_set=(valid_x, valid_y),use_best_model=True,verbose=True, \n                  cat_features= [i for i in range(len(train_df.columns)) if train_df.columns[i] in cat_cols])\n        pred_test = model.predict(test_X)\n        pred_test[pred_test<0]=0\n        pred_test_cat += pred_test\n        pred_train = model.predict(X)\n        pred_train[pred_train<0]=0\n        pred_train_cat += pred_train\n    pred_test_cat /= 5.\n    pred_train_cat /= 5.\n    return pred_test_cat, pred_train_cat","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fc55da8d975d13014c696c4a4742cae7bbf1b97f"},"cell_type":"code","source":"sub_cat_test, sub_cat_train = cat_train(train_df, target, test_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"089a259ba490b4af8736e8c42713e0200eaaf963"},"cell_type":"markdown","source":"## Stacking"},{"metadata":{"trusted":false,"_uuid":"efe5e0edf4271a9b88c750bb23be1bb31bde85a4"},"cell_type":"code","source":"last = pd.DataFrame()\nlast['fullVisitorId'] = train_ind\nlast['lgbm'] = sub_lgb_train\nlast['xbm'] = sub_xgb_train\nlast['cat'] = sub_cat_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aec9ee850dbddb176315f3f83c2deffc77b35aa8"},"cell_type":"code","source":"last_test = pd.DataFrame()\nlast_test['fullVisitorId'] = test_ind\nlast_test['lgbm'] = sub_lgb_test\nlast_test['xbm'] = sub_xgb_test\nlast_test['cat'] = sub_cat_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e138c0fed60c57707da5ca58dc49315d17076136"},"cell_type":"code","source":"from sklearn.linear_model import Ridge\nmodel = Ridge().fit(last, target)\npred = model.predict(last_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f5f4812bd74bf01044bf6b9f5adde641d0fc9869"},"cell_type":"code","source":"pred[pred<0] = 0\nsubmission = pd.DataFrame()\nsubmission['fullVisitorId'] = test_ind\nsubmission['PredictedLogRevenue'] = pred\nsubmission = submission.groupby('fullVisitorId').sum()['PredictedLogRevenue'].fillna(0).reset_index()\nsubmission.to_csv('submit_stack1.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e8d482be822a5dfcfe6cfe4c122917eb0d36fe9d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}