{"cells":[{"metadata":{"_uuid":"5fec18d7c50bf659e1c2c53c53b718fb9d91c32f"},"cell_type":"markdown","source":"Thanks to this kernels!<br>\nhttps://www.kaggle.com/ashishpatel26/1-67-pb-first-try-to-think<br>\nFrom this kernel: https://www.kaggle.com/qwe1398775315/eda-lgbm-bayesianoptimization<br>\nhttps://www.kaggle.com/artgor/eda-on-basic-data-and-lgb-in-progress"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport h2o\nimport seaborn as sns\nimport json\n%matplotlib inline\nfrom pandas.io.json import json_normalize","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b76c5e7ccdf2c1befe75f1ee7a29596176d3a31d"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split, KFold","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"json_cols = ['device', 'geoNetwork', 'totals', 'trafficSource']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07578c77c5e3ed87b5f6b19fa2a628a2e6d03343"},"cell_type":"code","source":"def load_df(filename):\n    path = \"../input/\" + filename\n    df = pd.read_csv(path, converters={column: json.loads for column in json_cols}, \n                     dtype={'fullVisitorId': 'str'})\n    \n    for column in json_cols:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3c1180b488c9db272640526cc07c337bbfb4303"},"cell_type":"code","source":"%%time\ntrain_df = load_df('train.csv')\ntest_df = load_df('test.csv')\ntest_ind = test_df['fullVisitorId'].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"008d6b9e30618ee9cc19b18e9595717e523bee23"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"752ded2ca39ac8c6b4c23b88b59ca0a6badf2d93"},"cell_type":"code","source":"target = train_df['totals.transactionRevenue'].fillna(0).astype(float)\ntarget = target.apply(lambda x: np.log(x) if x > 0 else x)\ndel train_df['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66dc16ab966ea945f63e7e0b70d51f3abd44c504"},"cell_type":"code","source":"columns = [col for col in train_df.columns if train_df[col].nunique() > 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f590eecc91a559d0b85e2311631c7e75db6f2b43"},"cell_type":"code","source":"train_df = train_df[columns].copy()\ntest_df = test_df[columns].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0d431bfd00d5133ad21b7871be8c03c505141755"},"cell_type":"code","source":"# some data processing\ntrain_df['date'] = pd.to_datetime(train_df['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))\ntest_df['date'] = pd.to_datetime(test_df['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"092e51e7f07cbf1f30c453563f9c398c38a0871f"},"cell_type":"code","source":"useless_columns = ['sessionId', 'visitId']\ntrain_df = train_df.drop(useless_columns, axis = 1)\ntest_df = test_df.drop(useless_columns, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ad8203540391bdfd1d9c3c59815e13244c4d83b"},"cell_type":"code","source":"for col in ['visitNumber', 'totals.hits', 'totals.pageviews']:\n    train_df[col] = train_df[col].astype(float)\n    test_df[col] = test_df[col].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15a2431d2d10fcccce86b85cf81479e3ec91d46f"},"cell_type":"markdown","source":"**Revenue**"},{"metadata":{"trusted":true,"_uuid":"8028c7fb4eb1f632b5c710619d87a0241ef7e4b4"},"cell_type":"code","source":"sns.distplot(target[target!=0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8aea6f47628ea4c7bcbaabdbbb9a0af9df0c517"},"cell_type":"code","source":"def feature_design(df):\n    df['month'] = df['date'].dt.month\n    df['day'] = df['date'].dt.day\n    df['weekday'] = df['date'].dt.weekday\n    df['weekofyear'] = df['date'].dt.weekofyear\n    \n    df['month_unique_user_count'] = df.groupby('month')['fullVisitorId'].transform('nunique')\n    df['day_unique_user_count'] = df.groupby('day')['fullVisitorId'].transform('nunique')\n    df['weekday_unique_user_count'] = df.groupby('weekday')['fullVisitorId'].transform('nunique')\n    df['weekofyear_unique_user_count'] = df.groupby('weekofyear')['fullVisitorId'].transform('nunique')\n    \n    df['browser_category'] = df['device.browser'] + '_' + df['device.deviceCategory']\n    df['browser_operatingSystem'] = df['device.browser'] + '_' + df['device.operatingSystem']\n    df['source_country'] = df['trafficSource.source'] + '_' + df['geoNetwork.country']\n    \n    df['visitNumber'] = np.log1p(df['visitNumber'])\n    df['totals.hits'] = np.log1p(df['totals.hits'])\n    df['totals.pageviews'] = np.log1p(df['totals.pageviews'].fillna(0))\n    \n    df['sum_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('sum')\n    df['count_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('count')\n    df['mean_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('mean')\n    df['sum_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('sum')\n    df['count_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('count')\n    df['mean_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('mean')\n    \n    df['mean_hits_per_day'] = df.groupby(['day'])['totals.hits'].transform('mean')\n    df['sum_hits_per_day'] = df.groupby(['day'])['totals.hits'].transform('sum')\n\n    df['sum_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('sum')\n    df['count_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('count')\n    df['mean_pageviews_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('mean')\n\n    df['sum_pageviews_per_region'] = df.groupby('geoNetwork.region')['totals.pageviews'].transform('sum')\n    df['count_pageviews_per_region'] = df.groupby('geoNetwork.region')['totals.pageviews'].transform('count')\n    df['mean_pageviews_per_region'] = df.groupby('geoNetwork.region')['totals.pageviews'].transform('mean')\n\n    df['sum_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('sum')\n    df['count_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('count')\n    df['mean_hits_per_network_domain'] = df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('mean')\n\n    df['sum_hits_per_region'] = df.groupby('geoNetwork.region')['totals.hits'].transform('sum')\n    df['count_hits_per_region'] = df.groupby('geoNetwork.region')['totals.hits'].transform('count')\n    df['mean_hits_per_region'] = df.groupby('geoNetwork.region')['totals.hits'].transform('mean')\n\n    df['sum_hits_per_country'] = df.groupby('geoNetwork.country')['totals.hits'].transform('sum')\n    df['count_hits_per_country'] = df.groupby('geoNetwork.country')['totals.hits'].transform('count')\n    df['mean_hits_per_country'] = df.groupby('geoNetwork.country')['totals.hits'].transform('mean')\n\n    df['user_pageviews_sum'] = df.groupby('fullVisitorId')['totals.pageviews'].transform('sum')\n    df['user_hits_sum'] = df.groupby('fullVisitorId')['totals.hits'].transform('sum')\n\n    df['user_pageviews_count'] = df.groupby('fullVisitorId')['totals.pageviews'].transform('count')\n    df['user_hits_count'] = df.groupby('fullVisitorId')['totals.hits'].transform('count')\n\n    df['user_pageviews_sum_to_mean'] = df['user_pageviews_sum'] / df['user_pageviews_sum'].mean()\n    df['user_hits_sum_to_mean'] = df['user_hits_sum'] / df['user_hits_sum'].mean()\n\n    df['user_pageviews_to_region'] = df['user_pageviews_sum'] / df['mean_pageviews_per_region']\n    df['user_hits_to_region'] = df['user_hits_sum'] / df['mean_hits_per_region']\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14648fa2b08af347aaea39b5a432fc43224c6a29"},"cell_type":"code","source":"%%time\ntrain_df =  feature_design(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e57cdedd7451f1e14f48f3b1abe14ddc521938e9"},"cell_type":"code","source":"%%time\ntest_df =  feature_design(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9211cf18b8b0a88844a9d9414c465d0a3ec0b036"},"cell_type":"code","source":"train_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a4d648bd323f0987959cf8364143c159e4542e2"},"cell_type":"code","source":"train_df = train_df.drop(['date', 'visitStartTime', 'fullVisitorId'], axis = 1)\ntest_df = test_df.drop(['date', 'visitStartTime', 'fullVisitorId'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f02219e725dc4321f458b30c476f1c327fe22ef"},"cell_type":"markdown","source":"Categorical columns"},{"metadata":{"trusted":true,"_uuid":"b70c86553d84bab16c533c3547a37ff2e922c4d0"},"cell_type":"code","source":"cat_cols = train_df.select_dtypes(exclude=['float64', 'int64']).columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07d0315611ec2afa0894412250bff9d93905686c"},"cell_type":"code","source":"for col in cat_cols:\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd91fa47f58a853562debefc2c95e1bd7e37c950"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e099f3d2c77141364ada9a20b5cd5032ffaed681"},"cell_type":"markdown","source":"## Let's start - LGBM\nFrom this kernel: https://www.kaggle.com/qwe1398775315/eda-lgbm-bayesianoptimization"},{"metadata":{"trusted":true,"_uuid":"fba14e7d2a48629e8182d7df966c143694b9312a"},"cell_type":"code","source":"import lightgbm\nfrom bayes_opt import BayesianOptimization","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2cbf32463eaddfd98b81ac8f00d8585ec8099e3"},"cell_type":"code","source":"def rmsle(y, pred):\n    assert len(y) == len(pred)\n    return np.sqrt(np.mean(np.power(y-pred, 2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16917be8a21d7d4d5516d8db3a8cb4a8b2ccd57b"},"cell_type":"code","source":"def lgb_eval(num_leaves,max_depth,lambda_l2,lambda_l1,min_child_samples,bagging_fraction,feature_fraction):\n    params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\", \n    \"num_leaves\" : int(num_leaves),\n    \"max_depth\" : int(max_depth),\n    \"lambda_l2\" : lambda_l2,\n    \"lambda_l1\" : lambda_l1,\n    \"num_threads\" : 4,\n    \"min_child_samples\" : int(min_child_samples),\n    \"learning_rate\" : 0.03,\n    \"bagging_fraction\" : bagging_fraction,\n    \"feature_fraction\" : feature_fraction,\n    \"subsample_freq\" : 5,\n    \"bagging_seed\" : 42,\n    \"verbosity\" : -1\n    }\n    lgtrain = lightgbm.Dataset(train_df, target,categorical_feature=categorical_features)\n    cv_result = lightgbm.cv(params,\n                       lgtrain,\n                       10000,\n                       categorical_feature=categorical_features,\n                       early_stopping_rounds=100,\n                       stratified=False,\n                       nfold=5)\n    return -cv_result['rmse-mean'][-1]\n\ndef lgb_train(num_leaves,max_depth,lambda_l2,lambda_l1,min_child_samples,bagging_fraction,feature_fraction):\n    params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\", \n    \"num_leaves\" : int(num_leaves),\n    \"max_depth\" : int(max_depth),\n    \"lambda_l2\" : lambda_l2,\n    \"lambda_l1\" : lambda_l1,\n    \"num_threads\" : 4,\n    \"min_child_samples\" : int(min_child_samples),\n    \"learning_rate\" : 0.01,\n    \"bagging_fraction\" : bagging_fraction,\n    \"feature_fraction\" : feature_fraction,\n    \"subsample_freq\" : 5,\n    \"bagging_seed\" : 42,\n    \"verbosity\" : -1\n    }\n    t_x,v_x,t_y,v_y = train_test_split(train_df, target,test_size=0.2)\n    lgtrain = lightgbm.Dataset(t_x, t_y,categorical_feature=categorical_features)\n    lgvalid = lightgbm.Dataset(v_x, v_y,categorical_feature=categorical_features)\n    model = lightgbm.train(params, lgtrain, 2000, valid_sets=[lgvalid], early_stopping_rounds=100, verbose_eval=100)\n    pred_test_y = model.predict(test_df, num_iteration=model.best_iteration)\n    return pred_test_y, model\n    \ndef param_tuning(init_points,num_iter,**args):\n    lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (25, 50),\n                                                'max_depth': (5, 15),\n                                                'lambda_l2': (0.0, 0.05),\n                                                'lambda_l1': (0.0, 0.05),\n                                                'bagging_fraction': (0.5, 0.8),\n                                                'feature_fraction': (0.5, 0.8),\n                                                'min_child_samples': (20, 50),\n                                                })\n\n    lgbBO.maximize(init_points=init_points, n_iter=num_iter,**args)\n    return lgbBO","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4df9a7f6c88510a2949e79d02d28bf8d1647fb56"},"cell_type":"code","source":"categorical_features = list(cat_cols)\nresult = param_tuning(5,20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b2034a009883f53a459eeaa9491df4d38b3775d"},"cell_type":"code","source":"params = result.res['max']['max_params']\nparams","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a99b19a8c34acccd275edc09c590331e4064f11f"},"cell_type":"code","source":"categorical_features = list(cat_cols)\nprediction1,model1 = lgb_train(**params)\nprediction2,model2 = lgb_train(**params)\nprediction3,model3 = lgb_train(**params)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da27af24da18dc3b173aabf4ac77d8d4d3a3c414"},"cell_type":"code","source":"prediction_lgb = (np.expm1(prediction1)+np.expm1(prediction2)+np.expm1(prediction3))/3\nprediction_lgb = [0 if x < 0 else x for x in prediction_lgb]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"613800e3814c857d0b14e89959419b7202a04a8b"},"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['fullVisitorId'] = test_ind\nsubmission['PredictedLogRevenue'] = prediction_lgb\nsubmission = submission.groupby('fullVisitorId').sum()['PredictedLogRevenue'].apply(np.log1p).fillna(0).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42e6b81e05d115a80018e6c080ce6dd8252dd5ad"},"cell_type":"code","source":"submission.to_csv('submit_lgbm.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98787a9c8ea99a13dbf471daff991a5679c2b2fd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}