{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from bayes_opt import BayesianOptimization\nimport datetime\nimport gc\nimport json\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport numpy as np\nimport os\nimport pandas as pd \nfrom pandas.io.json import json_normalize\nfrom pandas.core.common import SettingWithCopyWarning\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold,train_test_split\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn import preprocessing\nimport sys\nimport warnings\nwarnings.filterwarnings('ignore')\ngc.enable()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#csv_path differs from paul's original code, just modify if not running in kaggle kernel\n\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b5165c05b3d47251939839a2591a1b9806bf438"},"cell_type":"code","source":"#defining time predictors - Added \n\ndef add_time_features(df):\n    df['date'] = pd.to_datetime(df['date'], format='%Y%m%d', errors='ignore')\n    df['year'] = df['date'].apply(lambda x: x.year)\n    df['month'] = df['date'].apply(lambda x: x.month)\n    df['day'] = df['date'].apply(lambda x: x.day)\n    df['weekday'] = df['date'].apply(lambda x: x.weekday())\n    df['visitStartTime_'] = pd.to_datetime(df['visitStartTime'],unit=\"s\")\n    df['visitStartTime_year'] = df['visitStartTime_'].apply(lambda x: x.year)\n    df['visitStartTime_month'] = df['visitStartTime_'].apply(lambda x: x.month)\n    df['visitStartTime_day'] = df['visitStartTime_'].apply(lambda x: x.day)\n    df['visitStartTime_weekday'] = df['visitStartTime_'].apply(lambda x: x.weekday())\n    return df\ndate_features = [#\"year\",\"month\",\"day\",\"weekday\",'visitStartTime_year',\n    \"visitStartTime_month\",\"visitStartTime_day\",\"visitStartTime_weekday\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cff67ff00772227c394c8de512cf4dd737d44b02"},"cell_type":"code","source":"train = load_df(\"../input/train.csv\")\ntest = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c5a8d6636edde8057e037b6cd6ab9ba5c288c53"},"cell_type":"code","source":"for col in test.columns:\n    if len(test[col].value_counts()) == 1:\n        test.drop(col,inplace=True,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb7cddc5107cb66e283a284f8f9416f465561b68"},"cell_type":"code","source":"for col in train.columns:\n    if len(train[col].value_counts()) == 1:\n        train.drop(col,inplace=True,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d81ffae1cff49635ce4b4a7017c20b47df63985"},"cell_type":"code","source":"##DID NOT REMOVE(NEW)\ntest = test[test.columns.drop(list(test.filter(regex='trafficSource.adwordsClickInfo')))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37fbac6ac13bce31630baaa59181ecd7a4625944"},"cell_type":"code","source":"##DID NOT REMOVE(NEW)\ntrain = train[train.columns.drop(list(train.filter(regex='trafficSource.adwordsClickInfo')))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"393c50136e2068e87f84b1e126cb428c96035abb"},"cell_type":"code","source":"num_col = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"889cce636819130a4f10486c00f7e2bbb5eb9086"},"cell_type":"code","source":"#FILL NA (NEW)\n\nfor col in num_col:\n    train[col] = train[col].fillna(\"0\").astype(\"int32\")\n    test[col] = test[col].fillna(\"0\").astype(\"int32\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d876ead09de2373a3b6e419e227575ab246be937"},"cell_type":"code","source":"#####FEATURE ENGINEERING (NEW)\n\nnew_features = [\"hits_per_pageviews\"]     #combining hits and pageviews (see function below)\nnew_category_features = [\"is_high_hits\"]\ndef feature_engineering(df):\n    line = 4\n    df['hits_per_pageviews'] = (df[\"totals.hits\"]/(df[\"totals.pageviews\"])).apply(lambda x: 0 if np.isinf(x) else x)\n    df['is_high_hits'] = np.logical_or(train[\"totals.hits\"]>line,train[\"totals.pageviews\"]>line).astype(np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cfc094f9d92b19eb367c09ad2e14e0f162ee873"},"cell_type":"code","source":"#applying change, as well as time features (NEW)\n\nfeature_engineering(train)\nfeature_engineering(test)\nadd_time_features(train)\n_ = add_time_features(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2f17e5b39133c09ee7479610768f37840fced55"},"cell_type":"code","source":"#defining categorical features, target, and useless features (NEW)\n\ncategory_features = [\"channelGrouping\", \"device.browser\", \n            \"device.deviceCategory\", \"device.operatingSystem\", \n            \"geoNetwork.city\", \"geoNetwork.continent\", \n            \"geoNetwork.subContinent\",\n            \"trafficSource.medium\", \n            \"trafficSource.source\",\n            ] + date_features\ntarget = 'totals.transactionRevenue'\nuseless_col = [\"trafficSource.adContent\",                                                ##useless based on lgb feature importance/unique values/amount of NA\n              \"trafficSource.adwordsClickInfo.adNetworkType\", \n              \"trafficSource.adwordsClickInfo.page\",\n              \"trafficSource.adwordsClickInfo.slot\",\n              \"trafficSource.campaign\",\n              \"trafficSource.referralPath\",\n              'trafficSource.adwordsClickInfo.isVideoAd',\n              \"trafficSource.adwordsClickInfo.gclId\",\n              \"trafficSource.keyword\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ad76fa1b44339cd66e0a962da712f251a3b859f"},"cell_type":"code","source":"excluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime'\n]\n\ncategorical_features = [\n    _f for _f in train.columns\n    if (_f not in excluded_features) & (train[_f].dtype == 'object')\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62849e846cd2443c4f104f2140be6a4c664309b2"},"cell_type":"code","source":"for f in categorical_features:\n    train[f], indexer = pd.factorize(train[f])\n    test[f] = indexer.get_indexer(test[f])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2928703f4e8a1e00f1919a8b48637e40889c5b7"},"cell_type":"code","source":"#inputting value for na's (NEW)\ntrain[target] = train[target].fillna(\"0\").astype(\"int32\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c206eb96dcf67b4b8e0e091691e946a7ef6247fa"},"cell_type":"code","source":"def visit24(myTime):\n    visitCount=[]\n    for time in myTime:\n        visitBool = (myTime > time-pd.Timedelta('24 hours')) & (myTime < time)\n        visitCount.append(visitBool.sum())\n    return(pd.Series(visitCount))\n\ndef visit48(myTime):\n    visitCount=[]\n    for time in myTime:\n        visitBool = (myTime > time-pd.Timedelta('48 hours')) & (myTime < time)\n        visitCount.append(visitBool.sum())\n    return(pd.Series(visitCount))\n\ndef visit72(myTime):\n    visitCount=[]\n    for time in myTime:\n        visitBool = (myTime > time-pd.Timedelta('72 hours')) & (myTime < time)\n        visitCount.append(visitBool.sum())\n    return(pd.Series(visitCount))\n\ndef visit168(myTime):\n    visitCount=[]\n    for time in myTime:\n        visitBool = (myTime > time-pd.Timedelta('168 hours')) & (myTime < time)\n        visitCount.append(visitBool.sum())\n    return(pd.Series(visitCount))\n\nfor df in [train, test]:\n    df['visitTotal'] = df.groupby('fullVisitorId')['date'].transform('size')\n    #df['v24']=df.groupby('fullVisitorId')['date'].transform(visit24)\n    #df['v48']=df.groupby('fullVisitorId')['date'].transform(visit48)\n    #df['v72']=df.groupby('fullVisitorId')['date'].transform(visit72)\n    df['v168']=df.groupby('fullVisitorId')['date'].transform(visit168)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b953f76f392135b84eda727ce229703c158bf15d"},"cell_type":"code","source":"#taking cleaned data from paul's code and modifying features to improve relevancy (NEW)\nall_features = category_features+num_col+new_features+new_category_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe9bd338c360c2d6dea7fd2971932f2defac02e6"},"cell_type":"code","source":"#prepare for lgb\n\ntrain_x = train[all_features]\ntrain_y = train[target]\ntest_x = test[all_features]\nfor col in category_features:\n    print(\"transform column {}\".format(col))\n    lbe = LabelEncoder()\n    lbe.fit(pd.concat([train[col],test_x[col]]).astype(\"str\"))\n    train_x[col] = lbe.transform(train_x[col].astype(\"str\"))\n    test_x[col] = lbe.transform(test_x[col].astype(\"str\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d219a365ea39ef9add7c307c36d11846882d222"},"cell_type":"code","source":"#defining functions\n\ndef lgb_eval(num_leaves,max_depth,lambda_l2,lambda_l1,min_child_samples,learning_rate):\n    params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\", \n    \"num_leaves\" : int(num_leaves),\n    \"max_depth\" : int(max_depth),\n    \"lambda_l2\" : lambda_l2,\n    \"lambda_l1\" : lambda_l1,\n    \"num_threads\" : 4,\n    \"min_child_samples\" : int(min_child_samples),\n    \"learning_rate\" : learning_rate,\n    \"subsample_freq\" : 5,\n    \"bagging_seed\" : 42,\n    \"verbosity\" : -1\n    }\n    lgtrain = lgb.Dataset(train_x, label=np.log1p(train_y.apply(lambda x : 0 if x < 0 else x)),categorical_feature=category_features)\n    cv_result = lgb.cv(params,\n                       lgtrain,\n                       10000,\n                       categorical_feature=category_features,\n                       early_stopping_rounds=100,\n                       stratified=False,\n                       nfold=5)\n    return -cv_result['rmse-mean'][-1]\n\ndef lgb_train(num_leaves,max_depth,lambda_l2,lambda_l1,min_child_samples, learning_rate):\n    params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\", \n    \"num_leaves\" : int(num_leaves),\n    \"max_depth\" : int(max_depth),\n    \"lambda_l2\" : lambda_l2,\n    \"lambda_l1\" : lambda_l1,\n    \"num_threads\" : 4,\n    \"min_child_samples\" : int(min_child_samples),\n    \"learning_rate\" : learning_rate,\n    \"subsample_freq\" : 5,\n    \"bagging_seed\" : 42,\n    \"verbosity\" : -1\n    }\n    t_x,v_x,t_y,v_y = train_test_split(train_x,train_y,test_size=0.2)\n    lgtrain = lgb.Dataset(t_x, label=np.log1p(t_y.apply(lambda x : 0 if x < 0 else x)),categorical_feature=category_features)\n    lgvalid = lgb.Dataset(v_x, label=np.log1p(v_y.apply(lambda x : 0 if x < 0 else x)),categorical_feature=category_features)\n    model = lgb.train(params, lgtrain, 2000, valid_sets=[lgvalid], early_stopping_rounds=100, verbose_eval=100)\n    pred_test_y = model.predict(test_x, num_iteration=model.best_iteration)\n    return pred_test_y, model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba778fb218220d0a440cd877877f2c7139fe64e5"},"cell_type":"code","source":"#param_tuning defined above, (init_points, num_iter,**args)...utilizes bayesian optimization\n##defining bo ranges for parameter tuning    \n# def param_tuning(init_points,num_iter,**args):\n#     lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (25, 50),\n#                                                 'max_depth': (5, 15),\n#                                                 'lambda_l2': (0.0, 0.05),\n#                                                 'lambda_l1': (0.0, 0.05),\n#                                                 'bagging_fraction': (0.5, 0.8),\n#                                                 'feature_fraction': (0.5, 0.8),\n#                                                 'min_child_samples': (20, 50),\n#                                                 })\n\n#     lgbBO.maximize(init_points=init_points, n_iter=num_iter,**args)\n#     return lgbBO\n# result = param_tuning(5,15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c5250df800ebbb08793ab295991df87f69ed2f2"},"cell_type":"code","source":"def param_tuning_boundaryShift(init_points,num_iter,**args):\n    lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (20, 40),\n                                                'max_depth': (14, 30),\n                                                'lambda_l2': (0.0, 0.05),\n                                                'lambda_l1': (0.0, 0.05),\n                                                'min_child_samples': (15, 40), 'learning_rate': (0.01, 0.04)                                            \n                                                })\n\n    lgbBO.maximize(init_points=init_points, n_iter=num_iter,**args)\n    return lgbBO\nresult = param_tuning_boundaryShift(20,10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4349f63d7fa94b4d372cf40d096cb6da3dc1c8a"},"cell_type":"code","source":"result.res['max']['max_params']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"443c5d65fe174d8a6b29aba48bf6087ce11ff230"},"cell_type":"code","source":"pred1,model1 = lgb_train(**result.res['max']['max_params'])\npred2,model2 = lgb_train(**result.res['max']['max_params'])\npred3,model3 = lgb_train(**result.res['max']['max_params'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2b29cc358b7e41e4fc4307ca611dead13a9579e"},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}