{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport time\nimport pickle\nimport feather\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm._tqdm_notebook import tqdm_notebook as tqdm\ntqdm.pandas();\n\nimport glob\ndef get_path(str, first=True, parent_dir='../input/**/'):\n    res_li = glob.glob(parent_dir+str)\n    return res_li[0] if first else res_li","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain_df = feather.read_dataframe(get_path('train.ftr'))\ntest_df = feather.read_dataframe(get_path('test.ftr'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9ef25d001badca35b1709edc641b484865126b7"},"cell_type":"code","source":"def proc_column_name(df):\n    cols = []\n    for c in df.columns:\n        if '.' in c:\n            cols.append(c.replace('.', '_'))\n        else:\n            cols.append(c)\n    df.columns = cols\n    return df\n\ntrain_df = proc_column_name(train_df)\ntest_df = proc_column_name(test_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5ec4f493deec8ba22a3c13874a7861c90f1e1e5"},"cell_type":"markdown","source":"## From\n- https://www.kaggle.com/gpreda/ga-customer-revenue-simple-lightgbm/"},{"metadata":{"trusted":true,"_uuid":"2041ed83aad61d6710c5a02f93095fa999f0db59"},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import TimeSeriesSplit, KFold\nfrom sklearn.metrics import mean_squared_error\n\nDEV = True\nDEV = False\n\nPATH = '../input/google-analytics-customer-revenue-prediction/'\nNUM_ROUNDS = 20000 if not DEV else 20\nVERBOSE_EVAL = 10 if not DEV else 10\nSTOP_ROUNDS = 100 if not DEV else 10\nN_SPLITS = 5 if not DEV else 3\n\njson_cols = ['device', 'geoNetwork', 'totals', 'trafficSource']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c32a9903fff3f997446c907ea2428d648aafa1b0"},"cell_type":"code","source":"def read_parse_dataframe(file_name):\n    #full path for the data file\n    path = PATH + file_name\n    #read the data file, convert the columns in the list of columns to parse using json loader,\n    #convert the `fullVisitorId` field as a string\n    data_df = pd.read_csv(path, \n        converters={column: json.loads for column in cols_to_parse}, \n        dtype={'fullVisitorId': 'str'})\n    #parse the json-type columns\n    for col in cols_to_parse:\n        #each column became a dataset, with the columns the fields of the Json type object\n        json_col_df = json_normalize(data_df[col])\n        #json_col_df.columns = [f\"{col}_{sub_col}\" for sub_col in json_col_df.columns]\n        #we drop the object column processed and we add the columns created from the json fields\n        data_df = data_df.drop(col, axis=1).merge(json_col_df, right_index=True, left_index=True)\n    return data_df\n\ndef process_date_time(data_df):\n    print(\"process date time ...\")\n    data_df['date'] = data_df['date'].astype(str)\n    data_df[\"date\"] = data_df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n    data_df[\"date\"] = pd.to_datetime(data_df[\"date\"])   \n    data_df[\"year\"] = data_df['date'].dt.year\n    data_df[\"month\"] = data_df['date'].dt.month\n    data_df[\"day\"] = data_df['date'].dt.day\n    data_df[\"weekday\"] = data_df['date'].dt.weekday\n    data_df['weekofyear'] = data_df['date'].dt.weekofyear\n    data_df['month_unique_user_count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day_unique_user_count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday_unique_user_count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n    return data_df\n\ndef process_format(data_df):\n    print(\"process format ...\")\n    for col in ['visitNumber', 'totals_hits', 'totals_pageviews']:\n        data_df[col] = data_df[col].astype(float)\n    data_df['trafficSource_adwordsClickInfo_isVideoAd'].fillna(True, inplace=True)\n    data_df['trafficSource_isTrueDirect'].fillna(False, inplace=True)\n    return data_df\n    \ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['browser_category'] = data_df['device_browser'] + '_' + data_df['device_deviceCategory']\n    data_df['browser_operatingSystem'] = data_df['device_browser'] + '_' + data_df['device_operatingSystem']\n    data_df['source_country'] = data_df['trafficSource_source'] + '_' + data_df['geoNetwork_country']\n    return data_df\n\ndef process_totals(data_df):\n    print(\"process totals ...\")\n    data_df['visitNumber'] = np.log1p(data_df['visitNumber'])\n    data_df['totals_hits'] = np.log1p(data_df['totals_hits'])\n    data_df['totals_pageviews'] = np.log1p(data_df['totals_pageviews'].fillna(0))\n    data_df['mean_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('mean')\n    data_df['sum_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('sum')\n    data_df['max_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('max')\n    data_df['min_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('min')\n    data_df['var_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('var')\n    return data_df\n\ndef process_geo_network(data_df):\n    print(\"process geo network ...\")\n    data_df['sum_pageviews_per_network_domain'] = data_df.groupby(\n        'geoNetwork_networkDomain')['totals_pageviews'].transform('sum')\n    data_df['count_pageviews_per_network_domain'] = data_df.groupby(\n        'geoNetwork_networkDomain')['totals_pageviews'].transform('count')\n    data_df['mean_pageviews_per_network_domain'] = data_df.groupby(\n        'geoNetwork_networkDomain')['totals_pageviews'].transform('mean')\n    data_df['sum_hits_per_network_domain'] = data_df.groupby(\n        'geoNetwork_networkDomain')['totals_hits'].transform('sum')\n    data_df['count_hits_per_network_domain'] = data_df.groupby(\n        'geoNetwork_networkDomain')['totals_hits'].transform('count')\n    data_df['mean_hits_per_network_domain'] = data_df.groupby(\n        'geoNetwork_networkDomain')['totals_hits'].transform('mean')\n    return data_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd1e3ec8db4b0a27fff36c3e9e79c906f7256531","_kg_hide-output":true},"cell_type":"code","source":"%%time\n#Feature processing\n## Load data\n# train_df = read_parse_dataframe('train.csv')\ntrain_df = process_date_time(train_df)\n# test_df = read_parse_dataframe('test.csv')\ntest_df = process_date_time(test_df)\n\n## Drop columns\ncols_to_drop = [col for col in train_df.columns if train_df[col].nunique(dropna=False) == 1]\ntrain_df.drop(cols_to_drop, axis=1, inplace=True)\ntest_df.drop([col for col in cols_to_drop if col in test_df.columns], axis=1, inplace=True)\n\n###only one not null value\ntrain_df.drop(['trafficSource_campaignCode'], axis=1, inplace=True)\n\n###converting columns format\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].astype(float)\ntrain_df['totals_transactionRevenue'] = train_df['totals_transactionRevenue'].fillna(0)\ntrain_df['totals_transactionRevenue'] = np.log1p(train_df['totals_transactionRevenue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee0fb3b1ea3456ad66f67638ad247f6de6ccd0b2","_kg_hide-output":true},"cell_type":"code","source":"%%time\n## Features engineering\ntrain_df = process_format(train_df)\ntrain_df = process_device(train_df)\ntrain_df = process_totals(train_df)\ntrain_df = process_geo_network(train_df)\n\ntest_df = process_format(test_df)\ntest_df = process_device(test_df)\ntest_df = process_totals(test_df)\ntest_df = process_geo_network(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef1e47bfb40f362433049f2eacd5e14e90e9fbd2","_kg_hide-output":true},"cell_type":"code","source":"%%time\n## Categorical columns\nprint(\"process categorical columns ...\")\nnum_cols = [\n    'month_unique_user_count', 'day_unique_user_count', 'weekday_unique_user_count',\n    'visitNumber', 'totals_hits', 'totals_pageviews', \n    'mean_hits_per_day', 'sum_hits_per_day', 'min_hits_per_day', 'max_hits_per_day', 'var_hits_per_day',\n    'sum_pageviews_per_network_domain', 'count_pageviews_per_network_domain', 'mean_pageviews_per_network_domain',\n    'sum_hits_per_network_domain', 'count_hits_per_network_domain', 'mean_hits_per_network_domain'\n]\n            \nnot_used_cols = [\"visitNumber\", \"date\", \"fullVisitorId\", \"sessionId\", \n        \"visitId\", \"visitStartTime\", 'totals_transactionRevenue', 'trafficSource_referralPath']\ncat_cols = [col for col in train_df.columns if col not in num_cols and col not in not_used_cols]\nfor col in cat_cols:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47f059bdb7946fb0a3cbe0e2fb8543c29d199fb8"},"cell_type":"code","source":"# Model\nprint(\"prepare model ...\")\n#train_df = train_df.sort_values('date')\ntrain_df = train_df.sort_values('visitStartTime')\n\nfullVisitorId = train_df['fullVisitorId'].values\ngroup = LabelEncoder().fit_transform(fullVisitorId)\n\nX = train_df.drop(not_used_cols, axis=1)\ny = train_df['totals_transactionRevenue']\nX_test = test_df.drop([col for col in not_used_cols if col in test_df.columns], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90ac0214fd18172754bd53f84670ea429d1cb7da"},"cell_type":"code","source":"## customized functions\n## https://lightgbm.readthedocs.io/en/latest/Python-API.html#scikit-learn-api\n## https://github.com/Microsoft/LightGBM/blob/master/examples/python-guide/advanced_example.py\n\ndef fobj(preds, dataset): # -> grad, hess\n    labels = dataset.get_label()\n    ids = dataset.ids\n    preds[preds<0] = 0\n    pairs = dict(y_true=np.expm1(labels), y_pred=np.expm1(preds), ids=ids)\n    pairs = pd.DataFrame(pairs)\n    errors = np.log1p(pairs.groupby('ids').sum()).reset_index()\n    errors['error'] = errors['y_pred'] - errors['y_true']\n    pairs = pairs.merge(errors[['ids', 'error']], how='left', on='ids')\n    return 2 * pairs['error'].values, 2 * np.ones(labels.shape[0])\n\ndef feval(preds, dataset): # -> eval_name, eval_result, is_bigger_better\n    labels = dataset.get_label()\n    ids = dataset.ids\n    preds[preds<0] = 0\n    rmse_per_user = eval_metric(labels, preds, ids)\n    eval_name = 'MyRMSE'\n    eval_result = rmse_per_user\n    is_bigger_better = False\n    return eval_name, eval_result, is_bigger_better\n\ndef eval_metric(y_true, y_pred, ids):\n    pairs = dict(y_true=np.expm1(y_true), y_pred=np.expm1(y_pred), ids=ids)\n    pairs = pd.DataFrame(pairs)\n    pairs = np.log1p(pairs.groupby('ids').sum()).values\n    rmse_per_user = np.mean((pairs[:, 0] - pairs[:, 1])**2)**.5\n    return rmse_per_user","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9336ba96d0f99a47ac3ff472f340e0f9fd669dd4","_kg_hide-output":true},"cell_type":"code","source":"%%time\n## Model parameters\nparams = {\n    \"objective\" : \"regression\", \n    \"metric\" : \"None\", #\"rmse\", \n    \"max_depth\": 8, \n    \"min_child_samples\": 20, \n    \"reg_alpha\": 1, \n    \"reg_lambda\": 1,\n    \"num_leaves\" : 257, \n    \"learning_rate\" : 0.01, \n    \"subsample\" : 0.8, \n    \"colsample_bytree\" : 0.8, \n    \"subsample_freq \": 5\n}\nfolds = KFold(n_splits=N_SPLITS, shuffle=True, random_state=42)\n## Model\nprint(\"model ...\")\n\npred_val = np.zeros(X.shape[0])\npred_test = np.zeros(test_df.shape[0])\neval_hist_li = []\nscores = []\n\ndataset = lgb.Dataset(X, y.values, free_raw_data=False)\n\nfor fold_n, (train_index, test_index) in enumerate(folds.split(X)):\n    print('Fold:', fold_n)\n    ids_trn, ids_val = group[train_index], group[test_index]\n    dtrain = dataset.subset(train_index)\n    dvalid = dataset.subset(test_index)\n    dtrain.ids = ids_trn\n    dvalid.ids = ids_val\n    evals_result = {}\n    model = lgb.train(\n        params,\n        dtrain,\n        num_boost_round=NUM_ROUNDS,\n        valid_sets=[dtrain, dvalid],\n        valid_names=['train','valid'],\n        evals_result=evals_result,\n        early_stopping_rounds=STOP_ROUNDS,\n        verbose_eval=VERBOSE_EVAL,\n        feval=feval,\n        #fobj=fobj\n    )\n    eval_hist_li.append(evals_result)\n    pred_val[test_index] = model.predict(X.iloc[test_index])\n    pred_val[pred_val<0] = 0\n    scores.append(eval_metric(y[test_index], pred_val[test_index], ids_val))\n    pred_test += model.predict(X_test)\n    \npred_test /= N_SPLITS\npred_test[pred_test<0] = 0\n\nnp.save('pred_val.npy', pred_val)\nnp.save('pred_test.npy', pred_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"385809138c67d2c42f67b3e26c0fdfb18daf543c"},"cell_type":"code","source":"print('cv scores', scores)\noof_score = eval_metric(y, pred_val, group)\nprint('oof_score', oof_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"681460a40265f81256b85686fcbaa718941ebfa7"},"cell_type":"code","source":"# Submission\nsubmission = test_df[['fullVisitorId']].copy()\nsubmission.loc[:, 'PredictedLogRevenue'] = np.expm1(pred_test)\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].apply(lambda x : 0.0 if x < 0 else x)\ngrouped_test = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum().reset_index()\ngrouped_test['PredictedLogRevenue'] = np.log1p(grouped_test['PredictedLogRevenue'])\ngrouped_test.to_csv(f'sub_{oof_score:.6f}.csv',index=False)\nos.listdir('.')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}