{"cells":[{"metadata":{"_uuid":"65b641fae4426fe726e6e213d9afebdae4667c66"},"cell_type":"markdown","source":"**Summary**\n\n   - **Problem:** predict the total revenue from each visitors to Google Store\n   - **Data:**  \"per visit\" information is provided.  Each visitor may have several visits to the store.\n   - **Startegy**:\n       - Classify each session based on having/not-having revenue\n       - Predict each session's revenue\n       - Sum up revenue's from each visitor"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport logging\nimport json\nfrom pandas.io.json import json_normalize\nimport datetime\n\nimport tensorflow as tf\nfrom keras import Sequential, layers, optimizers, backend as K\nfrom keras.utils import to_categorical\nfrom keras.callbacks import EarlyStopping\nimport lightgbm as lgb\n\nfrom sklearn.utils import class_weight\nfrom sklearn.model_selection import train_test_split, KFold, StratifiedKFold, GroupKFold\nfrom sklearn.preprocessing import StandardScaler, Normalizer, normalize\nfrom sklearn.metrics import confusion_matrix, classification_report\nfrom sklearn.metrics import precision_score, recall_score, precision_recall_curve\n\nfrom matplotlib import pyplot as plt\nplt.style.use('ggplot')\nimport seaborn as sns\n\ndef get_logger(fname='google_store.log', logger_name=__name__):\n    logger = logging.getLogger(logger_name)\n    logger.setLevel(logging.INFO)\n\n    formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(name)s >> %(message)s')\n\n    file_handler = logging.FileHandler(fname)\n    file_handler.setLevel(logging.INFO)\n    file_handler.setFormatter(formatter)\n\n    stream_handler = logging.StreamHandler()\n    stream_handler.setFormatter(formatter)\n    stream_handler.setLevel(logging.INFO)\n\n    logger.addHandler(file_handler)\n    logger.addHandler(stream_handler)\n    logger.info('Initializing \\'{}\\' logger...'.format(logger_name))\n    \n    return logger\n\ndef cdf(data):\n    \"\"\"Compute CDF for a one-dimensional array of measurements.\"\"\"\n    n = len(data)\n    x = np.sort(data)\n    y = np.arange(1, n+1) / n\n    return x, y\n\ndef plot_hist_cdf(data, x_label='', title = '', figsize=(10,5), bins=50,xlim=None):\n    fig, ax1 = plt.subplots(figsize=figsize);\n    ax1.hist(data, bins=bins);\n    ax1.set_xlabel(x_label);\n    ax1.set_ylabel('Count');\n    ax1.set_xlim(xlim);\n\n    ax2=ax1.twinx();\n    cdf_x, cdf_y = cdf(data);\n    ax2.plot(cdf_x, cdf_y, c='b');\n    ax2.set_ylabel('Cumulative');\n    ax2.set_ylim(0,);\n    ax1ylims = ax1.get_ybound()\n    ax2ylims = ax2.get_ybound()\n    minresax1=3\n    minresax2=.2\n    ax1factor = minresax1 * 6\n    ax2factor = minresax2 * 6\n    ax1.set_yticks(np.linspace(ax1ylims[0],\n                               ax1ylims[1]+(ax1factor -\n                               (ax1ylims[1]-ax1ylims[0]) % ax1factor) %\n                               ax1factor,\n                               7))\n    ax2.set_yticks(np.linspace(ax2ylims[0],\n                               ax2ylims[1]+(ax2factor -\n                               (ax2ylims[1]-ax2ylims[0]) % ax2factor) %\n                               ax2factor,\n                               7))\n    plt.title(title);\n    plt.show()\n\ndef col_summary(dt, col):\n    out = dict([('name',[col]),('label',['']),('type', [dt[col].dtype]),('perc_nulls',[dt[col].isnull().sum()/len(dt[col])*100]),\n     ('num_uniques', [dt[col].nunique(dropna=False)]),('examples', [str(dt[col].unique()[0:5])])])\n    return out\n\ndef tab_summary(dt):\n    summary = pd.DataFrame()\n    for col in dt.columns:\n        summary = pd.concat([summary, pd.DataFrame(col_summary(dt, col))])\n    summary.reset_index(inplace=True, drop=True)\n    return summary\n\ndef get_most_common(data_table, column, count):\n    df = data_table.groupby(column)['HAS.transactionRevenue'].agg([('count','count'),('num_transactions','sum')], axis=1).sort_values('count',ascending=False).reset_index()\n    return df[column][:count].str.lower().values\n\n# https://www.kaggle.com/julian3833/1-quick-start-read-csv-and-flatten-json-fields\ndef load_df(csv_path='input/train.csv', JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource'], nrows=None, name=''):\n    log.info('Loading {}...'.format(csv_path))\n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'},nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    \n    df.name = name\n    log.info('{} size = {}'.format(df.name, len(df)))\n    return df\n\ndef table2features(table_in, cols_to_exclude):\n    out = pd.DataFrame()\n    features=[]\n    for col in table_in.columns:\n        if col not in cols_to_exclude:\n            out[col] = table_in[col]\n            features.append(col)\n            if table_in[col].dtypes == bool:\n                out[col] = table_in[col].astype(int)\n    out = out.fillna(0)\n    return features, out.values\n\ndef sensitivity(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    return true_positives / (possible_positives + K.epsilon())\n\ndef specificity(y_true, y_pred):\n    true_negatives = K.sum(K.round(K.clip((1-y_true) * (1-y_pred), 0, 1)))\n    possible_negatives = K.sum(K.round(K.clip(1-y_true, 0, 1)))\n    return true_negatives / (possible_negatives + K.epsilon())\n\ndef precision_recall_report(y_target, y_pred):\n    precision, recall, threshold = precision_recall_curve(y_true = y_target, probas_pred =y_pred)\n    dim = min(len(precision),len(recall),len(threshold))\n    plt.plot(threshold[:dim], precision[:dim])\n    plt.plot(threshold[:dim], recall[:dim])\n    plt.xlim(0,1)\n    plt.legend(['Precision','Recall'])\n\ntry:\n    if log:\n        log.info('Logger is already running')\nexcept:\n    log = get_logger()\n    log.disabled = False","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Loading train/test\nnrows=None\ntrain = load_df('../input/train.csv', nrows=nrows, name='train_dataset')\ntest = load_df('../input/test.csv', nrows=nrows, name='test_dataset')\nlog.info('Data loaded.')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2e1682315df13b828b4130b4ea51c5cef5d04e4"},"cell_type":"markdown","source":"**Data cleaning and feature engineering**"},{"metadata":{"trusted":true,"_uuid":"33b3699311d4ef9528d446dd49ea82c901dfe451"},"cell_type":"code","source":"log.info('Dropping columns unique to train data...')\n# dropping columns that do not exist in test table\nexcl = 'totals.transactionRevenue'\nfor col in train.columns:\n    if col != excl and col not in test.columns:\n        train.drop(col, axis=1, inplace=True)\n        log.info('\\tDropped {}.'.format(col))\n        \nlog.info('Dropping columns without variability...')\n# dropping columns with only 1 distinct obervations\nfor col in train.columns:\n    if train[col].nunique(dropna=False)==1:\n        for tab in [train, test]:\n            tab.drop(col, axis=1, inplace=True)\n        log.info('\\tDropped {}.'.format(col))\n    \nlog.info('Adding date and time columns...')\n#adding date/time\nfor tab in [train, test]:\n    tab['date'] = pd.to_datetime(tab['visitStartTime'], unit='s')\n    tab['hour'] = tab['date'].dt.hour\n    tab['dayofweek'] = tab['date'].dt.weekday\n    tab['dayofmonth'] = tab['date'].dt.day\n    tab['month'] = tab['date'].dt.month\n    tab['trafficSource.adwordsClickInfo.gclId']=tab['trafficSource.adwordsClickInfo.gclId'].isna()\n    \nlog.info('Converting totals to numeric columns...')\n# numeric columns\nto_numeric =['totals.bounces', 'totals.hits', 'totals.newVisits', 'totals.pageviews', \n             'trafficSource.adwordsClickInfo.page']\nfor tab in [train, test]:\n    for col in to_numeric:\n        tab[col] = tab[col].fillna(0).astype(np.int64)\n\nlog.info('Computing log of transaction revenues...')\n# Computing log of Transaction Revenue\ntrain['log.totals.transactionRevenue'] =np.log1p(train['totals.transactionRevenue'].fillna(0).astype(np.int64))\ntrain['HAS.transactionRevenue']=(train['log.totals.transactionRevenue']>0).astype(np.int32)\n\nlog.info('Adding time to the next/previous session...')\n# Adding time to next/previus sessions\n# adopted from https://www.kaggle.com/ashishpatel26/future-is-here\nfor i in range(1,3):\n    for tab in [test, train]:\n        tab.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)\n        tab['time_to_prev_session_{}'.format(i)] = \\\n            ((tab['date'] - tab[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(i))*\\\n            (tab['fullVisitorId']==tab['fullVisitorId'].shift(i))).fillna(0).astype(np.int64)/1e9/3600\n        tab['time_to_next_session_{}'.format(i)] = \\\n            -((tab['date'] - tab[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-i))*\\\n            (tab['fullVisitorId']==tab['fullVisitorId'].shift(-i))).fillna(0).astype(np.int64)/1e9/3600\nfor tab in [train, test]:\n    tab['nb_pageviews'] = tab['date'].map(\n        tab[['date', 'totals.pageviews']].groupby('date')['totals.pageviews'].sum())\n    tab['ratio_pageviews'] = tab['totals.pageviews'] / tab['nb_pageviews']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cec67aff57ec2f7d36b860974a9b4be32901afd0"},"cell_type":"markdown","source":"**Some EDA**"},{"metadata":{"trusted":true,"_uuid":"5e20ea6086e84292238575c9a50a32aa0e134244"},"cell_type":"code","source":"summary = tab_summary(train)\nsummary.sort_values('num_uniques')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4690016852e4c9193fecfafdd0888ddbe8398058"},"cell_type":"code","source":"_ = train[train['HAS.transactionRevenue']==1]['HAS.transactionRevenue'].values.sum()\n__ = len(train)-_\nlabels = 'No Revenue', 'With Revenue'\nsizes = [__, _]\ncolors = ['yellowgreen','gold']\nexplode = (0.1, 0)  # explode 1st slice\nplt.figure(figsize=(5,5))\nplt.pie(sizes, explode=explode, labels=labels, colors=colors,\n        autopct='%1.1f%%', shadow=True, startangle=30)\nplt.title('Fraction of sessions with revenue')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00825116595c67478d4fbc2e59d1dce3f5b8c749"},"cell_type":"code","source":"_ = train[train['HAS.transactionRevenue']==1]['log.totals.transactionRevenue'].values\nplot_hist_cdf(_,bins=50, title='Distribution of transaction revenue',x_label='log.totals.transactionRevenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56f6ecf22c33a3ccdbb4b84b8104aee015f90143"},"cell_type":"code","source":"for col in train.columns:\n    if train[col].nunique(dropna=False)<700 and col!='HAS.transactionRevenue':\n        plt.figure(figsize=(15,5))\n        df_HAS = train.groupby(col)['HAS.transactionRevenue'].agg([('has_transactions','sum')], axis=1)\n        df_ALL = train.groupby(col)['HAS.transactionRevenue'].agg([('all_visits','count')], axis=1)\n        df = pd.concat([df_HAS, df_ALL], axis=1)\n        df['has_transactions'] = df['has_transactions']/df['has_transactions'].sum()*100\n        df['all_visits'] = df['all_visits']/df['all_visits'].sum()*100\n        df.reset_index(inplace=True)\n        df2 = pd.melt(df, id_vars=col, value_vars=['has_transactions', 'all_visits'], var_name='perc_visits')\n        sns.barplot(x=col, y='value', hue='perc_visits', data=df2)\n        plt.ylabel('% of all visits')\n        plt.xticks(rotation=90)\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"89deeccbd00e408d48459f07eb77f227d6bae04e"},"cell_type":"markdown","source":"**Feature Engineering: session level mapping**"},{"metadata":{"trusted":true,"_uuid":"fbc478bda32f76838da8eb5a10bd7b6354434d5b"},"cell_type":"code","source":"# adapted from https://www.kaggle.com/prashantkikani/teach-lightgbm-to-sum-predictions-fe\nbrowsers = get_most_common(train, 'device.browser',5)\nos = get_most_common(train, 'device.operatingSystem',6)\ncountries = get_most_common(train, 'geoNetwork.country',100)\ncities = get_most_common(train, 'geoNetwork.city',100)\nregions = get_most_common(train, 'geoNetwork.region',20)\nsources = get_most_common(train, 'trafficSource.source',10)\n\ndef map_category(x, categories):\n    if x in categories:\n        return x.lower()\n    else:\n        return 'others'\n\nlog.info('Feature mapping and defining interaction features...')\nfor tab in[train, test]:\n    tab['device.browser'] = tab['device.browser'].map(lambda x:map_category(str(x).lower(), browsers)).astype('str')\n    tab['device.operatingSystem'] = tab['device.operatingSystem'].map(lambda x:map_category(str(x).lower(), os)).astype('str')\n    tab['geoNetwork.country'] = tab['geoNetwork.country'].map(lambda x:map_category(str(x).lower(), countries)).astype('str')\n    tab['geoNetwork.city'] = tab['geoNetwork.city'].map(lambda x:map_category(str(x).lower(), cities)).astype('str')\n    tab['geoNetwork.region'] = tab['geoNetwork.region'].map(lambda x:map_category(str(x).lower(), regions)).astype('str')\n    tab['trafficSource.source'] = tab['trafficSource.source'].map(lambda x:map_category(str(x).lower(), sources)).astype('str')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"100c4f2dd54b85239f2b293a9a56aeef75daed59"},"cell_type":"markdown","source":"![](http://)**Feature Engineering: visitor level aggregation**"},{"metadata":{"trusted":true,"_uuid":"5e8c127ee0f1400b50dde71567baa655032ea640"},"cell_type":"code","source":"# Aggregating features at visitor level\ncols_to_agg=['visitNumber','totals.bounces', 'totals.hits',\n            'totals.newVisits','totals.pageviews', 'hour', 'nb_pageviews', \n             'ratio_pageviews', 'time_to_prev_session_1', 'time_to_next_session_1']\n\naggs = {'sum_':'sum', 'mean_':'mean'}\n\nlog.info('Adding aggregated features to train set...')\n_ = train.groupby('fullVisitorId')[cols_to_agg].agg(aggs)\n_.columns = _.columns.map(''.join)\n_.fillna(0, inplace=True)\ntrain = train.join(_,on='fullVisitorId')\nlog.info('train data shape: {}'.format(train.shape))\n\nlog.info('Adding aggregated features to test set...')\n_ = test.groupby('fullVisitorId')[cols_to_agg].agg(aggs)\n_.columns = _.columns.map(''.join)\n_.fillna(0, inplace=True)\ntest = test.join(_,on='fullVisitorId')\nlog.info('test data shape: {}'.format(test.shape))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"425cd9e83690f1832619eb499af6672c058cad6a"},"cell_type":"markdown","source":"**Classification: LGBM**"},{"metadata":{"trusted":true,"_uuid":"d86bb1d62cb4bcb3fc26f524a0213702d98c2b04"},"cell_type":"code","source":"# Features to exclude\nexclude_features = ['date', 'fullVisitorId', 'sessionId', 'visitId', 'visitStartTime', \n                   'totals.transactionRevenue', 'log.totals.transactionRevenue', \n                   'HAS.transactionRevenue',\n                    'trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.isVideoAd', \n                    'trafficSource.adwordsClickInfo.slot', 'trafficSource.campaign', 'trafficSource.isTrueDirect', \n                    'geoNetwork.continent', 'geoNetwork.networkDomain', 'trafficSource.referralPath']\n\n# One-hot-coding\ndummies_max = 10\nlog.info('One-hot-coding of features with less than {} categories:'.format(dummies_max))\nto_dummies =[]\nfor col in train.columns:\n    nuniq = train[col].nunique(dropna=False)\n    if nuniq>2 and nuniq<dummies_max and col not in exclude_features:\n        to_dummies.append(col)\n        log.info('\\tone-hot-coding: {}'.format(col))\n        \nlog.info('\\tInitial size of train data set: {}'.format(train.shape))\n_ = pd.concat([train, test], sort=False)\n_ = pd.get_dummies(_, dummy_na=False, columns=to_dummies, drop_first=True)\ntrain = _[:len(train)]\ntest = _[len(train):]\ntest = test.drop(columns=['totals.transactionRevenue', 'log.totals.transactionRevenue', 'HAS.transactionRevenue'],axis=1)\nlog.info('\\tFinal size of train data set: {}'.format(train.shape))\n\n# Factorizing\ncat_features = [col for col in train.columns \n                if (col not in exclude_features) & (train[col].dtypes == 'object')]\nlog.info('Factorizing categorical features...')\nfor col in cat_features:\n    train[col], indexer = pd.factorize(train[col])\n    test[col] = indexer.get_indexer(test[col])\n\n# Get test/train arrays\nlog.info('Getting test/train arrays...')\ny_clf = train['HAS.transactionRevenue'].values\nfeatures_train, X_train = table2features(train, exclude_features)\nfeatures_test, X_test = table2features(test, exclude_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"668b28597fa290375625cebe3b31048547a34495"},"cell_type":"code","source":"# LGBM model\ndef get_lgbm_clf(num_leaves=100, lr=0.02):\n    model_clf = lgb.LGBMClassifier(num_leaves=num_leaves, learning_rate=lr, n_estimators=1000,\n                                    subsample=.9, colsample_bytree=.9, random_state=42)\n    return model_clf\n\n# StratifiedKFold Training\nn_splits=5\nmodels_clf1 = []\nmodels_clf2 = []\nfold_ = 1\nfeature_importance = pd.DataFrame()\nclass_weights = class_weight.compute_class_weight('balanced',[0,1],y_clf.flatten())\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=32)\n\nfor train_index, val_index in skf.split(X_train, y_clf):\n    log.info('Trainign with fold = {}'.format(fold_))\n    trn_x, trn_y = X_train[train_index], y_clf[train_index]\n    val_x, val_y = X_train[val_index], y_clf[val_index]\n    model_clf1 = get_lgbm_clf()\n    # LGBM classification\n    model_clf1.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=100, verbose=100)\n    # LGBM: recording feature importance\n    _ = pd.DataFrame()\n    _['feature'] = features_train\n    _['gain'] = model_clf1.booster_.feature_importance(importance_type='gain')\n    _['fold'] = fold_\n    fold_ += 1\n    feature_importance = pd.concat([feature_importance, _], axis=0, sort=False)\n    \n    #recoding models\n    models_clf1.append(model_clf1)\nlog.info('Finished training.')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53d52d765a6909da0b4c408df2d5f785ecab7512","scrolled":false},"cell_type":"code","source":"# Making predictions\nlog.info('Predictions from LGBM model...')\ny_preds1_train = np.zeros((n_splits, len(X_train)))\ny_preds1_test = np.zeros((n_splits, len(X_test)))\nfor i in range(n_splits):\n    y_preds1_train[i] = models_clf1[i].predict_proba(X_train, num_iteration=models_clf1[i].best_iteration_)[:,1]\n    y_preds1_test[i] = models_clf1[i].predict_proba(X_test, num_iteration=models_clf1[i].best_iteration_)[:,1]\ntrain['predicted_prob_clf1'] = y_preds1_train.mean(axis=0)\ntest['predicted_prob_clf1'] = y_preds1_test.mean(axis=0)\n\n# Feature importance plot\nfeature_importance['gain_log'] = np.log1p(feature_importance['gain'])\nmean_gain = feature_importance[['gain','feature']].groupby('feature').mean()\nfeature_importance['mean_gain'] = feature_importance['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(5,30))\nsns.barplot(x='gain_log', y='feature', data=feature_importance.sort_values('gain_log', ascending=False))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a389eca71b71335a4be86520369f5ea68aebec4f"},"cell_type":"markdown","source":"**Regression: LGBM**"},{"metadata":{"trusted":true,"_uuid":"0c3c863ec2cc4f367fd5a1a6f89b53297dd20ed1"},"cell_type":"code","source":"# Creating dataset arrays\nlog.info('Creating datasets for LGBM regression...')\ny_reg = train['log.totals.transactionRevenue'].values\ny_clf = train['HAS.transactionRevenue'].values\nfeatures_train, X_train = table2features(train, exclude_features)\nfeatures_test, X_test = table2features(test, exclude_features)\n\ndef get_lgbm_reg(num_leaves=100, lr=0.02):\n    model_reg =  lgb.LGBMRegressor(num_leaves=num_leaves, learning_rate=lr,\n        n_estimators=1000, subsample=.9, colsample_bytree=.9, random_state=42)\n    return model_reg\n\n# Training\nlog.info('Training LGBM regressor...')\nn_splits=5\nmodels_reg = []\nfold_ = 1\nmean_rmse = 0\nfeature_importance = pd.DataFrame()\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\nfor train_index, val_index in skf.split(X_train, y_clf):\n    log.info('Training fold {}'.format(fold_))\n    trn_x, trn_y = X_train[train_index], y_reg[train_index]\n    val_x, val_y = X_train[val_index], y_reg[val_index]\n    \n    # regression model\n    model_reg = get_lgbm_reg(lr=0.03)\n    model_reg.fit(trn_x, trn_y, eval_set=[(val_x, val_y)], early_stopping_rounds=100,\n        verbose=100, eval_metric='rmse')\n    \n    #recording feature importance\n    _ = pd.DataFrame()\n    _['feature'] = features_train\n    _['gain'] = model_reg.booster_.feature_importance(importance_type='gain')\n    _['fold'] = fold_\n    fold_ += 1\n    feature_importance = pd.concat([feature_importance, _], axis=0, sort=False)\n    \n    #recoding models\n    models_reg.append(model_reg)\n    mean_rmse += model_reg.best_score_['valid_0']['rmse']/n_splits\nlog.info('Mean RMSE: {:.3f}'.format(mean_rmse))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e27c0b7fe365882a8ccf11ec641aa42320b167f9"},"cell_type":"code","source":"# creating prediction arrays\nlog.info('Predicting log of revenues...')\ncutoff = 0\ny_preds_train = np.zeros((n_splits, len(X_train)))\ny_preds_test = np.zeros((n_splits, len(X_test)))\nfor i in range(n_splits):\n    y_preds_train[i] = models_reg[i].predict(X_train, num_iteration=models_reg[i].best_iteration_)\n    y_preds_test[i] = models_reg[i].predict(X_test, num_iteration=models_reg[i].best_iteration_)\ny_pred_train = y_preds_train.mean(axis=0)\ny_pred_test = y_preds_test.mean(axis=0)\ny_pred_train[y_pred_train<cutoff]=0\ny_pred_test[y_pred_test<cutoff]=0\ntrain['predicted_log.revenue1'] = y_pred_train\ntest['predicted_log.revenue1'] = y_pred_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"166bb4b314e3b05230c0979f425102cb4cb70b3f"},"cell_type":"code","source":"# Prediction vs target plots\ncutoff=10\n_, __ = cdf(trn_y[trn_y>0])\nplt.plot(_,__)\n\n_,__ = cdf(y_pred_train[y_pred_train>cutoff])\nplt.plot(_,__)\n\nfor i in range(n_splits):\n    _, __ = cdf(y_preds_train[i][y_preds_train[i]>cutoff])\n    plt.plot(_, __)\n\nplt.legend(['target','mean prediction', 'fold 1', 'fold 2','fold 3','fold 4','fold 5'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17e9999ceea8abbadb5c0b99fdfc03c659ef001b"},"cell_type":"code","source":"# Feature importance plot\nfeature_importance['gain_log'] = np.log1p(feature_importance['gain'])\nmean_gain = feature_importance[['gain','feature']].groupby('feature').mean()\nfeature_importance['mean_gain'] = feature_importance['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(5,30))\nsns.barplot(x='gain_log', y='feature', data=feature_importance.sort_values('gain_log', ascending=False))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9b08eb08ab788a027f4c344e79abbaf15459732"},"cell_type":"markdown","source":"**Submission** "},{"metadata":{"trusted":true,"_uuid":"563fac6e5cd54fba7f3700f30f79c208acf6bf2c"},"cell_type":"code","source":"test['PredictedLogRevenue'] = np.expm1(test['predicted_log.revenue1'])\nout = test[['fullVisitorId','PredictedLogRevenue']].groupby('fullVisitorId', axis=0).sum()\nout['PredictedLogRevenue'] = np.log1p(out['PredictedLogRevenue'])\nout.to_csv('submission.csv')\nout.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d349b2b8a2c820bf76b4b1afe38f31ee0a88a7a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e48c6f91a0da3c51ccd89ccc3d6e81c1ed89f719"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}