{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac38ca4720ea785e7ed2ae6475ac87ea8d1dfbc0"},"cell_type":"code","source":"import os\nimport json\nfrom pandas.io.json import json_normalize\n%matplotlib inline\nimport seaborn as sns\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2155adbd8357ba8af15d9989d2a5f02ef11c7309"},"cell_type":"markdown","source":"# Data Imput"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Loading data and flattening JSON columns\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     # Set the date, fullVisitorId, sessionId as string of constant\n                     dtype={'date': str, 'fullVisitorId': str, 'sessionId': str}, \n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(list(df[column]))\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9531c4c9a1588113dcbd0867e2d591d55d7c99a8"},"cell_type":"code","source":"%%time\ntrain_df = load_df()\ntest_df = load_df('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c185f2cae5dcce0acc2237fe5415985ab29505d3"},"cell_type":"markdown","source":"# EDA\n## Missing Value Detection"},{"metadata":{"trusted":true,"_uuid":"c09169cc907e7a413b496e0a18c056698d303ba7"},"cell_type":"code","source":"def na_detect(df):\n    total = df.isnull().sum().sort_values(ascending = False)\n    percent = (df.isnull().sum() / df.isnull().count() * 100 ).sort_values(ascending = False)\n    df_opt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n    \n    plt.figure(figsize=(20,20))\n    fig, ax = plt.subplots()\n    col_na = total[total>0]\n    bar_na = ax.barh(col_na.index, col_na.values, 0.8)\n    for i, v in enumerate(col_na.values):\n        ax.text(v + 5, i - .15 , str(v), color='red')#, fontweight='bold')\n    plt.title('Variables with Missing Value')\n    plt.xlabel('Quantity of Missing Value')\n    plt.ylabel('Columns')\n    plt.show()\n    \n    print (df_opt[~(df_opt['Total'] == 0)])\n    \n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"82c9b648dde7139cf2c112cb8773e81b5131e65f"},"cell_type":"code","source":"na_detect(train_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7280d199a134c3a43833269208f649c4d29c567"},"cell_type":"markdown","source":"## Difference Between Train and Test Dataset"},{"metadata":{"trusted":true,"_uuid":"dcaba518327b2e92aa408b45e3a6ca194e6f4fb1"},"cell_type":"code","source":"set(train_df.columns).difference(set(test_df.columns))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2bbec582a21702dbee9b5bc518121590ca69b5bc"},"cell_type":"markdown","source":"Besides the response variable 'totals.transactionRevenue', the training set also has a column 'trafficSource.campaignCode'  which doesn't exist in test set."},{"metadata":{"_uuid":"c2e89647cca4772ba45640a4291a6c869624de43"},"cell_type":"markdown","source":"## TimeStamp/Date Conversion  "},{"metadata":{"trusted":true,"_uuid":"3789bf2262363a108b25dc3751b3c730c399ba32"},"cell_type":"code","source":"def date_convert(df):\n    df['visitdate'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    #df['visitdate'] = pd.datetime.utcfromtimestamp(test_df['visitStartTime'])\n    df['wday'] = df['visitdate'].dt.weekday\n    df['hour'] = df['visitdate'].dt.hour\n    df['day'] = df['visitdate'].dt.day\n    df['month'] = df['visitdate'].dt.month\n    return    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b41e74a3d24b1d69e5572aa5f0367f646660754f"},"cell_type":"code","source":"for df in [train_df, test_df]:\n    date_convert(df)\nprint('TrainSet:', train_df.shape)\nprint('TestSet:', test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7cbe02645b1d6796c09049d08ebe725f9e49a311"},"cell_type":"markdown","source":"## Constant Variable Detection and Removal"},{"metadata":{"trusted":true,"_uuid":"b8efaf1fcca6dae2119d37dbd9489f8101f5d0a3"},"cell_type":"code","source":"def constant_process(df):\n    num_constant = 0\n    constant_cols = []\n    for col in df.columns:\n        if df[col].nunique()==1:\n            constant_cols.append(col)\n            num_constant = num_constant+1\n            \n    print('Number of Constant Variables:', num_constant)\n    print(constant_cols)\n    df = df.drop(constant_cols, axis=1)\n    print('Shape: ', df.shape)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44b62dde8a7b5005b87582ada7d10205a4490a32"},"cell_type":"code","source":"ctrain_df = constant_process(train_df)\nctest_df = constant_process(test_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e27bc0bf177de7b26bfb8ab492f1ece5ee0a7c6e","trusted":true},"cell_type":"code","source":"print('Unique Variables in Train:', ctrain_df['sessionId'].nunique())\nprint('Unique Variables in Test:', ctest_df['sessionId'].nunique())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29a4487df3fe77b54bfaf12ad6d19ea2f08e69b1"},"cell_type":"markdown","source":"We found the 'sessionId' exists duplicate which is wired because it is supposed to be unique as identifier."},{"metadata":{"trusted":true,"_uuid":"3e45b59a5e665fbe32c54effdbc3a7535c25d704"},"cell_type":"code","source":"dup_session = ctrain_df[ctrain_df.duplicated(subset='sessionId', keep=False)].sort_values('sessionId',ascending = False)\ndup_session.head(2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e687f218d76be978b179e31546d0cc7e58db9829"},"cell_type":"markdown","source":"The rest of columns with missing values are trafficSource.keyword and trafficSource.referralPath."},{"metadata":{"_uuid":"aa535cdca020d2061038bac4869dc16600e446ba"},"cell_type":"markdown","source":"# Numerical Variables Processing\nConsidering the 'totals' of nunerical variables, we convert them into numerical type of float and replace the NAs in 'totals.transactionRevenue' with 0."},{"metadata":{"trusted":true,"_uuid":"b4ae5c8cb9028667df6a67f996bd2d0de2c30672"},"cell_type":"code","source":"ctrain_df[\"totals.transactionRevenue\"].fillna(0, inplace=True)\nctrain_df['totals.transactionRevenue'] = ctrain_df['totals.transactionRevenue'].astype(int)\nctrain_df['totals.hits'] = ctrain_df['totals.hits'].astype(int)\nctrain_df['totals.pageviews'] = ctrain_df['totals.hits'].astype(int)\nctest_df['totals.hits'] = ctest_df['totals.hits'].astype(int)\nctest_df['totals.pageviews'] = ctest_df['totals.hits'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6333bc8ea8c73adc89a54cc272fdbbf74c0edfe"},"cell_type":"markdown","source":"# Numerical Feature Distribution"},{"metadata":{"_uuid":"1118f1cc404baeb45c8ab8f21e6aeadd84d94eb6","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(14,5))\nplt.subplot(1,2,2)\nax = sns.distplot(np.log1p(ctrain_df[ctrain_df['totals.transactionRevenue'] > 0][\"totals.transactionRevenue\"]), kde=True)\nax.set_xlabel('Transaction Revenue Log', fontsize=15)\nax.set_ylabel('Distribuition', fontsize=15)\nax.set_title(\"Distribuition of Revenue Log\", fontsize=20)\nplt.subplot(1,2,1)\nsns.distplot(ctrain_df[\"totals.transactionRevenue\"], kde=True)\nplt.xlabel('Transaction Revenue', fontsize=15)\nplt.ylabel('Distribuition', fontsize=15)\nplt.title(\"Distribuition of Revenue\", fontsize=20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e62ea0767563e792e1057d2ded55236da75c2274"},"cell_type":"markdown","source":"The Revenue is typically long-tail distributed but the effective revenue which is greater than 0 is approximately norally distributed. We also check the missing values in the valid Revenue rows as follow."},{"metadata":{"_uuid":"df7323c90f0834725349cf469284d622bbbdaa7b","trusted":true},"cell_type":"code","source":"valid_df = ctrain_df[ctrain_df['totals.transactionRevenue'] > 0]\nna_detect(valid_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc2495b3d6836b7c1fa6d2e793685d3ee31fbdd9"},"cell_type":"markdown","source":"We remove the columns with over 95% missing values."},{"metadata":{"trusted":true,"_uuid":"1bd4435ca4169afb473dfe0f1e408e5b29ebd400"},"cell_type":"code","source":"ctrain_df = ctrain_df.drop(['trafficSource.adContent', 'trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId', 'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot'], axis=1)\nprint('Train Shape: ' ,ctrain_df.shape)\nctest_df = ctest_df.drop(['trafficSource.adContent', 'trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId', 'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot'], axis=1)\nprint('Test Shape:' ,ctest_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81081ff44eb9762f82ac00b508bd23655d0321e3"},"cell_type":"code","source":"plt.figure(figsize=(14,5))\nplt.subplot(1,2,1)\nsns.distplot(ctrain_df[\"totals.hits\"], kde=True)\nplt.xlabel('Hits', fontsize=15)\nplt.ylabel('Distribuition', fontsize=15)\nplt.title(\"Distribuition of Hits\", fontsize=20)\nplt.subplot(1,2,2)\nsns.distplot(ctrain_df[\"totals.pageviews\"], kde=True)\nplt.xlabel('Page Views', fontsize=15)\nplt.ylabel('Distribuition', fontsize=15)\nplt.title(\"Distribuition of Page Views\", fontsize=20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21d7b892cde67215da0c0dc9780396a92d36d757"},"cell_type":"markdown","source":"According  to the long-tail distribution of 'totals.pageviews', we replace the missing values with medians. "},{"metadata":{"trusted":true,"_uuid":"ffb509ae3c2e85838cff995eb9c482a3238fe86c"},"cell_type":"code","source":"ctrain_df[\"totals.pageviews\"].fillna(value=ctrain_df['totals.pageviews'].median(), inplace=True)\nctest_df[\"totals.pageviews\"].fillna(value=ctest_df['totals.pageviews'].median(), inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4464ef2a7ee111713b8812d2058f69f68d5e7aac"},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nsns.distplot(ctrain_df[\"visitNumber\"], kde=True)\nplt.xlabel('Visit Number', fontsize=15)\nplt.ylabel('Distribuition', fontsize=15)\nplt.title(\"Distribuition of Visit Number\", fontsize=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad2d7741d8c6973f76b110e5029961e17e6e98d2"},"cell_type":"markdown","source":"+ In addition, all numerical features including visitNumber, pageviews, and hits are measured in same scale level. Therefore, we don't need to normalize them.\n+ All numerical variables we concern are long-tail distributed and continuous, therefore we can't use Correlation Coefficient to measure the correlations. Therefore, we decide to build a baseline tree-based model to measure the feature importance.\n"},{"metadata":{"_uuid":"f2139fc3114a6de9d1961125d5552cddcade9302"},"cell_type":"markdown","source":"# Discrete Variables Processing - One Hot Encoding"},{"metadata":{"trusted":true,"_uuid":"631e1ba400b559687b7b4c1c960f189d72cadf8b"},"cell_type":"code","source":"non_relevant = [\"date\", \"fullVisitorId\", \"sessionId\", \"visitId\", \"visitStartTime\", \"visitdate\", \"totals.transactionRevenue\"]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8ea904b6523e0afa8e68610d055546a9cff9ee3"},"cell_type":"markdown","source":"We use Label Encoding to save memory because of the tree-based models."},{"metadata":{"trusted":true,"_uuid":"894745f294dc79ebb6f604cff0a0724c9df0ad15"},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\ncategorical_cols = [c for c in ctrain_df.columns if not c.startswith(\"total\")]\ncategorical_cols = [c for c in categorical_cols if c not in non_relevant]\nfor c in categorical_cols:\n\n    le = LabelEncoder()\n    train_vals = list(ctrain_df[c].values.astype(str))\n    test_vals = list(ctest_df[c].values.astype(str))\n    \n    le.fit(train_vals + test_vals)\n    \n    ctrain_df[c] = le.transform(train_vals)\n    ctest_df[c] = le.transform(test_vals)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f854faa22483ef78915292d44f169383e830cb84"},"cell_type":"markdown","source":"# Response Variable"},{"metadata":{"trusted":true,"_uuid":"f67877cc1db0b8969bd7c5c9879bafb35523040a"},"cell_type":"code","source":"train_y = ctrain_df['totals.transactionRevenue']\ndel ctrain_df['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fda9cc9e9e76a515ea13249e1ca6a2eb93f7ecd4"},"cell_type":"markdown","source":"# Cross Validation "},{"metadata":{"trusted":true,"_uuid":"6a7c17070eb14ad1aec6dfad0afcc63b09401926"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    unique_sessions = np.array(sorted(df['sessionId'].unique()))\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for dev_s, val_s in folds.split(X=unique_sessions, y=unique_sessions, groups=unique_sessions):\n        fold_ids.append(\n            [\n                ids[df['sessionId'].isin(unique_sessions[dev_s])],\n                ids[df['sessionId'].isin(unique_sessions[val_s])]\n            ]\n        )\n\n    return fold_ids\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e45fb80543530f3d969632001a989095ed36b852"},"cell_type":"code","source":"import lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"09f1265f7fc66ae2c0dc3503e3e4326010767a79"},"cell_type":"markdown","source":"# LightBM"},{"metadata":{"trusted":true,"_uuid":"e53f6fa8b19c61142eb3c4012576945c05689bee","scrolled":true},"cell_type":"code","source":"%%time\nfeatures = [f for f in ctrain_df.columns if f not in non_relevant]\nprint(features)\n\nfolds = get_folds(df=ctrain_df, n_splits=5)\n\nimportances = pd.DataFrame()\ndev_reg_preds = np.zeros(ctrain_df.shape[0])\nval_reg_preds = np.zeros(ctest_df.shape[0])\n\nfor f, (dev, val) in enumerate(folds):\n    dev_x, dev_y = ctrain_df[features].iloc[dev], train_y.iloc[dev]\n    val_x, val_y = ctrain_df[features].iloc[val], train_y.iloc[val]\n    \n    reg = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1\n    )\n    \n    reg.fit(\n        dev_x, np.log1p(dev_y),\n        eval_set=[(val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        verbose=100,\n        eval_metric='rmse'\n    )\n    \n    importance_df = pd.DataFrame()\n    importance_df['feature'] = features\n    importance_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    importance_df['fold'] = f + 1\n    importances = pd.concat([importances, importance_df], axis=0, sort=False)\n    dev_reg_preds[val] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    dev_reg_preds[dev_reg_preds < 0] = 0\n    preds = reg.predict(ctest_df[features], num_iteration=reg.best_iteration_)\n    preds[preds < 0] = 0\n    val_reg_preds += np.expm1(preds)/len(folds)\nprint('RMSE=' ,metrics.mean_squared_error(np.log1p(train_y), dev_reg_preds) ** .5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e70a6d2c50b23579b1097c67b38174a33d72e15"},"cell_type":"code","source":"val_reg_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8084ff416a9f565bf81434afb67fe02ccc2fc149"},"cell_type":"markdown","source":"# Feature Importance"},{"metadata":{"trusted":true,"_uuid":"152190972ead2a87000b0672c9a1988eb00b998e","scrolled":true},"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nimportances['gain_log'] = np.log1p(importances['gain'])\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b301064c5cec0b73c561b6255fc3a76c321d79e6"},"cell_type":"markdown","source":"# Prediction and Submission"},{"metadata":{"trusted":true,"_uuid":"02addfe938fb7561acf9e094220c1e5ffd1e7444"},"cell_type":"code","source":"ctest_df[\"PredictedLogRevenue\"] = val_reg_preds\nsubmission = ctest_df.groupby(\"fullVisitorId\").agg({\"PredictedLogRevenue\" : \"sum\"}).reset_index()\nsubmission[\"PredictedLogRevenue\"] = np.log1p(submission[\"PredictedLogRevenue\"])\nsubmission[\"PredictedLogRevenue\"] =  submission[\"PredictedLogRevenue\"].apply(lambda x : 0.0 if x < 0 else x)\nsubmission.to_csv(\"baseline.csv\", index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}