{"cells":[{"metadata":{"_uuid":"e8e112aa5b5db7303e39d3a9cf1702848962c6c2"},"cell_type":"markdown","source":"### Introduction\nYou can refer the data analysis with this kernel : \nhttps://www.kaggle.com/super13579/basic-feature-analysis-date-categorical-revenue\n* Keep try different feature to find the key feature on training\n* Use K cross validate to find the best score\n* After find the best score , try to do ensemble learning\n"},{"metadata":{"_uuid":"3249aaa68de87bedff790ce737eefa9407e8d99f"},"cell_type":"markdown","source":"## Feature process in this kernel\n* Process Date feature (add day, week, hour, \"revisit time\")\n* Totals feature don't do anything, only fillna(0)\n* Do one hot encoding for categorical feature that have unique value <15\n* Do ranking encoding for categorical feature that have unique value >15"},{"metadata":{"_uuid":"01428e29aa8cfc464a455d6f26476317c0af0117"},"cell_type":"markdown","source":"## Model used in this kernel\nLightGBM with K-Cross validation, K=5"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nimport gc\nimport time\nfrom pandas.core.common import SettingWithCopyWarning\nimport warnings\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\n\n# I don't like SettingWithCopyWarnings ...\nwarnings.simplefilter('error', SettingWithCopyWarning)\ngc.enable()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8c6c0d5d8cf00737b8ae536b0d528b32fac9166"},"cell_type":"markdown","source":"### Get the extracted data\n* preprocessed dataset by olivier https://www.kaggle.com/ogrellier/create-extracted-json-fields-dataset"},{"metadata":{"trusted":true,"_uuid":"6fe04db087c3d25b8b4a5873ee6f104e427525f0"},"cell_type":"code","source":"train = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', \n                    dtype={'date': str, 'fullVisitorId': str, \"visitId\":str, 'sessionId':str}, nrows=None)\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', \n                   dtype={'date': str, 'fullVisitorId': str, \"visitId\":str, 'sessionId':str}, nrows=None)\n\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d38f36c4f89315b617872c089a540a54619ed68a"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75f8c9b709401e43f6abedc4df87bb1727b7b020"},"cell_type":"code","source":"y_reg = train['totals.transactionRevenue'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5dd71de5f74267e2bbbdc5468b4c3100228fd6e"},"cell_type":"markdown","source":"## Add Date feature\n* add revisit features\n* add data extract feature"},{"metadata":{"trusted":true,"_uuid":"59f453ac48c14cc41c7e9ddc266d5e5255402bbf"},"cell_type":"code","source":"train['target'] = y_reg\ndef extract_new_feature(df): \n    print(\"Start extract date...\")\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['day_of_week'] = df['date'].dt.dayofweek\n    df['hour'] = df['date'].dt.hour\n    df['day'] = df['date'].dt.day\n    df['month'] = df['date'].dt.month\n    print(\"Finished extract date...\")\nextract_new_feature(train)\nextract_new_feature(test)\n\ndef add_time_period_of_same_ID(df): \n    print(\"Start add time period feature...\")\n    df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)\n    df['next_revisit_time'] = (\n        df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)\n    ).astype(np.int64) // 1e9 // 60 // 60\n    df['prev_revisit_time'] = (\n        df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)\n    ).astype(np.int64) // 1e9 // 60 // 60\n    print(\"Finished time periodfeature...\")\n    \nadd_time_period_of_same_ID(train)\nadd_time_period_of_same_ID(test)\ny_reg = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a44c9cbcc3f8cae0397811782ed690c8efda09f"},"cell_type":"markdown","source":"## Find categorical features"},{"metadata":{"trusted":true,"_uuid":"5d68951a42961d0ec048cece25825b0c22bb4d33"},"cell_type":"code","source":"categorical_features_train = train.select_dtypes(include=[np.object])\ncategorical_features_test = test.select_dtypes(include=[np.object])\ncategorical_features_train.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c64e206190d37dfc5783d3ea803bd40aa9b32db9"},"cell_type":"markdown","source":"## Process totals features\n* fill nan feature to 0"},{"metadata":{"trusted":true,"_uuid":"738c406861dce2b6e168563091e20a16d9c84656"},"cell_type":"code","source":"train['totals.pageviews']=train['totals.pageviews'].astype('float64')\ntrain['totals.hits']=train['totals.hits'].astype('float64')\ntest['totals.pageviews']=test['totals.pageviews'].astype('float64')\ntest['totals.hits']=test['totals.hits'].astype('float64')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"240084dd9a6f1fa301e0379418fd27318709984d"},"cell_type":"markdown","source":"## Do one hot encoding for categorical unique count <10"},{"metadata":{"trusted":true,"_uuid":"546928364d11bc2fbe46ded48fdb83f276a13f7f"},"cell_type":"code","source":"df_combine=pd.concat([train,test],ignore_index=True)\nprint(df_combine.shape)\n#Find One_hot features that unique count <15\none_hot_features = df_combine[list(categorical_features_test)].nunique().reset_index()\none_hot_features.columns = ['features','unique_count']\none_hot_features = one_hot_features.loc[one_hot_features['unique_count'] < 10,\"features\"]\none_hot_features = list(one_hot_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cfb3cea00338413db005cca696712a7fed76589"},"cell_type":"code","source":"#Process one_hot_features\nfor i in one_hot_features:\n    print(\"Process feature =====>\"+str(i))\n    df_combine[\"one_hot_feature\"] = df_combine[i]\n    df_combine[\"one_hot_feature\"] =  str(i) + \".\" + df_combine[\"one_hot_feature\"].astype('str')\n    one_hot_combine = pd.get_dummies(df_combine[\"one_hot_feature\"])\n    print(one_hot_combine.shape)\n    df_combine = df_combine.join(one_hot_combine)\n    del df_combine[\"one_hot_feature\"]\n    del df_combine[i]\n    del one_hot_combine\n    print(df_combine.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"84db9c68d972ef8d031e52bdd6c4648d8667e2ab"},"cell_type":"code","source":"train = df_combine[:len(train)]\nprint(train.shape)\ntest = df_combine[len(train):]\nprint(test.shape)\ndel df_combine","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a556f77293c806b1ffb28b3c9933af79cbdcab3"},"cell_type":"markdown","source":"## Do ranking encoding  for categorical unique count >10\n* Idea comes from rahal's kernel : https://www.kaggle.com/rahullalu/gstore-eda-lgbm-baseline-1-4260"},{"metadata":{"trusted":true,"_uuid":"13b61e48a9424859837fd7fcad8e43f2809ecc1e"},"cell_type":"code","source":"excluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime']\ncategorical_larger_15_feature = [i for i in categorical_features_train if i not in one_hot_features and i not in excluded_features]\nprint(categorical_larger_15_feature)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d240c00fc21eef1793b4d69a461f54d35a3390c"},"cell_type":"markdown","source":"* Cause referral Path and networkDomain have many unique count, let's bypass it "},{"metadata":{"trusted":true,"_uuid":"d27d27322429e916364a0cb17a8f448921a86be8"},"cell_type":"code","source":"not_do_ranking =['trafficSource.referralPath','geoNetwork.networkDomain']\nranking_feature = [i for i in categorical_larger_15_feature if i not in not_do_ranking]\nprint(ranking_feature)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"592d8110339718377e814b356026e58ea114a311"},"cell_type":"code","source":"def Ranking_process(df,df_test):\n    for col in ranking_feature:\n        if df[col].dtype=='object':\n            print(\"Process Ranking of \"+str(col)+\" feature...\")\n            df[col].fillna('others',inplace=True)\n            col_list=[col,'totals.transactionRevenue']\n            df_gropby=df[col_list].fillna(0).groupby(col).mean().reset_index()\n            df_gropby.columns = col_list\n            df_gropby['rank']=df_gropby['totals.transactionRevenue'].rank(ascending=1)\n            replace_dict={}\n            final_dict={}\n            for k,col_val in enumerate(df_gropby[col].values):\n                replace_dict[col_val]=df_gropby.iloc[k,2]\n            final_dict[col]=replace_dict\n            df[col]=df[col].map(replace_dict)\n            df_test[col]=df_test[col].map(replace_dict)\n            #df.replace(final_dict,inplace=True)\n            del df_gropby,replace_dict,final_dict\n            gc.collect()\n            print(\"Finished process Ranking of \"+str(col)+\" feature\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"feb1928e899fac566b9496a86e76ab745995a24a"},"cell_type":"code","source":"Ranking_process(train,test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d207a80600a44e758b6011f24c5808f9b164866"},"cell_type":"markdown","source":"## Factorize other categoricals features"},{"metadata":{"trusted":true,"_uuid":"9e9c1c6b85cb25030b8107fde17ea17a1587c3cc"},"cell_type":"code","source":"excluded_features = [\n    'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', \n    'visitId', 'visitStartTime'\n]\n\ncategorical_features = [\n    _f for _f in train.columns\n    if (_f not in excluded_features) & (train[_f].dtype == 'object')\n]\nprint(categorical_features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f00eb6b6166cd5be25ebad22aa9dc91a8a3ded2"},"cell_type":"code","source":"\nfor f in categorical_features:\n    train[f], indexer = pd.factorize(train[f])\n    test[f] = indexer.get_indexer(test[f])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e0c2150eb3a7dccfafc9229a9635855d6f878de"},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7517b473bd776f51925ab0243cf1e3b6592ce0e"},"cell_type":"markdown","source":"## Training model with K cross validation (Light GBM)\n* reference this kernel : https://www.kaggle.com/ogrellier/i-have-seen-the-future"},{"metadata":{"trusted":true,"_uuid":"4cbcad3c4d8e62f95630f58468dfc411b7443679"},"cell_type":"code","source":"folds = get_folds(df=train, n_splits=5)\n\ntrain_features = [_f for _f in train.columns if _f not in excluded_features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9c155197545dc43ec2f3e23e25955532f7ad25d","scrolled":false},"cell_type":"code","source":"importances = pd.DataFrame()\noof_reg_preds = np.zeros(train.shape[0])\nsub_reg_preds = np.zeros(test.shape[0])\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = train[train_features].iloc[trn_], y_reg.iloc[trn_]\n    val_x, val_y = train[train_features].iloc[val_], y_reg.iloc[val_]\n    \n    reg = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1\n    )\n    reg.fit(\n        trn_x, np.log1p(trn_y),\n        eval_set=[(val_x, np.log1p(val_y))],\n        early_stopping_rounds=50,\n        verbose=100,\n        eval_metric='rmse'\n    )\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0, sort=False)\n    \n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    _preds = reg.predict(test[train_features], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_reg_preds += np.expm1(_preds) / len(folds)\n    \nmean_squared_error(np.log1p(y_reg), oof_reg_preds) ** .5\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b23cdb7ee1e35c6deba9edd6ffd87a4cdc72920"},"cell_type":"markdown","source":"### Display feature importances"},{"metadata":{"trusted":true,"_uuid":"61d4aacc57c2c7c4d9ce0b1cab7221ca7c69138f"},"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore', FutureWarning)\n\nimportances['gain_log'] = np.log1p(importances['gain'])\nmean_gain = importances[['gain', 'feature']].groupby('feature').mean()\nimportances['mean_gain'] = importances['feature'].map(mean_gain['gain'])\n\nplt.figure(figsize=(8, 12))\nsns.barplot(x='gain_log', y='feature', data=importances.sort_values('mean_gain', ascending=False))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c64447b5858dc899b6b78d26a4e68a8582995051"},"cell_type":"markdown","source":"### Create user level predictions"},{"metadata":{"trusted":true,"_uuid":"1fc92504b40407cff766489db1966e729d0466df"},"cell_type":"code","source":"train['predictions'] = oof_reg_preds\ntest['predictions'] = np.log1p(sub_reg_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a08e90b3b572016f149acc6304135711ae3e4e36"},"cell_type":"code","source":"test_result = test[['fullVisitorId','predictions']].groupby('fullVisitorId').sum().reset_index()\ntrain_result = train[['fullVisitorId','predictions']].groupby('fullVisitorId').sum().reset_index()\ntest_result.columns = ['fullVisitorId','PredictedLogRevenue']\ntrain_result.columns = ['fullVisitorId','PredictedLogRevenue']\ntest_result.to_csv('Ranking_onehot_test.csv',index = False)\ntrain_result.to_csv('Ranking_onehot_train.csv',index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8eb52c144da60b24ff57f709cd83ab42ea055fc4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73a0db5cfa89fa95a120d6c02b768272f30bca2b"},"cell_type":"markdown","source":"### Train a model at Visitor level"},{"metadata":{"trusted":true,"_uuid":"6b71545579a68255d07bb6c77c201cf948a33e4f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}