{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5fb1592e2f5bb93b44e71fcb483e3016c4db80c5"},"cell_type":"markdown","source":"In this kernel, we extract features, categorize target space to target classess and do a simple feature evaluation using random forest.\nFinally we will do a regression with ensemble of lgb regressions.\nIf you want complete EDA on data, check my another kernel in [link](http://www.kaggle.com/smasar/tutorial-preprocessing-processing-evaluation).\nDuring the kernel development we periodically call garbage collector manually for reducing the memory usage.\nAny comment will be appreciated in advance."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import json\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nfrom sklearn import preprocessing\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import GridSearchCV, train_test_split\nfrom sklearn.model_selection import GroupKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f855609bc7456dcf832fbe78c6d1c6bbf35d53dd"},"cell_type":"code","source":"df_train = pd.read_csv(filepath_or_buffer=\"../input/train.csv\",dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)\ndf_test = pd.read_csv(filepath_or_buffer=\"../input/test.csv\",dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aa81f9f8c80ab0f342f79723b40dc9a7da5dd59d"},"cell_type":"markdown","source":"DROPPING EXTRA FEATURES"},{"metadata":{"trusted":true,"_uuid":"e992d09ef9c091de395bf9c50b08e972954da1be"},"cell_type":"code","source":"df_train = df_train.drop([\"date\", \"sessionId\",\"socialEngagementType\", \"visitId\"],axis = 1)\ndf_test = df_test.drop([\"date\", \"sessionId\",\"socialEngagementType\", \"visitId\"],axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98f37e5a6883651df70005be32f844b5cf0ddb1d"},"cell_type":"code","source":"df_train.shape, df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2b473d557729794728ddea7cbd27ac020b7e0c1"},"cell_type":"markdown","source":"**TOTALS**"},{"metadata":{"trusted":true,"_uuid":"59c35ec313b1b3389ad201037a414975266b2297"},"cell_type":"code","source":"totals_columns = [\"transactionRevenue\", \"newVisits\", \"bounces\", \"pageviews\", \"hits\"]\ntmp_totals_df = pd.DataFrame(df_train.totals.apply(json.loads).tolist())[totals_columns]\ndf_train = pd.concat([df_train,tmp_totals_df] , axis=1)\n\ntotals_columns.remove(\"transactionRevenue\")\ntmp_totals_df = pd.DataFrame(df_test.totals.apply(json.loads).tolist())[totals_columns]\ndf_test = pd.concat([df_test,tmp_totals_df],axis = 1 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8a593c0b2d3b89d2ab5a0dc843b0cfe48c1ade8"},"cell_type":"code","source":"df_train[\"transactionRevenue\"] = df_train.transactionRevenue.fillna(0.0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f1785315101e74e1f693560cd2c9e548012500d0"},"cell_type":"markdown","source":"GC"},{"metadata":{"trusted":true,"_uuid":"0a576f8aacbec787ce0d6c2ed015ad6c0f0e678b"},"cell_type":"code","source":"df_train = df_train.drop([\"totals\"],axis=1)\ndf_test = df_test.drop([\"totals\"],axis=1)\ndel tmp_totals_df\ndf_train.shape, df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb44c70c136fb54ef9b13202aaf775bf5e02a2fe"},"cell_type":"markdown","source":"**GEONETWORK**"},{"metadata":{"trusted":true,"_uuid":"77f5b0e4934645b7ed5ff4a6e1004f497c04c404"},"cell_type":"code","source":"geo_columns = [\"continent\",\"subContinent\",\"country\", \"city\",\"region\", \"metro\" , \"networkDomain\"]\ntmp_geo_df = pd.DataFrame(df_train.geoNetwork.apply(json.loads).tolist())[geo_columns]\ndf_train = pd.concat([df_train, tmp_geo_df],axis=1)\n\ntmp_geo_df = pd.DataFrame(df_test.geoNetwork.apply(json.loads).tolist())[geo_columns]\ndf_test = pd.concat([df_test, tmp_geo_df],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c6779bd026c9bfbb953fe9f39be9e4edc10ceaa"},"cell_type":"markdown","source":"GC"},{"metadata":{"trusted":true,"_uuid":"052ec91860a47a3b75ae00c15221c476ab5a7344"},"cell_type":"code","source":"del tmp_geo_df\ndf_train = df_train.drop([\"geoNetwork\"],axis=1)\ndf_test = df_test.drop([\"geoNetwork\"],axis=1)\ndf_train.shape, df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c667197b49ad37df34ab0de6458f2fff19102a94"},"cell_type":"markdown","source":"**DEVICES**"},{"metadata":{"trusted":true,"_uuid":"b3643b421a00048de1d4d24eb468dd3206e25aa9"},"cell_type":"code","source":"devices_columns = [\"browser\",\"deviceCategory\",\"isMobile\",\"operatingSystem\"]\ntmp_device_df = pd.DataFrame(df_train.device.apply(json.loads).tolist())[devices_columns]\ndf_train=pd.concat([df_train,tmp_device_df],axis = 1)\n\ntmp_device_df = pd.DataFrame(df_test.device.apply(json.loads).tolist())[devices_columns]\ndf_test=pd.concat([df_test,tmp_device_df],axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2844c704dd24c9e476fc09b118e49ffebc50167f"},"cell_type":"markdown","source":"GC"},{"metadata":{"trusted":true,"_uuid":"70b387b330b62a5e1707537da62ca8e2ecc8e5d8"},"cell_type":"code","source":"del tmp_device_df \ndf_train = df_train.drop([\"device\"], axis = 1)\ndf_test = df_test.drop([\"device\"], axis = 1)\ndf_train.shape, df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8b6908831d82416e9591fb30635b4752dd199c7"},"cell_type":"markdown","source":"**TRAFFIC**"},{"metadata":{"trusted":true,"_uuid":"776a6bf47748d592c6a3bf03dc1187b353ee708b"},"cell_type":"code","source":"trafficSource_columns = [\"campaign\",\"medium\" , \"source\",\"adContent\",\"isTrueDirect\", \"keyword\",\"referralPath\"]\ntmp_traffic_df = pd.DataFrame(df_train.trafficSource.apply(json.loads).tolist())[trafficSource_columns]\ndf_train = pd.concat([df_train,tmp_traffic_df] , axis = 1 )\n\ntmp_traffic_df = pd.DataFrame(df_test.trafficSource.apply(json.loads).tolist())[trafficSource_columns]\ndf_test = pd.concat([df_test,tmp_traffic_df] , axis = 1 )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a55d6d7f645a2edebc918cdb3d4ae9ee8ddec884"},"cell_type":"markdown","source":"GC"},{"metadata":{"trusted":true,"_uuid":"1d8a86c5ed52383d4ec3247a2e296e07256732fe"},"cell_type":"code","source":"del tmp_traffic_df \ndf_train = df_train.drop([\"trafficSource\"] , axis = 1 )\ndf_test = df_test.drop([\"trafficSource\"] , axis = 1 )\ndf_train.shape , df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29a18f21aa3cabc8ff75ffd1d3775143ef97181e"},"cell_type":"markdown","source":"FILLING NaN"},{"metadata":{"trusted":true,"_uuid":"2eb31d10ca35bc715b343c3c2db72f06ec376ee6"},"cell_type":"code","source":"df_train = df_train.fillna(0)\ndf_test = df_test.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fcac5a43b41c5bf6f3e19bcbd6ea2935ebfebf52"},"cell_type":"markdown","source":"**VISIT_DATE**"},{"metadata":{"trusted":true,"_uuid":"709953c96b4930e02f52b830aaf6445bcf8f6484"},"cell_type":"code","source":"for df in [df_train, df_test]:\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['day_of_week'] = df['date'].dt.dayofweek\n    df['hour_of_day'] = df['date'].dt.hour\n    df['day_of_month'] = df['date'].dt.day","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91cf021e862192ed713831b7255bfce6c181592e"},"cell_type":"markdown","source":"GC"},{"metadata":{"trusted":true,"_uuid":"e3e623295b98c9d40ba35dd4b41f2c746a3d781e"},"cell_type":"code","source":"df_train = df_train.drop([\"date\",\"visitStartTime\"],axis=1)\ndf_test = df_test.drop([\"date\",\"visitStartTime\"],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82d4ddb1bec788233e1b59e64afb632b86dc1311"},"cell_type":"markdown","source":"PREPARING TARGETS "},{"metadata":{"trusted":true,"_uuid":"170215261d84d61f5f4fbe62ab75a32fc4f00328"},"cell_type":"code","source":"df_train[\"transactionRevenue\"] = df_train.transactionRevenue.astype(np.float)\ny_reg = df_train['transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d16b6476d4b207cdaeae11fb4ff9892c08cb881d"},"cell_type":"markdown","source":"GETTING FEATURES TYPES"},{"metadata":{"trusted":true,"_uuid":"3447d22464e83bd33d4619768799829bbce55f0e"},"cell_type":"code","source":"features_type = [\n    (f,df_train[f].dtype) for f in df_train.columns\n]\nfeatures_type","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"05633e5c0e1a69393dfcb0e94db7f08c9e3c7421"},"cell_type":"markdown","source":"change booleans to categorical (lgb works with categorical features and numerical features only)"},{"metadata":{"trusted":true,"_uuid":"84748fd2964849a58b5582bb61ad35d7af0d87ba"},"cell_type":"code","source":"df_train[\"isMobile\"] = df_train.isMobile.astype('str')\ndf_test[\"isMobile\"] = df_test.isMobile.astype('str')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b5a5f85b09b053f64e91dcbaa9176591366deb0f"},"cell_type":"markdown","source":"final dataframe before starting regression"},{"metadata":{"trusted":true,"_uuid":"0c5064d4b103371a362edc17386587a0f23438c4"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d5cf2fec669c873fe9280fabe5d729e4b10ac8b"},"cell_type":"markdown","source":"LABELING FOR GBM COMPATIBILITY"},{"metadata":{"trusted":true,"_uuid":"cd55cb0b14d37d72d7f72b99e5792d246c664f82"},"cell_type":"code","source":"categorical_features = [\n    f for f in df_train.columns if (df_train[f].dtype == 'object') & (f != \"fullVisitorId\")\n]\ncategorical_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6ba4712e91fc4350ed6a315253e06415db528a3"},"cell_type":"code","source":"for f in categorical_features:\n    df_train[f], indexer = pd.factorize(df_train[f])\n    df_test[f] = indexer.get_indexer(df_test[f])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c14abe5de7fdf6870c2e3e89d9295c15d274779"},"cell_type":"code","source":"df_train.shape, df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c59f382f612195a0c64381da91fe31e352594ec"},"cell_type":"code","source":"df_train_rf = df_train.copy()\na_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=98.7),\nb_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=98.8),\nc_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=98.9),\nd_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=99),\ne_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=99.1),\nf_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=99.3),\ng_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=99.5),\nh_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=99.6),\ni_class = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=99.9),\nmax_revenue = np.percentile(a=list(df_train_rf[\"transactionRevenue\"]) ,q=100),\n\n\"\"\"percentiles are: 98.7% percentile: {}, 98.8% percentile: {}, 98.9% percentile: {}, 99.1% percentile: {}, 99.3% percentile: {}, 60% percentile: {},\n99.5% percentile: {}, 99.6% percentile: {}, 99.9% percentile: {}, 100% percentile: {}\"\"\".format(\n    np.percentile(a=a_class ,q=98.7),\n    np.percentile(a=b_class ,q=98.8),\n    np.percentile(a=c_class ,q=98.9),\n    np.percentile(a=d_class ,q=99),\n    np.percentile(a=e_class ,q=99.1),\n    np.percentile(a=f_class ,q=99.3),\n    np.percentile(a=g_class ,q=99.5),\n    np.percentile(a=h_class ,q=99.6),\n    np.percentile(a=i_class ,q=99.9),\n    np.percentile(a=max_revenue ,q=100),\n)\n\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<a_class) , \"rate\"] = 1\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<b_class) & (df_train_rf[\"transactionRevenue\"]>=a_class) , \"rate\"] = 2\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<c_class) & (df_train_rf[\"transactionRevenue\"]>=b_class) , \"rate\"] = 3\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<d_class) & (df_train_rf[\"transactionRevenue\"]>=c_class) , \"rate\"] = 4\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<e_class) & (df_train_rf[\"transactionRevenue\"]>=d_class) , \"rate\"] = 5\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<f_class) & (df_train_rf[\"transactionRevenue\"]>=e_class) , \"rate\"] = 6\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<g_class) & (df_train_rf[\"transactionRevenue\"]>=f_class) , \"rate\"] = 7\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<h_class) & (df_train_rf[\"transactionRevenue\"]>=g_class) , \"rate\"] = 8\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]<i_class) & (df_train_rf[\"transactionRevenue\"]>=h_class) , \"rate\"] = 9\ndf_train_rf.loc[(df_train_rf[\"transactionRevenue\"]>=i_class) , \"rate\"] = 10\n\n\ndf_train_rf[\"rate\"].astype(int)\n\ndf_train_rf = df_train_rf.drop([\"transactionRevenue\"],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7cd5752f330e6abb3e3b1bad5f7113594fe422bf"},"cell_type":"markdown","source":"The data is unbalanced based on the transactionRevenue\nsubsampling and balancing data can improve the accuracy."},{"metadata":{"trusted":true,"_uuid":"68d292eac8355633cbb565ec45a7ebdfb6a1ec17"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrf = RandomForestClassifier(n_estimators=100, max_depth=20,random_state=0)\ntrain, test = train_test_split(df_train_rf, test_size=0.2)\nrf.fit(train.loc[:, train.columns != \"rate\"], train[[\"rate\"]])\nfeature_importance_rev = dict(sorted(zip(map(lambda x: round(x, len(train.columns)), rf.feature_importances_), train.columns),\n                                     reverse=True))\nfeature_importance = dict()\nfor i, j in feature_importance_rev.items():\n    feature_importance[j] = i\nfeature_importance\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfeature_importance_df = pd.DataFrame.from_dict(feature_importance,orient=\"index\")\nfeature_importance_df = feature_importance_df.rename(index=str, columns={0: \"importance\"})\nax = feature_importance_df.plot(kind='bar', figsize = (12,8))\nplt.bar(feature_importance_df.index,feature_importance_df[\"importance\"])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4602322bdb9761b8cb578de9f51f2104cb8010f"},"cell_type":"markdown","source":"**SYMPLE REGRESSION**\n\nOn the following simple regression ensembling will be deployed based on the features extracted."},{"metadata":{"trusted":true,"_uuid":"01d89bb43c29e398bc1e7afe82b2deb164ab550a"},"cell_type":"code","source":"def get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6573165c7510b8688912d11c70e8f62ce82e993e"},"cell_type":"markdown","source":"Ensembling with 5 subsets ...\nAbsolutely increasing the folds can increase the performance."},{"metadata":{"trusted":true,"_uuid":"a7346ef3e38c5a024a1014438761cfcf3f1c61ec"},"cell_type":"code","source":"df_train.fullVisitorId = df_train[\"fullVisitorId\"].astype('str')\nfolds = get_folds(df=df_train, n_splits=5)\ntrain_features = [f for f in df_train.columns if f not in  [\"fullVisitorId\",\"transactionRevenue\"]]\ntrain_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"655709d42a6d3e42011e1f2366cb678e8f80632c"},"cell_type":"code","source":"importances = pd.DataFrame()\noof_reg_preds = np.zeros(df_train.shape[0])\nsub_reg_preds = np.zeros(df_test.shape[0])\nfor fold_, (trn_, val_) in enumerate(folds):\n    trn_x, trn_y = df_train[train_features].iloc[trn_], y_reg.iloc[trn_]\n    val_x, val_y = df_train[train_features].iloc[val_], y_reg.iloc[val_]\n    reg = lgb.LGBMRegressor(\n        num_leaves=31,\n        learning_rate=0.03,\n        n_estimators=1000,\n        subsample=.9,\n        colsample_bytree=.9,\n        random_state=1\n    )\n    reg.fit(X=trn_x,y=np.log1p(trn_y.astype(np.float)),\n        eval_set=[(val_x, np.log1p(val_y.astype(np.float)))],\n        early_stopping_rounds=50,\n        verbose=100,\n        eval_metric='rmse'\n    )\n    imp_df = pd.DataFrame()\n    imp_df['feature'] = train_features\n    imp_df['gain'] = reg.booster_.feature_importance(importance_type='gain')\n    \n    imp_df['fold'] = fold_ + 1\n    importances = pd.concat([importances, imp_df], axis=0)\n    \n    oof_reg_preds[val_] = reg.predict(val_x, num_iteration=reg.best_iteration_)\n    oof_reg_preds[oof_reg_preds < 0] = 0\n    _preds = reg.predict(df_test[train_features], num_iteration=reg.best_iteration_)\n    _preds[_preds < 0] = 0\n    sub_reg_preds += np.expm1(_preds) / len(folds)\n    \nmean_squared_error(np.log1p(y_reg.astype(np.float)), oof_reg_preds) ** .5","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fd7be256a86932479f3aed558668fb10696d34d9"},"cell_type":"markdown","source":"prepration for submission ..."},{"metadata":{"trusted":true,"_uuid":"367c348bae68699ba2b2f55975852cd7925aa16e","_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"df_test['PredictedLogRevenue'] = np.log1p(sub_reg_preds)\npre_submission = df_test[[\"fullVisitorId\" , \"PredictedLogRevenue\"]].groupby(by=[\"fullVisitorId\"]).mean()\npre_submission.to_csv(path_or_buf=\"regression_final.csv\", index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}