{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom datetime import datetime\n\nfrom sklearn.model_selection import GridSearchCV\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nfrom os.path import join as pjoin\n\ndata_root = '../input/make-data-ready'\nprint(os.listdir(data_root))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fed79ee0e6bd26d348d02b8c7b59886c3afe22c2"},"cell_type":"markdown","source":"# Import and load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nimport xgboost as xgb\n\nfrom pprint import pprint\nimport math\n\nfrom scipy.stats import kurtosis, skew\n\nfrom IPython import embed\nfrom IPython.terminal.embed import InteractiveShellEmbed\n\nfrom sklearn.model_selection import KFold\n\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42d57caae8977328d06ce1191632cd93f93cceff"},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport shap\nplt.rcParams['figure.figsize'] = (12,6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3cc9e31f9c51f1fd66593d2fd0cb3963e880f12"},"cell_type":"code","source":"def load_data(data='train',n=2):\n    df = pd.DataFrame()\n    for i in range(n) :\n        if data=='train':\n            if i > 8 :\n                break\n            dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))\n        elif data=='test':\n            if i > 2 :\n                break\n            dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))\n        df = pd.concat([df,dfpart])\n        del dfpart\n    return df\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"711b2ff1d23636474cbf8cbd63ed852d4853499d","scrolled":false},"cell_type":"code","source":"%%time\ndf_train = load_data(n=9)\ndf_test = load_data('test',n=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfebbead5b39c0d8099676bf9b5e396903c17e81"},"cell_type":"code","source":"# delete date-related features\ndel_cols=[item for item in df_train.columns if \"Date\" in item]\ndf_train=df_train.drop(del_cols,axis=1).copy()\ndf_test=df_test.drop(del_cols,axis=1).copy()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"510b48a17ef755b0115c0cb303ce9fbb08233513"},"cell_type":"code","source":"df_all = pd.concat([df_train,df_test]).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"660a0bb0b6ac94d469e290bddbf0b32ca55bf0ff"},"cell_type":"code","source":"print(f'# of columns has na value: {(df_test.isnull().sum().sort_values(ascending=False) > 0).sum()}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15c61070c623b87850a9d8b4bb98e5a106fab002"},"cell_type":"code","source":"# train_v2.csv - from August 1st 2016 to April 30th 2018.\n# test_v2.csv - from May 1st 2018 to October 15th 2018.\n# sample_submission_v2.csv - from December 1st 2018 to January 31st 2019\n\n# divide df_all into\n# df_train_x - from August 1st 2016 to June 30st 2018, 5+12+6=23 months\n# <-> 1.5 month (Jul, Aug * 0.5)\n# df_train_y -  from August 15th 2018 to October 15th 2018, 2 months\n# -> divide new_test into in-sample and out-of-sample to avoid overfitting\n#   (only includes visitors Jan 15st 2018- June 30th 2018)\n\ndf_train_x = df_all[df_all.date <= \"2018-06-30\"].copy()\ndf_train_y = df_all[df_all.date >= \"2018-08-15\"].copy()\n\nid_train = df_all[(df_all.date >= \"2018-01-15\") & (df_all.date <= \"2018-06-30\")].fullVisitorId.drop_duplicates()\ndf_train_x = df_train_x[df_train_x.fullVisitorId.isin(id_train)].copy()\ndf_train_y = df_train_y[df_train_y.fullVisitorId.isin(id_train)].copy()\n\n# Apply the trained model to \n# df_test_x - from November 15th 2016 to October 15th 2018, 23 months\n# <-> 1.5 month (Oct *0.5, Nov)\n# df_test_y - from December 1st 2018 to January 31st 2019, 2 months \n#   (only includes visitors May 1st 2018- Oct 15th 2018)\n\ndf_test_x = df_all[(df_all.date >= \"2016-11-15\") & (df_all.date <= \"2018-10-15\")].copy()\nid_test = df_all[(df_all.date >= \"2018-05-01\") & (df_all.date <= \"2018-10-15\")].fullVisitorId.drop_duplicates()\ndf_test_x = df_test_x[df_test_x.fullVisitorId.isin(id_test)].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62574f7c4633b9ef0ff520eced902e2dab335463"},"cell_type":"code","source":"print({\"all\":df_all.shape,\n       \"train_x\":df_train_x.shape,\n       \"train_y\":df_train_y.shape,\n       \"test_x\":df_test_x.shape,\n       \"id_train\":id_train.shape,\n       \"id_test\":id_test.shape})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d1cce1b74b954505eef1cfd50663cd949191be0"},"cell_type":"code","source":"#sns.lineplot(data=df_train_y.groupby(\"date\")[\"fullVisitorId\"].count())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce4b764fa1da023f498e0214c4b9370d2ba8a9b1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91b28125bce1d482a8cbaee3861b5956f6278618"},"cell_type":"code","source":"#df_all.groupby(\"fullVisitorId\")[\"totals_transactionRevenue\"].sum().describe()\n#df_train.groupby(\"fullVisitorId\")[\"totals_transactionRevenue\"].sum().describe()\n#df_test.groupby(\"fullVisitorId\")[\"totals_transactionRevenue\"].sum().describe()\n#df_all.groupby(\"fullVisitorId\")[\"device_deviceCategory\"].agg(lambda x:x.value_counts().index[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"681efdce6908de79594cf8c4bf3dcff29273c1ac"},"cell_type":"code","source":"\ndf_all.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3609ea5ecb4ea0a721136c3fc804b48877df40fd"},"cell_type":"markdown","source":"# Base model"},{"metadata":{"trusted":true,"_uuid":"6c8fa14fd59dfe63ee72f2b5f8f1f8768894170e"},"cell_type":"code","source":"%time\nglobal df_train_x_encoded,df_train_y_encoded, df_all_encoded, df_test_x_encoded\ndf_train_x_encoded = df_train_x.copy()\ndf_train_y_encoded = df_train_y.copy()\ndf_all_encoded = df_all.copy()\ndf_test_x_encoded = df_test_x.copy()\n\n## agg for last\n\n# date\n# the end of day of the dataset 10-15\nimport datetime\ndf_train_x_encoded[\"date\"]=(datetime.datetime(2018,6,30)-df_train_x_encoded.date).dt.days.astype(\"int64\")/365\ndf_train_y_encoded[\"date\"]=(datetime.datetime(2018,10,15)-df_train_y_encoded.date).dt.days.astype(\"int64\")/365\ndf_test_x_encoded[\"date\"]=(datetime.datetime(2018,10,15)-df_test_x_encoded.date).dt.days.astype(\"int64\")/365\n\n# last_cols =  [\"channelGrouping\", \"device_browser\", \n#             \"device_deviceCategory\", \"device_operatingSystem\", \n#             \"geoNetwork_city\", \"geoNetwork_continent\", \n#             \"geoNetwork_country\", \"geoNetwork_metro\",\n#             \"geoNetwork_networkDomain\", \"geoNetwork_region\", \n#             \"geoNetwork_subContinent\", \"trafficSource_adContent\", \n#             \"trafficSource_adwordsClickInfo.adNetworkType\", \n#             \"trafficSource_adwordsClickInfo.gclId\", \n#             \"trafficSource_adwordsClickInfo.slot\", \"trafficSource_campaign\",\n#             \"trafficSource_keyword\", \"trafficSource_medium\", \n#             \"trafficSource_referralPath\", \"trafficSource_source\",\n#             'trafficSource_adwordsClickInfo.isVideoAd',\n#             'trafficSource_isTrueDirect', 'device_isMobile'] + [\"fullVisitorId\",\"date\"]\n\ncat_cols=[\"channelGrouping\",\"device_browser\",\"device_deviceCategory\",\n              \"device_operatingSystem\",\"geoNetwork_city\",\"geoNetwork_continent\",\"geoNetwork_country\",\"trafficSource_medium\",\n              \"trafficSource_adwordsClickInfo.isVideoAd\",\"trafficSource_isTrueDirect\",\"device_isMobile\"]\nlast_cols = cat_cols + [\"fullVisitorId\",\"date\"]\n\ndf_train_x_agg_last = df_train_x_encoded[last_cols].groupby(\"fullVisitorId\",as_index=False).last().sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\ndf_test_x_agg_last = df_test_x_encoded[last_cols].groupby(\"fullVisitorId\",as_index=False).last().sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\ndf_train_y_agg_last = df_train_y_encoded[last_cols].groupby(\"fullVisitorId\",as_index=False).last().sort_values(\"fullVisitorId\").reset_index(drop=True).copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05daca7c341dbec83b8c8e5661138441e8ea3faf"},"cell_type":"code","source":"import category_encoders as ce\n\nec= ce.OrdinalEncoder(cols=cat_cols,handle_unknown='impute')\n\nec.fit(pd.concat([df_train_x_agg_last,df_train_y_agg_last,df_test_x_agg_last]))\ndf_train_x_agg_last=ec.transform(df_train_x_agg_last)\ndf_train_y_agg_last=ec.transform(df_train_y_agg_last)\ndf_test_x_agg_last=ec.transform(df_test_x_agg_last)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"719a0d3e0e5ed794ba6159e6d501e342ba6091b5"},"cell_type":"code","source":"df_train_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c803f389fc7a88ef267e7ed031e0b0649a910ce8","_kg_hide-output":false},"cell_type":"code","source":"%%time\n\nnum_cols = [item for item in df_train.columns if \"totals\" in item]\nsum_cols = num_cols + [\"fullVisitorId\"]\n\ndf_train_x_agg_sum = df_train_x_encoded[sum_cols].groupby(\"fullVisitorId\",as_index=False).sum().sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\ndf_test_x_agg_sum = df_test_x_encoded[sum_cols].groupby(\"fullVisitorId\",as_index=False).sum().sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\ndf_train_y_agg_sum = df_train_y_encoded[sum_cols].groupby(\"fullVisitorId\",as_index=False).sum().sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\n\n# totals_transactionRevenue\ndf_train_x_agg_sum['totals_transactionRevenue'] = np.log1p(df_train_x_agg_sum['totals_transactionRevenue'])\ndf_test_x_agg_sum['totals_transactionRevenue'] = np.log1p(df_test_x_agg_sum['totals_transactionRevenue'])\ndf_train_y_agg_sum['totals_transactionRevenue'] = np.log1p(df_train_y_agg_sum['totals_transactionRevenue'])\n\ndf_train_x_agg = pd.merge(df_train_x_agg_sum,df_train_x_agg_last, how='left',on=\"fullVisitorId\").sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\ndf_train_y_agg = pd.merge(df_train_y_agg_sum,df_train_y_agg_last, how='left',on=\"fullVisitorId\").sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\ndf_test_x_agg = pd.merge(df_test_x_agg_sum,df_test_x_agg_last, how='left',on=\"fullVisitorId\").sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\n\ndf_train_y_agg=pd.merge(id_train.to_frame(), df_train_y_agg, how='left',on=\"fullVisitorId\").sort_values(\"fullVisitorId\").reset_index(drop=True).copy()\ndf_train_y_agg[\"totals_transactionRevenue\"]=df_train_y_agg[\"totals_transactionRevenue\"].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3c9418131bd6d103900a3ef3d8c68c9abc0d3a2"},"cell_type":"code","source":"print({\"all\":df_all.shape,\n       \"train_x\":df_train_x_agg.shape,\n       \"train_y\":df_train_y_agg.shape,\n       \"test_x\":df_test_x_agg.shape,\n       \"id_train\":id_train.shape,\n       \"id_test\":id_test.shape})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57e6dc763bb83dca1206d3c4f60b461fbaae3237"},"cell_type":"code","source":"df_train_x_agg[df_train_x_agg.totals_transactionRevenue>0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fbd47b9e47a9d96f3d0cee73a5f84bf8222334f"},"cell_type":"code","source":"# data setting\n\ntrain_x = df_train_x_agg.drop([\"fullVisitorId\"],axis=1).values#[0:10000]\ntrain_y = df_train_y_agg.totals_transactionRevenue.values#[0:10000]\n\ntest_x_id = df_test_x_agg.fullVisitorId.values\ntest_x = df_test_x_agg.drop([\"fullVisitorId\"],axis=1).values\n\n# Grid Search\ngrid_params={'max_depth': [4,5,6],\n        'subsample': [0.95],\n        'colsample_bytree': [1.0]\n}\n\nxgb_model = xgb.XGBClassifier(early_stopping_rounds=50,eval_metric=\"rmse\",verbose=3,n_jobs=-1)\n\ngrid_search = GridSearchCV(xgb_model,\n                  grid_params,\n                  cv=3,\n                  scoring=\"neg_mean_squared_error\",\n                  n_jobs=1,\n                  verbose=3)\ngrid_search.fit(train_x,train_y)\npredict = grid_search.predict(test_x)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5dc30a7c4a402a664bcade5046b382ec160a9268"},"cell_type":"markdown","source":"# Submit"},{"metadata":{"trusted":true,"_uuid":"a5c973b9c7ef1d533fcb4c807b7d39a123dfd876"},"cell_type":"code","source":"predict_df=pd.DataFrame()\npredict_df[\"fullVisitorId\"]=test_x_id\npredict_df[\"PredictedLogRevenue\"]=predict\npredict_df.to_csv(\"xgboost_cv.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}