{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom datetime import datetime\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nfrom os.path import join as pjoin\n\ndata_root = '../input/make-data-ready'\nprint(os.listdir(data_root))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fed79ee0e6bd26d348d02b8c7b59886c3afe22c2"},"cell_type":"markdown","source":"# Import and load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.ensemble import RandomForestRegressor as RFF\nimport xgboost as xgb\n\nfrom pprint import pprint\nimport math\n\nfrom scipy.stats import kurtosis, skew\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42d57caae8977328d06ce1191632cd93f93cceff"},"cell_type":"code","source":"\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport shap\nplt.rcParams['figure.figsize'] = (12,6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3cc9e31f9c51f1fd66593d2fd0cb3963e880f12"},"cell_type":"code","source":"def load_data(data='train',n=2):\n    df = pd.DataFrame()\n    for i in range(n) :\n        if data=='train':\n            if i > 8 :\n                break\n            dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))\n        elif data=='test':\n            if i > 2 :\n                break\n            dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))\n        df = pd.concat([df,dfpart])\n        del dfpart\n    return df\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"711b2ff1d23636474cbf8cbd63ed852d4853499d","scrolled":false},"cell_type":"code","source":"df_train = load_data(n=9)\ndf_test = load_data('test',n=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"660a0bb0b6ac94d469e290bddbf0b32ca55bf0ff"},"cell_type":"code","source":"print(f'# of columns has na value: {(df_test.isnull().sum().sort_values(ascending=False) > 0).sum()}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3609ea5ecb4ea0a721136c3fc804b48877df40fd"},"cell_type":"markdown","source":"# Base model"},{"metadata":{"trusted":true,"_uuid":"6c8fa14fd59dfe63ee72f2b5f8f1f8768894170e"},"cell_type":"code","source":"def rmse(y_true, y_pred):\n    return round(np.sqrt(mean_squared_error(y_true, y_pred)), 5)\n\ndef split_data(df=df_train,rate=.8):\n    # sort the date first\n    df = df.sort_values('date').copy()\n    \n    df.drop(['fullVisitorId','visitId','visitStartTime'],axis=1,inplace=True)\n    df['Revenue'] = np.log1p(df['Revenue'])\n    \n    global X_train,X_valid,y_train,y_valid\n    \n    n_train = int(len(df)*rate)\n    X_train = df.drop(['Revenue','date'],axis=1).iloc[:n_train]\n    X_valid = df.drop(['Revenue','date'],axis=1).iloc[n_train:]\n    \n    y_train = df['Revenue'].iloc[:n_train]\n    y_valid = df['Revenue'].iloc[n_train:]\n    \n    print(X_train.shape,X_valid.shape)\n    \n    \n\ndef encode_data(verbose=False):\n    global df_train_encoded,df_test_encoded\n    df_train_encoded = df_train.copy()\n    df_test_encoded = df_test.copy()\n    for col in df_train.columns:\n        if df_train_encoded[col].dtype == 'object' and col not in ['fullVisitorId','visitId','visitStartTime','date']:\n            if verbose:\n                print(col)\n            lb = LabelEncoder()\n            lb.fit( list(df_train_encoded[col].unique()) + list(df_test_encoded[col].unique()))\n            df_train_encoded[col] = lb.transform(df_train_encoded[col])\n            df_test_encoded[col] = lb.transform(df_test_encoded[col])\n        \ndef run_xgb():\n    params = {\n        'objective':'reg:linear',\n        'eval_metric':'rmse',\n        'learning_rate':.01,\n        'eta': 0.1, # Step size shrinkage used in update to prevents overfitting\n        'max_depth': 10,\n        'subsample': 0.6, # sample of rows\n        'colsample_bytree': 0.6, # sample of features\n#         'alpha':0.001, \n        'lambda':1, # l2 regu\n        'random_state': 42,\n        'silent':True\n        \n    }\n    xgb_train_data = xgb.DMatrix(X_train, y_train)\n    xgb_val_data = xgb.DMatrix(X_valid, y_valid)\n    \n    model = xgb.train(params, xgb_train_data,\n          num_boost_round=1000,\n          evals= [(xgb_train_data, 'train'), (xgb_val_data, 'valid')],\n          early_stopping_rounds=10, \n          verbose_eval=20\n         )\n    return model\n\ndef submit():\n    test_matrix = xgb.DMatrix(X_test)\n    y_pred = clf.predict(test_matrix,ntree_limit=clf.best_ntree_limit)\n    df_test['PredictedLogRevenue'] = y_pred\n    submit = df_test[['PredictedLogRevenue','fullVisitorId']].groupby('fullVisitorId').PredictedLogRevenue.sum().reset_index()\n    submit.to_csv('submit.csv',index=False)\n    \n    test(y_pred)\n\n    \ndef test(predict):\n    y_test = df_test['totals_transactionRevenue']\n    print(rmse(y_test,predict))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c803f389fc7a88ef267e7ed031e0b0649a910ce8"},"cell_type":"code","source":"def prepare_data(df_train,df_test,\n                 del_col=['fullVisitorId','visitId','visitStartTime','date']):\n    df_train = df_train.sort_values('date').copy()\n    \n    df_train = df_train.drop(del_col,axis=1).copy()\n    df_test = df_test.drop(del_col,axis=1).copy()\n    \n    # totals_transactionRevenue\n    df_train['totals_transactionRevenue'] = np.log1p(df_train['totals_transactionRevenue'])\n    df_test['totals_transactionRevenue'] = np.log1p(df_test['totals_transactionRevenue'])\n    \n    global X_train,X_valid,y_train,y_valid,X_test,y_test\n    # 80/20 : train/valid\n    n_train = int(len(df_train)*.8)\n    \n    # split\n    X_train = df_train.drop(['totals_transactionRevenue'],axis=1).iloc[:n_train]\n    X_valid = df_train.drop(['totals_transactionRevenue'],axis=1).iloc[n_train:]\n    \n    y_train = df_train['totals_transactionRevenue'].iloc[:n_train]\n    y_valid = df_train['totals_transactionRevenue'].iloc[n_train:]\n    \n    X_test = df_test.drop(['totals_transactionRevenue'],axis=1)\n    y_test = df_test['totals_transactionRevenue']\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"16605dd219a0e49bb3ab9ddd0629c34ea7f948e1"},"cell_type":"code","source":"encode_data(verbose=True)\nprepare_data(df_train_encoded,df_test_encoded,del_col=['fullVisitorId','visitId',\n            'visitStartTime','date','totals_totalTransactionRevenue','totals_transactions'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"c26b1e30bf504d9c9cec95146656358ed844970e"},"cell_type":"code","source":"clf = run_xgb()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b3094980c0ec926539821a462d0ee2d65abe9ea"},"cell_type":"code","source":"# try to find a good validation set\n# Why our score so different with the leader board?\n# check with the target in dataset first","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"392938eb9284e3f800c7f1254891827b39c68d7d"},"cell_type":"markdown","source":"# Feature important"},{"metadata":{"_uuid":"0101c08148886885c07d41dfa152fb94fc8e604d"},"cell_type":"markdown","source":"\"gain\" is the average gain of splits which use the feature"},{"metadata":{"trusted":true,"_uuid":"63cb30a84b68bec37fe1ffa53d6c708b489731fd"},"cell_type":"code","source":"xgb.plot_importance(clf,importance_type='gain')\nplt.title('Gain Feature important')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"81d96c5a110b876e555d19e538fc888d0b051001"},"cell_type":"markdown","source":"\"cover\" is the average coverage of splits which use the feature where coverage is defined as the number of samples affected by the split"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"34b2a8c2186d69867ae376671ff291fce662fc99"},"cell_type":"code","source":"xgb.plot_importance(clf,importance_type='cover')\nplt.title('Cover Feature important')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22b8e759c46de5a8031ad3512f7d0b7e679b0a84"},"cell_type":"markdown","source":"\"weight\" is the number of times a feature appears in a tree"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"e32fac1da52c2a2e7644e1af7c9a196f573f59dc"},"cell_type":"code","source":"xgb.plot_importance(clf,importance_type='weight')\nplt.title('Weight Feature important')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5dc30a7c4a402a664bcade5046b382ec160a9268"},"cell_type":"markdown","source":"# Submit"},{"metadata":{"trusted":true,"_uuid":"a5c973b9c7ef1d533fcb4c807b7d39a123dfd876"},"cell_type":"code","source":"submit()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f587453cb66d50c15822707425190512ddab4d0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b4c795987399a376ff7f856b2a43d62e76ef57e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"171b4868f331037343e0930111a72edb4c29860a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e32583cc487663dd61799cb1960f872eff20bd2c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d48f9ea4532b68da11401a92d6dc020f33c791a7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24e84945849d90581707fd71c587ead4ba709583"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}