{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport sklearn\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_squared_error\nfrom math import sqrt\nimport lightgbm as lgb\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aaf0befec114f1090ae0cb609ec4425039e77de0"},"cell_type":"code","source":"%%time\ntrain_df = load_df()\ntest_df = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a24ffade0847f86c8b72a3093a93c3bc9fdb32d"},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2948c5c36b6ebc40c67ef667b7b399333f2a28e0"},"cell_type":"code","source":"train_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d392221b65433bfff68c60f421c05cdc757aaec4"},"cell_type":"code","source":"#target variable\n#Since we are predicting the natural log of sum of all transactions of the user\n#let us sum up the transaction revenue at user level and take a log and then do a scatter plot.\ntrain_df[\"totals.transactionRevenue\"] = train_df[\"totals.transactionRevenue\"].astype('float')\ngdf = train_df.groupby(\"fullVisitorId\")[\"totals.transactionRevenue\"].sum().reset_index()\n\nplt.figure(figsize=(8,6))\nplt.scatter(range(gdf.shape[0]), np.sort(np.log1p(gdf[\"totals.transactionRevenue\"].values)))\nplt.xlabel('index', fontsize=12) \nplt.ylabel('TransactionRevenue', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2991311c6c74f04f1d33061481b085fccecccf5"},"cell_type":"code","source":"nzt=pd.notnull(train_df['totals.transactionRevenue']).sum() #number of instances with non zero revenue\nnzt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c0062dd5a6a39ddca03d7f9823cfa0f39ca63e0"},"cell_type":"code","source":"nzu=(gdf[\"totals.transactionRevenue\"]>0).sum() #number of unique customers with non zero revenue\nnzu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff45c41d7fd52439403ae6e1332604f9d41b895a"},"cell_type":"code","source":"train_df.fullVisitorId.nunique() #unique visiters in the train set\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bb06a101836107bcdb74a3db2c79a88b94e76fe"},"cell_type":"code","source":"test_df.fullVisitorId.nunique() #unique visiters in the train set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb9e3253815cf432c6b0efba444f8018a2877878"},"cell_type":"code","source":"len(set(train_df.fullVisitorId.unique()).intersection(set(test_df.fullVisitorId.unique()))) #unique visitors between train and test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff26d857f0cccfc2eadd278963be86fc70621eaf"},"cell_type":"code","source":"const_cols = [c for c in train_df.columns if train_df[c].nunique(dropna=False)==1 ] \nconst_cols #columns with constant values which can be dropped","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47de280a8c4a860ff482d9d6c2ca46e09aa512b0"},"cell_type":"code","source":"(set(train_df.columns).difference(set(test_df.columns))) #variable which are not common in both test and train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cedcc6df7c59ef658c824b93a635c78ddf3bb4bb"},"cell_type":"code","source":"cols_to_drop = const_cols + ['sessionId'] #drop constant columns\n\ntrain_df = train_df.drop(cols_to_drop + [\"trafficSource.campaignCode\"], axis=1)\ntest_df = test_df.drop(cols_to_drop, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c98dd649ed9f51b97bcc948549cf50db04cbb517"},"cell_type":"code","source":"train_df[\"totals.transactionRevenue\"].fillna(0, inplace=True) #impute 0 in Na's place\ntrain_y = train_df[\"totals.transactionRevenue\"].values\ntrain_id = train_df[\"fullVisitorId\"].values\ntest_id = test_df[\"fullVisitorId\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd9d1e654bbab4dbc0715d57ab68763c4516e1f7"},"cell_type":"code","source":"#Label encoder\ncat_cols = [\"channelGrouping\", \"device.browser\", \n            \"device.deviceCategory\", \"device.operatingSystem\", \n            \"geoNetwork.city\", \"geoNetwork.continent\", \n            \"geoNetwork.country\", \"geoNetwork.metro\",\n            \"geoNetwork.networkDomain\", \"geoNetwork.region\", \n            \"geoNetwork.subContinent\", \"trafficSource.adContent\", \n            \"trafficSource.adwordsClickInfo.adNetworkType\", \n            \"trafficSource.adwordsClickInfo.gclId\", \n            \"trafficSource.adwordsClickInfo.page\", \n            \"trafficSource.adwordsClickInfo.slot\", \"trafficSource.campaign\",\n            \"trafficSource.keyword\", \"trafficSource.medium\", \n            \"trafficSource.referralPath\", \"trafficSource.source\",\n            'trafficSource.adwordsClickInfo.isVideoAd', 'trafficSource.isTrueDirect']\nfor col in cat_cols:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60f6390cb5d0fc952ba56a0d0f126306c1a06a13"},"cell_type":"code","source":"#Convert numerical columns into float\nnum_cols = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", \"visitStartTime\", 'totals.bounces',  'totals.newVisits']    \nfor col in num_cols:\n    train_df[col] = train_df[col].astype(float)\n    test_df[col] = test_df[col].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc0ffc517dd5be5d105786bd0d3731cdf86cf14f"},"cell_type":"code","source":"train_df['date'].","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5167593fc0db5257c2e085ef979656c9af5dc615"},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e61b0d7f51242569ee2cd0e5dbfb1720719059e8"},"cell_type":"code","source":"import datetime\n# Split the train dataset into development and valid based on time \ndev_df = train_df.iloc[0:700000,:]\nval_df = train_df.iloc[700001:,:]\ndev_y = np.log1p(dev_df[\"totals.transactionRevenue\"].values)\nval_y = np.log1p(val_df[\"totals.transactionRevenue\"].values)\n\ndev_X = dev_df[cat_cols + num_cols] \nval_X = val_df[cat_cols + num_cols] \ntest_X = test_df[cat_cols + num_cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f724e8002f617c29c6eb6819bf10c55a49af10da"},"cell_type":"markdown","source":"def run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\", \n        \"num_leaves\" : 30,\n        \"min_child_samples\" : 100,\n        \"learning_rate\" : 0.1,\n        \"bagging_fraction\" : 0.7,\n        \"feature_fraction\" : 0.5,\n        \"bagging_frequency\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    pred_eval=model.predict(val_X,num_iteration=model.best_iteration)\n    pred_eval[pred_eval<0] = 0\n    rms = sqrt(mean_squared_error(val_y,pred_eval))\n    print(rms)\n    return pred_test_y, model\n"},{"metadata":{"_uuid":"7c953809fca36ded0b6e4e91b60116b4c56aef02"},"cell_type":"markdown","source":"pred_test, model,rms = run_lgb(dev_X, dev_y, val_X, val_y, test_X)"},{"metadata":{"trusted":true,"_uuid":"5f17df803c457202540ca9786684c7e2d5f1d635"},"cell_type":"code","source":"def run_catbst(train_X, train_y, val_X, val_y, test_X):\n    eval_set=[(val_X,val_y)]\n    clf=cb.CatBoostRegressor(depth=10, l2_leaf_reg= 1,\n                            learning_rate= 0.15 ,eval_metric=\"RMSE\",iterations=1000, early_stopping_rounds=100)\n    model=clf.fit(train_X,train_y,eval_set=eval_set)\n    pred_test_y= model.predict(test_X)\n    pred_eval=model.predict(val_X)\n    pred_eval[pred_eval<0] = 0\n    rms = sqrt(mean_squared_error(val_y,pred_eval))\n    return pred_test_y, model ,rms\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c21779f1dcd05b00dc7e72de762d2a8805ada6f1"},"cell_type":"code","source":"import catboost as cb\npred_test, model,rms = run_catbst(dev_X, dev_y, val_X, val_y, test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a44ebd95f80de15b849b7089b2a6d1a9782e3b9"},"cell_type":"code","source":"rms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca8b0e7c307d508bde34904fd059860c4a2d30e6"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"fullVisitorId\":test_id})\npred_test[pred_test<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(pred_test)\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"test1_cat.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}