{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a8ea4e5012e4d9e4cda2147238d6c3396c8afe2"},"cell_type":"markdown","source":"Importing Libraries"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\nimport gc\nimport sys\n\nfrom pandas.io.json import json_normalize\nfrom datetime import datetime\nfrom sklearn import preprocessing\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b952bbbb41790230dcf2cb238b713559927447f9"},"cell_type":"code","source":"def load_df(csv_path, nr=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    #converters = cv\n    #dtype = dt\n    #nrows = nr\n    #column_as_df = cad\n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'},\n                     nrows=nr)\n    \n    for column in JSON_COLUMNS:\n        cad = json_normalize(df[column])\n        cad.columns = [f\"{column}.{subcolumn}\" for subcolumn in cad.columns]\n        df = df.drop(column, axis=1).merge(cad, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5141b8f351e80d241b5ce56209839c2480a5da9b"},"cell_type":"code","source":"%%time\ntrain = load_df('../input/train.csv')\ntest = load_df('../input/test.csv')\n\nprint('train date:', min(train['date']), 'to', max(train['date']))\nprint('test date:', min(test['date']), 'to', max(test['date']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55bfb5651267ea08811ae0ce298667d83396ab89"},"cell_type":"code","source":"# only train feature\nfor c in train.columns.values:\n    if c not in test.columns.values: print(c)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45ecc5cc65f9f535f7aa600a9f8cfa09c33ec93c"},"cell_type":"code","source":"# totals, the sub-column transactionRevenue contains the revenue information we are trying to predict\n#train_rev = train_revenue\ntrain_revenue = train[~train['totals.transactionRevenue'].isnull()].copy()\nprint(len(train_revenue))\ntrain_revenue.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e159e7dce0c11cd9dadaa0521f49e902eaf12b6"},"cell_type":"code","source":"train['totals.transactionRevenue'].fillna(0, inplace=True)\ntrain['totals.transactionRevenue'] = np.log1p(train['totals.transactionRevenue'].astype(float))\nprint(train['totals.transactionRevenue'].describe())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8901d4f841cca3eea9132b9cb3cc49988cdc99c1"},"cell_type":"code","source":"ad = train.append(test, sort=False).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e48357be0054a031d1ccf6fe59f75976e180c34"},"cell_type":"code","source":"print(ad.info())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af6724dc0de152ea3a5585284fc6e8312e98f5b9"},"cell_type":"code","source":"null_cnt = train.isnull().sum().sort_values()\nprint(null_cnt[null_cnt > 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9944d3e6045eea83e1e55af8e83adfe5fe68c550"},"cell_type":"code","source":"ad['totals.pageviews'].fillna(1, inplace=True)\nad['totals.newVisits'].fillna(0, inplace=True)\nad['totals.bounces'].fillna(0, inplace=True)\nad['totals.pageviews'] = ad['totals.pageviews'].astype(int)\nad['totals.newVisits'] = ad['totals.newVisits'].astype(int)\nad['totals.bounces'] = ad['totals.bounces'].astype(int)\n\nad['trafficSource.isTrueDirect'].fillna(False, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"711f57057cc41e78899858c85ac56b91bd0684d8"},"cell_type":"code","source":"cc = [col for col in ad.columns if ad[col].nunique() == 1]\n\nprint('drop columns:', cc)\nad.drop(cc, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76907d78df17e2940d2a8a34e8a63023156c327a"},"cell_type":"code","source":"format_str = '%Y%m%d'\nad['formated_date'] = ad['date'].apply(lambda x: datetime.strptime(str(x), format_str))\nad['_month'] = ad['formated_date'].apply(lambda x:x.month)\nad['_quarterMonth'] = ad['formated_date'].apply(lambda x:x.day//8)\nad['_day'] = ad['formated_date'].apply(lambda x:x.day)\nad['_weekday'] = ad['formated_date'].apply(lambda x:x.weekday())\n\n#ad(['date','formated_date'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bbdd36dc6b2301f1f6fb872b9515ef309883d74"},"cell_type":"code","source":"print(ad['channelGrouping'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d82fcec5ef3cd226284605f5b5475c439408d005"},"cell_type":"code","source":"print('train all:', len(train))\nprint('train unique fullVisitorId:', train['fullVisitorId'].nunique())\nprint('train unique visitId:', train['visitId'].nunique())\nprint('-' * 30)\nprint('test all:', len(test))\nprint('test unique fullVisitorId:', test['fullVisitorId'].nunique())\nprint('test unique visitId:', test['visitId'].nunique())\n#print('common fullVisitorId:', len(pd.merge(train, test, how='inner', on='fullVisitorId'))) # 183434","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bba4dfdb582d6d8516e2d83773599bd3f1562b1"},"cell_type":"code","source":"print(ad['visitNumber'].value_counts()[:5])\nprint('-' * 30)\nprint(ad['totals.newVisits'].value_counts())\nprint('-' * 30)\nprint(ad['totals.bounces'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80d8ced7c1ca2a06d9b07a8e48ea29cdcc51c221"},"cell_type":"code","source":"ad['_visitStartHour'] = ad['visitStartTime'].apply(\n    lambda x: str(datetime.fromtimestamp(x).hour))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d51d770d5a1895e02e34d32afef475dd4ef6fa48"},"cell_type":"code","source":"print('train all sessionId:', len(train['sessionId']))\nprint('train unique sessionId:', train['sessionId'].nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dd9f5a219b5b9d5a106d2d5acffe2803cd005bb"},"cell_type":"code","source":"print('unique browser count:', train['device.browser'].nunique())\nprint('-' * 30)\nprint(ad['device.browser'].value_counts()[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa48ae604bb236efe58209ebc6b195a4adaa9907"},"cell_type":"code","source":"pd.crosstab(ad['device.deviceCategory'], ad['device.isMobile'], margins=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1716b90dac8faf91148a7f85896fea652d1f25a"},"cell_type":"code","source":"print('unique operatingSystem count:', train['device.operatingSystem'].nunique())\nprint('-' * 30)\nprint(ad['device.operatingSystem'].value_counts()[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da3145fe60b906a1edb586680fb44d6c6a691c83"},"cell_type":"code","source":"print(ad['geoNetwork.city'].value_counts()[:10])\nprint('-' * 30)\nprint(ad['geoNetwork.region'].value_counts()[:10])\nprint('-' * 30)\nprint(ad['geoNetwork.subContinent'].value_counts()[:10])\nprint('-' * 30)\nprint(ad['geoNetwork.continent'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d8a124d88a307e7958fd05afa420506d741b3ea"},"cell_type":"code","source":"print(ad['geoNetwork.metro'].value_counts()[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff142dda13bbffcb2d4ed81cf3aba51f87df96dc"},"cell_type":"code","source":"print(ad['geoNetwork.networkDomain'].value_counts()[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8d6f89b049f46b872b532408d636e00396d6966"},"cell_type":"code","source":"print(ad['totals.hits'].value_counts()[:10])\n\nad['totals.hits'] = ad['totals.hits'].astype(int)\nad['_meanHitsPerDay'] = ad.groupby(['_day'])['totals.hits'].transform('mean')\nad['_meanHitsPerWeekday'] = ad.groupby(['_weekday'])['totals.hits'].transform('mean')\nad['_meanHitsPerMonth'] = ad.groupby(['_month'])['totals.hits'].transform('mean')\nad['_sumHitsPerDay'] = ad.groupby(['_day'])['totals.hits'].transform('sum')\nad['_sumHitsPerWeekday'] = ad.groupby(['_weekday'])['totals.hits'].transform('sum')\nad['_sumHitsPerMonth'] = ad.groupby(['_month'])['totals.hits'].transform('sum')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56960afd509344ec7d7101d8bca72873f1e05bda"},"cell_type":"code","source":"print(ad['totals.pageviews'].value_counts()[:10])\nad['totals.pageviews'] = ad['totals.pageviews'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3526573f88dc56b61457328fb8476081c0786a67"},"cell_type":"code","source":"print(ad['trafficSource.adContent'].value_counts()[:10])\nprint('-' * 30)\nprint(train_revenue['trafficSource.adContent'].value_counts())\n\nad['_adContentGMC'] = (ad['trafficSource.adContent'] == 'Google Merchandise Collection').astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79d9f733932c39ee019cf2d5c90f20db012c1fd7"},"cell_type":"code","source":"print(ad['trafficSource.campaign'].value_counts()[:10])\nad['_withCampaign'] = (ad['trafficSource.campaign'] != '(not set)').astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01ff567b9b3e46e6244956d9fde17150b8f782ea"},"cell_type":"code","source":"print(ad['trafficSource.isTrueDirect'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3b076be6f2582591fba5cea657de305cf09cf18"},"cell_type":"code","source":"print(ad['trafficSource.keyword'].value_counts()[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc3da4b8ef71d72d5c57de0271fa94782c8c619a"},"cell_type":"code","source":"print(ad['trafficSource.medium'].value_counts())\nprint('-' * 30)\nprint(train_revenue['trafficSource.medium'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a993f82a80edf97dd566c1e16fc6343f89683efe"},"cell_type":"code","source":"print(ad['trafficSource.referralPath'].value_counts()[:10])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c72caaf4b9bc9077283e8707419045228a92df5b","scrolled":true},"cell_type":"code","source":"print(ad['trafficSource.source'].value_counts()[:10])\nad['_sourceGpmall'] = (ad['trafficSource.source'] == 'mall.googleplex.com').astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"209684163867b5f6b002e40ef45c4fb3ede6097a"},"cell_type":"code","source":"train_revenue = train_revenue.sort_values(['visitStartTime']).reset_index()\ntrain_revenue['_buyCount'] = train_revenue.groupby('fullVisitorId').cumcount() + 1\nad = pd.merge(ad, train_revenue[['_buyCount','fullVisitorId','visitId']], \n                    on=['fullVisitorId','visitId'], how='left')\nfor fvId in train_revenue['fullVisitorId'].unique():\n    visitor_data = ad[ad['fullVisitorId'] == fvId].sort_values(['visitStartTime'])['_buyCount'].reset_index()\n    ad.loc[ad['fullVisitorId'] == fvId, '_buyCount'] = visitor_data['_buyCount'].fillna(method='ffill').values\nad['_buyCount'].fillna(0, inplace=True)\nad['_buyRate'] = ad['_buyCount'] / ad['visitNumber']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc28746f400cd96516a35a15d98efe7aaa12fe59"},"cell_type":"code","source":"null_cnt = ad.isnull().sum().sort_values()\nprint(null_cnt[null_cnt > 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42733934adef6148cc3ab0aaa69fa6d544d128c1"},"cell_type":"code","source":"ad.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b32688b5833503f15043d8902063d9e3f299ee7a"},"cell_type":"code","source":"c = ['fullVisitorId',\n     'visitNumber',\n     'device.deviceCategory',\n     'geoNetwork.subContinent',\n     'totals.transactionRevenue',\n     'totals.newVisits',\n     'totals.hits',\n     'totals.pageviews',\n     '_month',\n     '_quarterMonth',\n     '_weekday',\n     '_visitStartHour',\n     '_adContentGMC',\n     '_withCampaign',\n     '_sourceGpmall',\n     '_buyRate']\nad = ad[c]\n\nfor i, t in ad.loc[:, ad.columns != 'fullVisitorId'].dtypes.iteritems():\n    if t == object:\n        ad = pd.concat([ad, pd.get_dummies(ad[i].astype(str), prefix=i)], axis=1)\n        ad.drop(i, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc59370233a27f5bee9b9046ef10818c864679ca"},"cell_type":"code","source":"ad.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a87db9fd478ee1818c045262827ef1a3b1ca360f"},"cell_type":"code","source":"train = ad[ad['totals.transactionRevenue'].notnull()]\ntest = ad[ad['totals.transactionRevenue'].isnull()].drop(['totals.transactionRevenue'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff7b9f649a263c253df2f7905e8acce7e20c6dd1"},"cell_type":"code","source":"train_id = train['fullVisitorId']\ntest_id = test['fullVisitorId']\n\nY_train_reg = train.pop('totals.transactionRevenue')\nY_train_cls = (Y_train_reg.fillna(0) > 0).astype(np.uint8)\n\nX_train = train.drop(['fullVisitorId'], axis=1)\nX_test  = test.drop(['fullVisitorId'], axis=1)\n\nprint(X_train.shape, X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8907571396e3fd981042bedbc4f5c2dc6f3f932b"},"cell_type":"code","source":"import sys\nimport gc\n\ndel ad, train, test, train_revenue\ngc.collect()\n\nprint(pd.DataFrame([[val for val in dir()], [sys.getsizeof(eval(val)) for val in dir()]],\n                   index=['name','size']).T.sort_values('size', ascending=False).reset_index(drop=True)[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3119f1362108f079c06c6b5c18ce0d75c0f0197a"},"cell_type":"code","source":"from sklearn import ensemble, metrics\nfrom sklearn.model_selection import cross_val_score, cross_val_predict\nfrom sklearn.model_selection import GridSearchCV, KFold, GroupKFold\nfrom sklearn.metrics import mean_squared_error, classification_report, roc_auc_score, f1_score, log_loss\n\nimport xgboost as xgb\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4392d46251251b2db3e3c8d95f783a3837e32fd7"},"cell_type":"code","source":"if '_revenueProba' in X_train.columns : del X_train['_revenueProba']\nif '_revenueProba' in X_test.columns : del X_test['_revenueProba']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85b54dc3170f8d54049e8c37039a30a8284d4236"},"cell_type":"code","source":"%%time\nreg = ensemble.GradientBoostingRegressor(n_estimators=500, learning_rate=0.05, max_depth=3, verbose=1, random_state=42)\nreg.fit(X_train, Y_train_cls)\npred_reg = reg.predict(X_test)\n\nprint(len(pred_reg), len(pred_reg[pred_reg > 0.1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"f145721bb43f3303e0c6f607961ad46990bc804d"},"cell_type":"code","source":"reg = ensemble.GradientBoostingRegressor(n_estimators=1000, learning_rate=0.5, max_depth=3, verbose=1, random_state=42)\nreg.fit(X_train[Y_train_reg > 0], Y_train_reg[Y_train_reg > 0])\n\npred = np.zeros(len(pred_reg))\nfor i in np.arange(len(pred_reg)):\n        pred[i] = reg.predict([X_test.iloc[i]])[0] * pred_reg[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c8982bcf6eaacab73849dc981821f4c31f700c1"},"cell_type":"code","source":"#submission = sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"031c60f95ccde565de0828d0a7d093c3b29d78b5"},"cell_type":"code","source":"sub = pd.DataFrame({'fullVisitorId':test_id, 'PredictedLogRevenue':pred})\nsub[\"PredictedLogRevenue\"] = sub[\"PredictedLogRevenue\"].apply(lambda x : 0.0 if x < 0 else x)\nsub[\"PredictedLogRevenue\"] = sub[\"PredictedLogRevenue\"].fillna(0.0)\nsub_sum = sub[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum().reset_index()\nsub_sum.to_csv(\"submission.csv\", index=False)\nsub_sum[sub_sum['PredictedLogRevenue'] > 0.0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8deb594ddd18ec1049101c3a92db07a42acb889"},"cell_type":"code","source":"sub_sum['PredictedLogRevenue'].describe()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}