{"cells":[{"metadata":{"trusted":true,"_uuid":"a8d76d39915f70521e311086cef11095976c1a28"},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8da74e3eeefe45915ed578edbbd55b3f2d1b0e90","scrolled":true},"cell_type":"code","source":"json_cols = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\n# train = pd.read_csv(\"C:\\\\Users\\\\Urmil\\\\Desktop\\\\study\\\\Masters\\\\DSF\\\\HW3\\\\all\\\\train.csv\", converters={column: json.loads for column in json_cols}, dtype={'fullVisitorId': 'str'}, nrows = 200000)\n# train = pd.read_csv(\"/media/urmil/Windows8_OS/Users/Urmil/Desktop/study/Masters/DSF/HW3/all/train.csv\", converters={column: json.loads for column in json_cols}, dtype={'fullVisitorId': 'str'}, nrows = 100000)\ntrain = pd.read_csv('../input/ga-customer-revenue-prediction/train.csv', converters={column: json.loads for column in json_cols}, dtype={'fullVisitorId': 'str'})\nfor column in json_cols:\n        column_as_df = json_normalize(train[column])\n        column_as_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_df.columns]\n        train = train.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\ntrain['fullVisitorId'] = train['fullVisitorId'].astype('str')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f727e4b2bc14154fcb7f8323d242811e18b153a1"},"cell_type":"code","source":"# test = pd.read_csv(\"C:\\\\Users\\\\Urmil\\\\Desktop\\\\study\\\\Masters\\\\DSF\\\\HW3\\\\all\\\\test.csv\", converters={column: json.loads for column in json_cols}, dtype={'fullVisitorId': 'str'}, nrows = 200000)\n# test = pd.read_csv(\"/media/urmil/Windows8_OS/Users/Urmil/Desktop/study/Masters/DSF/HW3/all/test.csv\", converters={column: json.loads for column in json_cols}, dtype={'fullVisitorId': 'str'}, nrows=100000)\ntest = pd.read_csv(\"../input/ga-customer-revenue-prediction/test.csv\", converters={column: json.loads for column in json_cols}, dtype={'fullVisitorId': 'str'})\nfor column in json_cols:\n        column_as_df = json_normalize(test[column])\n        column_as_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_df.columns]\n        test = test.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\ntest['fullVisitorId'] = test['fullVisitorId'].astype('str')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bd3b78d439bb7d11a0bbd285c72e00f4db38bb7"},"cell_type":"code","source":"for col in train.columns:\n    if col not in test.columns:\n        print(col)\n# Removing the columns that has constant values     \ntrain = train.loc[:, (train != train.iloc[0]).any()] \ntrain = train.dropna(axis=1, how='all')\n\n# Removing the 'trafficSource_campaignCode' as it is not in the test data\ntrain = train.drop(axis = 1,columns=['trafficSource_campaignCode'])\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1727f0a9421f89bf848b8a6d448c92c14067b787"},"cell_type":"code","source":"test = test.loc[:, (test != test.iloc[0]).any()] \ntest = test.dropna(axis=1, how='all')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4927a0a4a2444678183996ec7cbde2518b194c9b"},"cell_type":"markdown","source":"Anomalies in the data:\n1. There are many columns that have constant value.\n2. 'trafficSource_campaignCode' is not in the train data.\n3. Almost all the columns has NaN values.\n\nCleaning of the data:\n1. Remove the columns with the constant value as they have no effect on the prediction.\n2. Remove 'trafficSource_campaignCode' from training data\n\nColumns with constant values:\n1.  socialEngagementType\n2.  device_browserSize\n3.  device.browserVersion\n4.  device.flashVersion\n5.  device.language\n6.  device.mobileDeviceBranding\n7.  device.mobileDeviceInfo\n8.  device.mobileDeviceMarketingName\n9.  device.mobileDeviceModel\n10. device.mobileInputSelector\n11. device.operatingSystemVersion\n12. device.screenColors\n13. device.screenResolution\n14. geoNetwork.cityId\n15. geoNetwork.latitude\n16. geoNetwork.longitude\n17. geoNetwork.networkLocation\n18. totals.visits\n19. trafficSource.adwordsClickInfo.criteriaParameters"},{"metadata":{"trusted":true,"_uuid":"37852bfb0f47fed18d8b789ef3daad6fe2002e68"},"cell_type":"code","source":"len_train = len(train)\nlen_test = len(test)\ndef missing_cnt(col):\n    nanlist = [\"not available in demo dataset\", \"(not provided)\", \"(not set)\", \"<NA>\", \"unknown.unknown\",  \"(none)\", \"NaN\"]\n    word_cnt = train[col].value_counts()\n    for nan in nanlist:\n        if nan in word_cnt:\n            return word_cnt[nan]\n    return 0\naa = []\nfor col in train.columns:\n    aa.append(missing_cnt(col)/len_train*100.0)\n# aa.sort()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f6580970488bd686e306451d8053c5d45ccedf7"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.rcParams.update({'font.size': 10})\nplt.figure(figsize=(10,10))\nplt.barh(train.columns, aa)\nplt.xlabel('Missing values in percentage')\nplt.title('Percentage missing values for each parameter')\n \nplt.show()\ntrain_extra = train\n# train = train.drop(columns = ['trafficSource_campaign'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22ae54363f836d6cab686babd75d97902510048e"},"cell_type":"code","source":"import datetime\n\nfor col in train.columns:\n    if col == 'date':\n        train['date'] = train['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\n        continue\n    if train[col].dtype == 'O':\n        if col in ['totals_bounces','totals_hits', 'totals_newVisits', 'totals_pageviews', 'totals_transactionRevenue']:\n            train[col] = train[col].astype('float')\n        else:\n            train[col] = train[col].astype('str')\n            \nfor col in test.columns:\n    if col == 'date':\n        test['date'] = test['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\n        continue\n    if test[col].dtype == 'O':\n        if col in ['totals_bounces','totals_hits', 'totals_newVisits', 'totals_pageviews']:\n            test[col] = test[col].astype('float')\n        else:\n            test[col] = test[col].astype('str')\n\ndef extractMonth(DateTime):\n    return DateTime.month\n\ndef extractYear(DateTime):\n    return DateTime.year\n\ntrain['month'] = train['date'].apply(extractMonth)\ntrain['year'] = train['date'].apply(extractYear)\n\ntest['month'] = test['date'].apply(extractMonth)\ntest['year'] = test['date'].apply(extractYear)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5561f00f469628ec9662f8657268845b2b7897e2"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n#finding correlations(dependence) between the variables\ncorr_matrix = train.corr()\nf, ax = plt.subplots(figsize=(16,10))\nax = sns.heatmap(corr_matrix, annot=True)\nax.set_title(\"Correlation between numerical features\")\nfigure = ax.get_figure()    \n# figure.savefig('heatmap.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2dcfe32dffe5720374999b021f51879b7e185f8"},"cell_type":"code","source":"cnt_srs_browser = train.groupby('device_browser')['totals_transactionRevenue'].agg(['size', 'count', 'mean', 'sum'])\ncnt_srs_browser.columns = ['count', 'positive revenue count', 'mean', 'sum']\ncnt_srs_browser = cnt_srs_browser.sort_values(by='count', ascending=False)\n\ncnt_srs_OS = train.groupby('device_operatingSystem')['totals_transactionRevenue'].agg(['size', 'count', 'mean', 'sum'])\ncnt_srs_OS.columns = ['count', 'positive revenue count', 'mean', 'sum']\ncnt_srs_OS = cnt_srs_OS.sort_values(by='count', ascending=False)\n\ncnt_srs_deviceCategory = train.groupby('device_deviceCategory')['totals_transactionRevenue'].agg(['size', 'count', 'mean', 'sum'])\ncnt_srs_deviceCategory.columns = ['count', 'positive revenue count', 'mean', 'sum']\ncnt_srs_deviceCategory = cnt_srs_deviceCategory.sort_values(by='count', ascending=False)\n\nplt.rcParams.update({'font.size': 12})\nf, ((ax1, ax2, ax3, ax4), (ax5, ax6, ax7, ax8), (ax9, ax10, ax11, ax12)) = plt.subplots(nrows=3, ncols=4, sharey='row',figsize=(18,21))\nax1.barh(cnt_srs_browser.index[:10], cnt_srs_browser['count'].head(10))\nax1.set_title('device_browser-Total count')\nax2.barh(cnt_srs_browser.index[:10], cnt_srs_browser['positive revenue count'].head(10))\nax2.set_title('device_browser-Non zero revenue count')\nax3.barh(cnt_srs_browser.index[:10], cnt_srs_browser['sum'].head(10))\nax3.set_title('device_browser-Sum')\nax4.barh(cnt_srs_browser.index[:10], cnt_srs_browser['mean'].head(10))\nax4.set_title('device_browser-Mean')\n\nax5.barh(cnt_srs_OS.index[:10], cnt_srs_OS['count'].head(10))\nax5.set_title('OS-Total count')\nax6.barh(cnt_srs_OS.index[:10], cnt_srs_OS['positive revenue count'].head(10))\nax6.set_title('OS-Non zero revenue count')\nax7.barh(cnt_srs_OS.index[:10], cnt_srs_OS['sum'].head(10))\nax7.set_title('OS-Sum')\nax8.barh(cnt_srs_OS.index[:10], cnt_srs_OS['mean'].head(10))\nax8.set_title('OS-Mean')\n\nax9.barh(cnt_srs_deviceCategory.index[:10], cnt_srs_deviceCategory['count'].head(10))\nax9.set_title('deviceCategory-Total count')\nax10.barh(cnt_srs_deviceCategory.index[:10], cnt_srs_deviceCategory['positive revenue count'].head(10))\nax10.set_title('deviceCategory-Non zero revenue count')\nax11.barh(cnt_srs_deviceCategory.index[:10], cnt_srs_deviceCategory['sum'].head())\nax11.set_title('deviceCategory-Sum')\nax12.barh(cnt_srs_deviceCategory.index[:10], cnt_srs_deviceCategory['mean'].head())\nax12.set_title('deviceCategory-Mean')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3413a7d309542441577c53a45630fb1d0f94b8bb"},"cell_type":"code","source":"cnt_srs_continent = train.groupby('geoNetwork_continent')['totals_transactionRevenue'].agg(['size', 'count', 'mean', 'sum'])\ncnt_srs_continent.columns = ['count', 'positive revenue count', 'mean', 'sum']\ncnt_srs_continent = cnt_srs_continent.sort_values(by='count', ascending=False)\n\ncnt_srs_subContinent = train.groupby('geoNetwork_subContinent')['totals_transactionRevenue'].agg(['size', 'count', 'mean', 'sum'])\ncnt_srs_subContinent.columns = ['count', 'positive revenue count', 'mean', 'sum']\ncnt_srs_subContinent = cnt_srs_subContinent.sort_values(by='count', ascending=False)\n\ncnt_srs_country = train.groupby('geoNetwork_country')['totals_transactionRevenue'].agg(['size', 'count', 'mean', 'sum'])\ncnt_srs_country.columns = ['count', 'positive revenue count', 'mean', 'sum']\ncnt_srs_country = cnt_srs_country.sort_values(by='count', ascending=False)\n\nplt.rcParams.update({'font.size': 12})\nf, ((ax1, ax2, ax3, ax4), (ax5, ax6, ax7, ax8), (ax9, ax10, ax11, ax12)) = plt.subplots(nrows=3, ncols=4, sharey='row',figsize=(18,21))\nax1.barh(cnt_srs_continent.index[:10], cnt_srs_continent['count'].head(10))\nax1.set_title('Continent-Total count')\nax2.barh(cnt_srs_continent.index[:10], cnt_srs_continent['positive revenue count'].head(10))\nax2.set_title('Continent-Non zero revenue count')\nax3.barh(cnt_srs_continent.index[:10], cnt_srs_continent['sum'].head(10))\nax3.set_title('Continent-Sum')\nax4.barh(cnt_srs_continent.index[:10], cnt_srs_continent['mean'].head(10))\nax4.set_title('Continent-Mean')\n\nax5.barh(cnt_srs_subContinent.index[:10], cnt_srs_subContinent['count'].head(10))\nax5.set_title('subContinent-Total count')\nax6.barh(cnt_srs_subContinent.index[:10], cnt_srs_subContinent['positive revenue count'].head(10))\nax6.set_title('subContinent-Non zero revenue count')\nax7.barh(cnt_srs_subContinent.index[:10], cnt_srs_subContinent['sum'].head(10))\nax7.set_title('subContinent-Sum')\nax8.barh(cnt_srs_subContinent.index[:10], cnt_srs_subContinent['mean'].head(10))\nax8.set_title('subContinent-Mean')\n\nax9.barh(cnt_srs_country.index[:10], cnt_srs_country['count'].head(10))\nax9.set_title('Country-Total count')\nax10.barh(cnt_srs_country.index[:10], cnt_srs_country['positive revenue count'].head(10))\nax10.set_title('Country-Non zero revenue count')\nax11.barh(cnt_srs_country.index[:10], cnt_srs_country['sum'].head(10))\nax11.set_title('Country-Sum')\nax12.barh(cnt_srs_country.index[:10], cnt_srs_country['mean'].head(10))\nax12.set_title('Country-Mean')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b498df317dde9840113d14663a21b975598dd69"},"cell_type":"code","source":"cnt_srs = train.groupby('trafficSource_source')['totals_transactionRevenue'].agg(['size', 'count', 'mean', 'sum'])\ncnt_srs.columns = ['count', 'positive revenue count', 'mean', 'sum']\ncnt_srs = cnt_srs.sort_values(by='count', ascending=False)\nplt.rcParams.update({'font.size': 12})\nf, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, sharey=True, figsize=(18,7))\nax1.barh(cnt_srs.index[:10], cnt_srs['count'].head(10))\nax1.set_title('Total count')\nax2.barh(cnt_srs.index[:10], cnt_srs['positive revenue count'].head(10))\nax2.set_title('Non zero revenue count')\nax3.barh(cnt_srs.index[:10], cnt_srs['sum'].head(10))\nax3.set_title('Sum')\nax4.barh(cnt_srs.index[:10], cnt_srs['mean'].head(10))\nax4.set_title('Mean')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebc782a199c5d4cc055b8e0928d8ba0c95c0a4d0"},"cell_type":"code","source":"cnt_srs = train.groupby('channelGrouping')['totals_transactionRevenue'].agg(['size', 'count', 'mean', 'sum'])\ncnt_srs.columns = ['count', 'positive revenue count', 'mean', 'sum']\ncnt_srs = cnt_srs.sort_values(by='count', ascending=False)\n\nplt.rcParams.update({'font.size': 12})\nf, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, sharey=True, figsize=(10,8))\nax1.barh(cnt_srs.index, cnt_srs['count'])\nax1.set_title('Total count')\nax2.barh(cnt_srs.index, cnt_srs['positive revenue count'])\nax2.set_title('Non zero revenue count')\nax3.barh(cnt_srs.index, cnt_srs['sum'])\nax3.set_title('Sum')\nax4.barh(cnt_srs.index, cnt_srs['mean'])\nax4.set_title('Mean')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0971c52dae05acab431ece39b142bb4b47aea370"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10,6))\nax = plt.scatter(x=train['visitNumber'], y=train['totals_transactionRevenue'], s=4)\nplt.xlabel('visitNumber', fontsize=15)\nplt.ylabel('Total Transaction Revenue', fontsize=15)\nplt.title('Scatter plot visit number vs. total transaction revenue', fontsize=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd95b29319246c59bf1d44fd541c0dd8e7bb72b5"},"cell_type":"code","source":"cnt_srs_date = train.groupby('date')['totals_transactionRevenue'].agg(['size', 'count'])\ncnt_srs_date.columns = [\"count\", \"count of non-zero revenue\"]\ncnt_srs_date = cnt_srs_date.sort_index()\nplt.rcParams.update({'font.size': 12})\n# f, (ax1, ax2) = plt.subplots(2, 1, figsize=(10,8))\n\ncnt_srs_date['count'].plot(figsize = (15,5), fontsize=15, title='Date-Total count')\nplt.xlabel('Date', fontsize=15)\nplt.ylabel('Frequency', fontsize=15)\nplt.show()\ncnt_srs_date['count of non-zero revenue'].plot(figsize = (15,5), fontsize=15, title='Date-Non -zero revenue count')\nplt.xlabel('Date', fontsize=15)\nplt.ylabel('Frequency', fontsize=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"547e9ca2d8140e3e3c050d7f55d55156b315d9ed"},"cell_type":"code","source":"train1 = train\ntest1 = test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fb028933b7133a500a11477784b98a082f28fcc"},"cell_type":"code","source":"# # PPP\n# external = pd.read_csv('../input/External Data/Final_data.csv')\n\n# PPP_dict = {}\n# for i in range(len(external)):\n#     PPP_dict[external.iloc[i]['Country']] = int(external.iloc[i]['PPP'])\n\n# def countryPPP(country):\n#     if country in PPP_dict:\n#         return PPP_dict[country]\n#     return 0 \n\n# train['PPP'] = train['geoNetwork_country'].apply(countryPPP)\n# test['PPP'] = test['geoNetwork_country'].apply(countryPPP)\n\n# # cnt =0\n# # for i in range(len(train)):\n# #     if train.iloc[i]['PPP'] == 0:\n# #         cnt+=1\n# #         print(train.iloc[i]['geoNetwork_country'])\n# # print(cnt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc48b286a98bdaf6cb306f784185ce7874fe0404"},"cell_type":"code","source":"from sklearn import linear_model\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.cross_validation import train_test_split\nfrom sklearn.model_selection import ShuffleSplit\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn import preprocessing\n\ntrain['totals_transactionRevenue'].fillna(0, inplace=True)\ntrain['totals_transactionRevenue'] = np.log1p(train['totals_transactionRevenue'])\n\ncol_list = ['channelGrouping', 'sessionId', 'device_browser', 'device_deviceCategory', 'device_isMobile', \n            'device_operatingSystem', 'geoNetwork_city', 'geoNetwork_continent', 'geoNetwork_country', \n            'geoNetwork_metro', 'geoNetwork_networkDomain', 'geoNetwork_region', 'geoNetwork_subContinent', \n            'trafficSource_adContent', 'trafficSource_adwordsClickInfo.adNetworkType', \n            'trafficSource_adwordsClickInfo.gclId', 'trafficSource_adwordsClickInfo.isVideoAd', \n            'trafficSource_adwordsClickInfo.page', 'trafficSource_adwordsClickInfo.slot', 'trafficSource_campaign', \n            'trafficSource_isTrueDirect', 'trafficSource_keyword', 'trafficSource_medium', \n            'trafficSource_referralPath', 'trafficSource_source']\n\ntrain1 = train\ntest1 = test\nfor col in col_list:\n    print(col)\n    label = preprocessing.LabelEncoder()\n    label.fit(list(train[col]) + list(test[col]))\n    train[col] = label.transform(list(train[col]))\n    test[col] = label.transform(list(test[col]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c001494b318d0345abfe265dd8cb3ca47dda5cf"},"cell_type":"code","source":"# import datetime\n\ndf_x = train\ndf_x = df_x.drop(columns = ['totals_transactionRevenue', 'date', 'sessionId', 'fullVisitorId', 'visitId', 'visitStartTime'])\ndf_y = train['totals_transactionRevenue']\n\n\ntest_x = test\ntest_x = test_x.drop(columns = ['date', 'sessionId', 'fullVisitorId', 'visitId', 'visitStartTime'])\n\n# #spliting training, testing data\nx_train, x_test, y_train, y_test = train_test_split(df_x, df_y, test_size = 0.2, random_state = 3)\n\n# x_train = train[train['date']<=datetime.date(2017,5,31)]\n# x_test = train[train['date']>datetime.date(2017,5,31)]\n# y_train = np.log1p(x_train[\"totals_transactionRevenue\"].values)\n# y_test = np.log1p(x_test[\"totals_transactionRevenue\"].values)\n\n# x_train = x_train.drop(columns = ['totals_transactionRevenue', 'date', 'sessionId', 'fullVisitorId', 'visitId', 'visitStartTime'])\n# x_test = x_test.drop(columns = ['totals_transactionRevenue','date', 'sessionId', 'fullVisitorId', 'visitId', 'visitStartTime'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43ddd3cbf9e33317a7faa1c47ea60a69487f59c4"},"cell_type":"code","source":"# #XGBoost\n# # from sklearn import ensemble\n# import xgboost as xgb\n\n# submissionPath = '/home/urmil/Desktop/submission_xgb.csv'\n\n# #Creating XGB regressor\n# XGB = xgb.XGBRegressor(n_estimators=100, learning_rate=0.09, subsample=0.75, max_depth=6, n_jobs = 7)\n\n# #cross validation\n# # cross_val_generator = ShuffleSplit()\n# # print(cross_val_score(XGB, x_train, y_train, cv = cross_val_generator))\n\n# XGB.fit(x_train,y_train)\n\n# ans = XGB.predict(x_test)\n\n# #mean square error\n# mse = np.mean((ans-y_test)**2)\n# print('mean square error')\n# print(mse)\n# # root mean square error\n# rmse = np.sqrt(mse)\n# print('root mean square error')\n# print(rmse)\n\n# # # Make predictions using the testing set\n# pred_y = XGB.predict(test_x)\n\n# # # Generating output file\n# # pred_y = pd.Series(pred_y, name = 'fare_amount')\n# # submission = pd.DataFrame(data=test['key'])\n# # submission['fare_amount'] = pred_y\n# # print(submission.head())\n# # submission.to_csv(submissionPath, sep='\\t', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2890cb295f4ab15bf779df9e9343fec23c683700"},"cell_type":"code","source":"#LGBM\nimport lightgbm as lgbm\n\n# creating LGBM regressor\nLGBM = lgbm.LGBMRegressor(n_estimators=100, learning_rate=0.03, subsample=0.75, n_jobs = 7)\n\n# cross validation\n# cross_val_generator = ShuffleSplit()\n# print(cross_val_score(LGBM, x_train, y_train, cv = cross_val_generator))\nLGBM.fit(x_train,y_train)\n\nans = LGBM.predict(x_test)\n\n#mean square error\nmse = np.mean((ans-y_test)**2)\nprint('mean square error')\nprint(mse)\n\n# root means square error\nrmse = np.sqrt(mse)\nprint('root mean square error')\nprint(rmse)\norg_rmse = rmse\n\n# Make predictions using the testing set\npred_y = LGBM.predict(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73ecd521167e819be51a3a55a73f52e92102f0ec"},"cell_type":"code","source":"sub_df = pd.DataFrame()\nsub_df['fullVisitorId'] = test['fullVisitorId']\npred_y[pred_y<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(pred_y)\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv('submission_time.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2a1a20e9e4858c79880b6261181462848054697"},"cell_type":"code","source":"lgbm.plot_importance(LGBM, figsize=(12,15), height=0.7)\nplt.grid(False)\nplt.title(\"Importance of features based on LGBM\", fontsize=15)\nplt.xlabel('Feature Importance', fontsize=15)\nplt.ylabel('Features', fontsize=15)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d09a62e243634c70414ff937215153e74bf6e493"},"cell_type":"code","source":"# # Permutation test\n# import lightgbm as lgbm\n# import seaborn as sns\n\n# def LGBMFunction(train):\n#     df_x = train\n#     df_x = df_x.drop(columns = ['totals_transactionRevenue', 'date', 'sessionId', 'fullVisitorId', 'visitId'])\n#     df_y = train['totals_transactionRevenue']\n\n#     # Train test split\n#     x_train1, x_test1, y_train1, y_test1 = train_test_split(df_x, df_y, test_size = 0.2, random_state = 3)    \n    \n#     # creating LGBM regressor\n#     LGBM = lgbm.LGBMRegressor(n_estimators=100, learning_rate=0.03, subsample=0.75, n_jobs = 7)\n\n#     LGBM.fit(x_train1,y_train1)\n\n#     per_ans = LGBM.predict(x_test1)\n\n#     # mean square error\n#     per_mse = np.mean((per_ans-y_test1)**2)\n\n#     # root means square error\n#     per_rmse = np.sqrt(per_mse)\n    \n#     return per_rmse\n\n# train_ex = train\n# test_ex = test\n# result = []\n# col_list = ['channelGrouping', 'visitNumber', 'device_browser', 'device_deviceCategory', 'device_isMobile', \n#             'device_operatingSystem', 'geoNetwork_city', 'geoNetwork_continent', 'geoNetwork_country',\n#             'geoNetwork_metro', 'geoNetwork_networkDomain', 'geoNetwork_region', 'geoNetwork_subContinent', \n#             'totals_bounces', 'totals_hits', 'totals_newVisits', 'totals_pageviews', 'trafficSource_adContent', \n#             'trafficSource_adwordsClickInfo.adNetworkType', 'trafficSource_adwordsClickInfo.gclId', \n#             'trafficSource_adwordsClickInfo.isVideoAd', 'trafficSource_adwordsClickInfo.page', \n#             'trafficSource_adwordsClickInfo.slot', 'trafficSource_campaign', 'trafficSource_isTrueDirect', \n#             'trafficSource_keyword', 'trafficSource_medium', 'trafficSource_referralPath', 'trafficSource_source', \n#             'month', 'year']\n\n# col_list_imp = ['totals_pageviews', 'totals_hits', 'month', 'visitNumber', 'geoNetwork_country', \n#                 'trafficSource_referralPath', 'device_isMobile', 'geoNetwork_region', 'device_operatingSystem', \n#                 'geoNetwork_metro']\n\n# col_list_tp = ['trafficSource_adwordsClickInfo.gclId', 'totals_pageviews', 'device_isMobile', 'year',\n#                'geoNetwork_subContinent']\n# permutations = 100\n# pvalue_dict={}\n# for col in col_list:\n#     result = []\n#     for i in range(permutations):\n#         train_ex[col] = np.random.permutation(train_ex[col])\n#         result.append(LGBMFunction(train_ex))\n#     train_ex[col] = train[col]\n#     print(col)\n#     print(result)\n#     pvalue = sum(i < org_rmse for i in result) / permutations\n#     pvalue_dict[col] = pvalue\n#     print(pvalue)\n#     fig, ax = plt.subplots(figsize=(8,5))\n#     ax = sns.distplot(result, bins=min(100, permutations))\n#     ax.set_title(\"Permutation histogram for %s\" %col)\n#     ax.set(xlabel='RMSE', ylabel='Frequency')\n#     plt.axvline(org_rmse, color='k', linestyle='dashed', linewidth=2)\n#     plt.show()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"615562fdfbe70dc0ec3699c20e713854e0ebffb6"},"cell_type":"code","source":"# Q4 using logistic regression and SMOTE\n\nfrom sklearn.linear_model import LogisticRegression\nfrom imblearn.over_sampling import SMOTE\n\ndef buyOrNot(revenue):\n    if revenue > 0:\n        return 1\n    return 0\n\ndf_x = train\ndf_x = df_x.drop(columns = ['totals_transactionRevenue', 'date', 'sessionId', 'visitId'])\ndf_x_cols = df_x.columns\ndf_y = train['totals_transactionRevenue']\n\n# Spliting the data in the training and validation \nx_train, x_test, y_train, y_test = train_test_split(df_x, df_y, test_size = 0.2, random_state = 3)\n\n# converting the transition revenues to the binary classifications\ny_train = y_train.apply(buyOrNot)\ny_test = y_test.apply(buyOrNot)\n\n# Filling NaN with zeros\nx_train = x_train.fillna(0)\ny_train = y_train.fillna(0)\nx_test = x_test.fillna(0)\ny_test = y_test.fillna(0)\n\n# Generating sythentic data to balance the imbalanced classes\nsm = SMOTE(random_state=2)\nx_train, y_train = sm.fit_sample(x_train, y_train.ravel())\n\n# Converting Numpy matrices to the dataframes and rounding the catagorical variable to the nearist integer\nx_train = pd.DataFrame(x_train, columns=df_x_cols)\nx_train = x_train.round(0)\ny_train = pd.DataFrame(y_train, columns=['buyOrNot'])\ny_train = y_train.round(0)\nfor col in x_train.columns:\n    x_train[col] = x_train[col].astype(int)\ny_train['buyOrNot'] = y_train['buyOrNot'].astype(int) \n\n# Dropping the fullVisitorId column from the training and validation dataframes\nvisitorID = x_test['fullVisitorId']\nx_train = x_train.drop(columns = ['fullVisitorId'])\nx_test = x_test.drop(columns = ['fullVisitorId'])\n\n# creating the logistic regression model\nlogreg = LogisticRegression(random_state=0, solver='lbfgs')\n\n# fitting logistic regression model\nlogreg.fit(x_train,y_train)\n\n# final score of the model\nprint(\"Final score:\", logreg.score(x_test,y_test))\n\n#Classification probability for every user\nresult = logreg.predict_proba(x_test)\n\n# Matching the fullvisitorId with their respective probability to be in the positive revenue class\noneProb = {}\nfor i in range(len(result)):\n    oneProb[visitorID.iloc[i]] = result[i][1]\nsorted_by_value = sorted(oneProb.items(), key=lambda kv: kv[1], reverse=True)\nprint(\"Top 10\")\nfor i in range(10):\n    print(sorted_by_value[i][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91191623d72c1bc9e79ef54fb593aff26e37ae0f"},"cell_type":"code","source":"# Q4 using ad-hoc method\n\nf = {'visitNumber':['sum'], 'totals_transactionRevenue':['sum']}\nq4_df = train.groupby('fullVisitorId').agg(f)\nq4_df.columns = ['totalVisitNumber', 'totalRevenue']\nq4_df1 = q4_df.drop(q4_df[q4_df.totalVisitNumber < 10].index)\ntopUsers = [ (row.totalRevenue*1.0)/row.totalVisitNumber for index, row in q4_df1.iterrows() ]\nq4_df1['topUsers'] = topUsers\nq4_df1 = q4_df1.sort_values(by='topUsers', ascending=False)\nq4_df1.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}