{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=pd.read_csv(\"/kaggle/input/ga-customer-revenue-prediction/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sample = df.iloc[:3000, :]\ndf_sample.to_csv('/kaggle/working/df_sample.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import json\n# from pandas.io.json import json_normalize\n# # df2 = df_sample.copy().iloc[:3000, :]\n# df_totals = json_normalize(df['totals'].apply(lambda x: json.loads(x)))\n\n# # features = pd.get_dummies(df_concat.drop(columns=['transactionRevenue','trafficSource']))\n# # labels = df_concat['transactionRevenue'].fillna(0)\n# df_concat_total = pd.concat([df, df_totals], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_concat_total.to_csv('/kaggle/working/df_concat_total.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nfrom pandas.io.json import json_normalize\ndf2 = df_sample.copy().iloc[:3000, :]\ndf_device = json_normalize(df2['device'].apply(lambda x: json.loads(x)))\ndf_geo = json_normalize(df2['geoNetwork'].apply(lambda x: json.loads(x)))\ndf_totals = json_normalize(df2['totals'].apply(lambda x: json.loads(x)))\ndf_traffice = json_normalize(df2['trafficSource'].apply(lambda x: json.loads(x)))\ndf_concat = pd.concat([df2, df_device, df_geo, df_totals, df_traffice], axis=1)\ndf_concat.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import json\n# from pandas.io.json import json_normalize\n# df2 = df_sample.copy().iloc[:3000, :]\n\n# df_device = json_normalize(df2['device'].apply(lambda x: json.loads(x)))\n# df_geo = json_normalize(df2['geoNetwork'].apply(lambda x: json.loads(x)))\n# df_totals = json_normalize(df2['totals'].apply(lambda x: json.loads(x)))\n# df_traffice = json_normalize(df2['trafficSource'].apply(lambda x: json.loads(x)))\n# df_concat = pd.concat([df2, df_device, df_geo, df_totals, df_traffice], axis=1)\n# features = pd.get_dummies(df_concat.drop(columns=['transactionRevenue', 'device', 'geoNetwork', 'totals', 'trafficSource', 'sessionId', 'fullVisitorId']))\n# labels = df_concat['transactionRevenue'].fillna(0)\n# df_concat2 = pd.concat([features, labels], axis=1)\n# df_concat2.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"good_cols = df_concat[[\"channelGrouping\",\"date\",\"fullVisitorId\",\"visitId\",\"visitNumber\",\"visitStartTime\",\"browser\",\"deviceCategory\",\n              \"isMobile\",\"operatingSystem\",\"city\",\"continent\",\"country\",\"metro\",\"networkDomain\",\"region\",\"subContinent\",\"bounces\",                              \n              \"hits\",\"newVisits\",\"pageviews\",\"transactionRevenue\",\n              \"adContent\",\"adwordsClickInfo.adNetworkType\", \"adwordsClickInfo.gclId\",\"adwordsClickInfo.isVideoAd\",\n              \"adwordsClickInfo.page\", \"adwordsClickInfo.slot\",\"campaign\",\"isTrueDirect\",\"keyword\",\"medium\",\"referralPath\",\"source\"]]\n\n# KeyError: \"['transactions', 'sessionQualityDim', 'timeOnSite', 'totalTransactionRevenue'] not in index\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"good_cols.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# grouped = good_cols('fullVisitorId').groupby().min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# first_ses_from_the_period_start = df_concat['date'].min()\n# last_ses_from_the_period_end = df_concat['date'].max()\n# interval_dates = df_concat['date'].max() - df_concat['date'].min()\n# unique_date_num = length(unique(date))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transactionRevenue = df_concat[['fullVisitorId','transactionRevenue']].groupby('fullVisitorId').sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"first_ses_from_the_period_start = df_concat[['fullVisitorId','date']].groupby('fullVisitorId').min()\nlast_ses_from_the_period_end = df_concat[['fullVisitorId','date']].groupby('fullVisitorId').max()\ninterval_dates = df_concat[['fullVisitorId','date']].groupby('fullVisitorId').max() - df_concat[['fullVisitorId','date']].groupby('fullVisitorId').min()\n# unique_date_num = length(unique(date))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"channelGrouping = df_concat[['fullVisitorId', 'channelGrouping']].groupby('fullVisitorId').max()\nmaxVisitNum = df_concat[['fullVisitorId', 'visitNumber']].groupby('fullVisitorId').max()\nbrowser = df_concat[['fullVisitorId', 'browser']].groupby('fullVisitorId').max()\noperatingSystem = df_concat[['fullVisitorId','operatingSystem']].groupby('fullVisitorId').max()\ndeviceCategory = df_concat[['fullVisitorId','deviceCategory']].groupby('fullVisitorId').max()\ncontinent = df_concat[['fullVisitorId','continent']].groupby('fullVisitorId').max()\nsubContinent = df_concat[['fullVisitorId','subContinent']].groupby('fullVisitorId').max()\ncountry = df_concat[['fullVisitorId','country']].groupby('fullVisitorId').max()\nregion = df_concat[['fullVisitorId','region']].groupby('fullVisitorId').max()\nmetro = df_concat[['fullVisitorId','metro']].groupby('fullVisitorId').max()\ncity = df_concat[['fullVisitorId','city']].groupby('fullVisitorId').max()\nnetworkDomain = df_concat[['fullVisitorId','networkDomain']].groupby('fullVisitorId').max()\nsource = df_concat[['fullVisitorId','source']].groupby('fullVisitorId').max()\nmedium = df_concat[['fullVisitorId','medium']].groupby('fullVisitorId').max()\n# isVideoAd_mean = df_concat[['fullVisitorId','adwordsClickInfo.isVideoAd']].groupby('fullVisitorId').mean()\nisMobile = df_concat[['fullVisitorId','isMobile']].groupby('fullVisitorId').mean()\n# isTrueDirect = df_concat[['fullVisitorId','isTrueDirect']].groupby('fullVisitorId').mean()\nbounce_sessions = df_concat[['fullVisitorId','bounces']].groupby('fullVisitorId').sum()\nhits_sum = df_concat[['fullVisitorId','hits']].groupby('fullVisitorId').sum()\n# hits_mean = df_concat[['fullVisitorId','hits']].groupby('fullVisitorId').mean()\nhits_min = df_concat[['fullVisitorId','hits']].groupby('fullVisitorId').min()\nhits_max = df_concat[['fullVisitorId','hits']].groupby('fullVisitorId').max(),\n# hits_median = df_concat[['fullVisitorId','hits']].groupby('fullVisitorId').median()\n# hits_sd = df_concat[['fullVisitorId','hits']].groupby('fullVisitorId').std()\npageviews_sum = df_concat[['fullVisitorId','pageviews']].groupby('fullVisitorId').sum()\n# pageviews_mean = df_concat[['fullVisitorId','pageviews']].groupby('fullVisitorId').mean()\npageviews_min = df_concat[['fullVisitorId','pageviews']].groupby('fullVisitorId').min()\npageviews_max = df_concat[['fullVisitorId','pageviews']].groupby('fullVisitorId').max()\n# pageviews_median = df_concat[['fullVisitorId','pageviews']].groupby('fullVisitorId').median()\n# pageviews_sd = df_concat[['fullVisitorId','pageviews']].groupby('fullVisitorId').std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"session_cnt = len(df_concat['visitStartTime'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"good_cols2 = [[\"first_ses_from_the_period_start\",\"last_ses_from_the_period_end\", \n               \"interval_dates\", \"channelGrouping\",\"maxVisitNum\", \"browser\", \"operatingSystem\", \n               \"deviceCategory\", \"continent\", \"subContinent\", \"country\", \"region\", \"metro\", \n               \"city\",\"networkDomain\", \"source\", \"medium\", \"isMobile\", \"bounce_sessions\", \n               \"hits_sum\", \"hits_min\", \"hits_max\",\n              \"pageviews_sum\",\"pageviews_min\", \"pageviews_max\" , \"session_cnt\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"good_cols2 = pd.concat([[first_ses_from_the_period_start,last_ses_from_the_period_end, \n               interval_dates, channelGrouping,maxVisitNum, browser, operatingSystem, \n               deviceCategory, continent, subContinent, country, region, metro, \n               city,networkDomain, source, medium, isMobile, bounce_sessions, \n               hits_sum, hits_min, hits_max,\n              pageviews_sum,pageviews_min, pageviews_max , session_cnt]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y:目的変数に該当する列\nY = np.array(transactionRevenue)\n# X:説明変数に該当する列\nX = np.array(good_cols2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor\n\nX_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.3, random_state=0)\nX_train, X_valid, Y_train, Y_valid = train_test_split(X_train, Y_train, test_size=0.3, random_state=0)\n\nimport lightgbm as lgb\nlightgbm_model = lgb.LGBMRegressor()\nlightgbm_model.fit(X_train, Y_train)\nY_pred = lightgbm_model.predict(X_valid)\nlightgbm_model.score(X_train, Y_train)\n\nlightgbm_model_mse = mean_squared_error(np.log(Y_valid), np.log(Y_pred))\nprint('MSE(lightgbm):', np.sqrt(lightgbm_model_mse))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"channelGrouping = df_concat['channelGrouping'].max()\nmaxVisitNum = df_concat['visitNumber'].max()\nbrowser = df_concat['browser'].max()\noperatingSystem = df_concat['operatingSystem'].max()\ndeviceCategory = df_concat['deviceCategory'].max()\ncontinent = df_concat['continent'].max()\nsubContinent = df_concat['subContinent'].max()\ncountry = df_concat['country'].max()\nregion = df_concat['region'].max()\nmetro = df_concat['metro'].max()\ncity = df_concat['city'].max()\nnetworkDomain = df_concat['networkDomain'].max()\nsource = df_concat['source'].max()\nmedium = df_concat['medium'].max()\nisVideoAd_mean = df_concat['adwordsClickInfo.isVideoAd'].mean()\nisMobile = df_concat['isMobile'].mean()\nisTrueDirect = df_concat['isTrueDirect'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"bounce_sessions = df_concat['bounces'].sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"hits_sum = df_concat['hits'].sum()\nhits_mean = df_concat['hits'].mean()\nhits_min = df_concat['hits'].min()\nhits_max = df_concat['hits'].max(),\nhits_median = df_concat['hits'].median()\nhits_sd = df_concat['hits'].std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"pageviews_sum = df_concat['pageviews'].sum()\npageviews_mean = df_concat['pageviews'].mean()\npageviews_min = df_concat['pageviews'].min()\npageviews_max = df_concat['pageviews'].max()\npageviews_median = df_concat['pageviews'].median()\npageviews_sd = df_concat['pageviews'].std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"session_cnt = len(df_concat['visitStartTime'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat['transactionRevenue'] = df_concat['transactionRevenue'].values.astype(int)\ncor_ = df_concat2.corr().abs()\ncor_ = cor_.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"transactionRevenue = df_concat['transactionRevenue'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transactions = df_concat['transactions'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y:目的変数に該当する列\nY = np.array(df_concat['transactionRevenue'])\n# X:説明変数に該当する列\nX = channelGrouping + maxVisitNum\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor\n\nX_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.3, random_state=0)\nX_train, X_valid, Y_train, Y_valid = train_test_split(X_train, Y_train, test_size=0.3, random_state=0)\n\nimport lightgbm as lgb\nlightgbm_model = lgb.LGBMRegressor()\nlightgbm_model.fit(X_train, Y_train)\nY_pred = lightgbm_model.predict(X_valid)\nlightgbm_model.score(X_train, Y_train)\n\nlightgbm_model_mse = mean_squared_error(np.log(Y_valid), np.log(Y_pred))\nprint('MSE(lightgbm):', np.sqrt(lightgbm_model_mse))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_concat2.to_csv('/kaggle/working/df_concat2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# excluded_features = [\n#     'date', 'fullVisitorId', 'sessionId', 'transactionRevenue', \n#     'visitId', 'visitStartTime', 'non_zero_proba', 'vis_date'\n# ]\n\n# categorical_features = [\n#     _f for _f in df_concat2.columns\n#     if (_f not in excluded_features) & (df_concat2[_f].dtype == 'object')\n# ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for f in categorical_features:\n#     df_concat2[f], indexer = pd.factorize(df_concat2[f])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.model_selection import KFold, GroupKFold\n# folds = GroupKFold(n_splits=5)\n\n# df_concat2_features = [_f for _f in df_concat2.columns if _f not in excluded_features]\n# print(df_concat2_features)\n# oof_clf_preds = np.zeros(df_concat2.shape[0])\n# sub_clf_preds = np.zeros(df_concat2.shape[0])\n# for fold_, (trn_, val_) in enumerate(folds.split(cor_ , cor_ , groups=df_concat2['fullVisitorId'])):\n#     trn_x, trn_y = df_concat2[df_concat2_features].iloc[trn_], y_clf.iloc[trn_]\n#     val_x, val_y = df_concat2[df_concat2_features].iloc[val_], y_clf.iloc[val_]\n    \n#     clf = lgb.LGBMClassifier(\n#         num_leaves=31,\n#         learning_rate=0.03,\n#         n_estimators=1000,\n#         subsample=.9,\n#         colsample_bytree=.9,\n#         random_state=1\n#     )\n#     clf.fit(\n#         trn_x, trn_y,\n#         eval_set=[(val_x, val_y)],\n#         early_stopping_rounds=50,\n#         verbose=50\n#     )\n    \n#     oof_clf_preds[val_] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]\n#     print(roc_auc_score(val_y, oof_clf_preds[val_]))\n#     sub_clf_preds += clf.predict_proba(test[train_features], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits\n    \n# roc_auc_score(y_clf, oof_clf_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_cor = cor_[['transactionRevenue']].sort_values('transactionRevenue', ascending=False)\nsorted_cor[\n    sorted_cor['transactionRevenue'] > 0.03\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y:目的変数に該当する列\nY = np.array(df_concat2['transactionRevenue'])\n# X:説明変数に該当する列\nX = np.array(df_concat2[['pageviews_28','hits_42', 'pageviews_29','hits_72', 'pageviews_50', 'hits_38', 'pageviews_33',\n'pageviews_21', 'hits_41',\n'pageviews_34', 'source_mall.googleplex.com', 'referralPath_/','channelGrouping_Referral', 'metro_Chicago IL',\n'region_Illinois', 'city_Chicago', 'channelGrouping_Referral', 'hits_24', 'hits_35','hits_24','networkDomain_(not set)',\n'newVisits_1','medium_cpm','source_dfa','channelGrouping_Display','country_United States',\n'pageviews_14', 'subContinent_Northern America','city_New York', 'metro_New York NY', 'hits_17', 'pageviews_16',\n'isTrueDirect_True','city_Sunnyvale','continent_Americas', 'hits_19', 'pageviews_1', 'bounces_1', 'hits_1',\n'metro_San Francisco-Oakland-San Jose CA','operatingSystem_Macintosh','region_California','city_San Jose',\n'channelGrouping_Social','hits_50', 'source_youtube.com','region_not available in demo dataset',\n'city_not available in demo dataset', 'metro_not available in demo dataset', 'hits_11','hits_28',\n'pageviews_36', 'continent_Asia', 'metro_(not set)', 'source_google', 'city_San Francisco', 'continent_Europe',\n'medium_organic', 'channelGrouping_Organic Search', 'keyword_(not provided)']])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.ensemble import RandomForestRegressor\n\nX_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.3, random_state=0)\nX_train, X_valid, Y_train, Y_valid = train_test_split(X_train, Y_train, test_size=0.3, random_state=0)\nY_valid.astype(int).hist()\n\n# randomforest_model = RandomForestRegressor(n_estimators=10, random_state=0)\n# randomforest_model.fit(X_train, Y_train)\n# Y_pred = randomforest_model.predict(X_valid)\n# # print(Y_valid.values.astype(float))\n# # print(Y_pred)\n# randomforest_model_mse = mean_squared_error(np.log(1+Y_valid.values.astype(float)), np.log(1+Y_pred))\n\n# print('MSE(ランダムフォレスト):', np.sqrt(randomforest_model_mse))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# XGboostのライブラリをインポート\nimport xgboost as xgb\n# モデルのインスタンス作成\nXGboost_model = xgb.XGBRegressor()\nXGboost_model.fit(X_train, Y_train)\nY_pred = XGboost_model.predict(X_valid)\n\nXGboost_model_mse = np.log(mean_squared_error(Y_valid, Y_pred))\nprint('MSE(XGboost):', np.sqrt(XGboost_model_mse))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nlightgbm_model = lgb.LGBMRegressor()\nlightgbm_model.fit(X_train, Y_train)\nY_pred = lightgbm_model.predict(X_valid)\nlightgbm_model.score(X_train, Y_train)\n\nlightgbm_model_mse = np.log(mean_squared_error(Y_valid, Y_pred))\nprint('MSE(lightgbm):', np.sqrt(lightgbm_model_mse))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import Perceptron\nclf = Perceptron()\n\nclf.fit(X_train, Y_train)\nY_pred = clf.predict(X_valid)\nclf.score(X_train, Y_train)\n\nclf_mse = np.log(mean_squared_error(Y_valid, Y_pred))\nprint('MSE(perceptron):', np.sqrt(clf_mse))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X2 = df_concat2\nY2 = df_concat2['transactionRevenue']\n\nX2_train, X2_test, Y2_train, Y2_test = train_test_split(X2, Y2, test_size=0.3, random_state=0)\nX2_train, X2_valid, Y2_train, Y2_valid = train_test_split(X2_train, Y2_train, test_size=0.3, random_state=0)\n\nlightgbm_model = lgb.LGBMRegressor()\nlightgbm_model.fit(X2_train, Y2_train)\nY2_pred = lightgbm_model.predict(X2_valid)\nlightgbm_model.score(X2_train, Y2_train)\n\nlightgbm_model_mse = np.log(mean_squared_error(Y2_valid, Y2_pred))\nprint('MSE(lightgbm):', np.sqrt(lightgbm_model_mse))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in ['visitNumber', 'hits', 'pageviews', 'transactionRevenue']:\n    df_concat2[col] = df_concat2[col].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import Lasso\nfrom sklearn.linear_model import Ridge\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nimport itertools\n\ndef get_best_features(X, Y, feature_names, model):\n    # すべての説明変数名の組み合わせを入れるリスト\n    _name_list = []\n\n    # 指定された長さの組み合わせを作成する\n    for i in range(1, len(feature_names)+1):\n        for sublist in itertools.permutations(feature_names, i):\n            _name_list.append(list(sublist))\n\n    # 最良のMSEを入れる変数（大きい値を入れておく）\n    _best_mse = 999999999999\n    _best_mse_name = ''\n\n    # 総当りで比較する\n    for _name in _name_list:\n        _X = np.array(X[_name])\n\n        # データセットの分割。\n        _X_train, _X_test, _Y_train, _Y_test = train_test_split(_X, Y, test_size=0.3, random_state=0)\n        _X_train, _X_valid, _Y_train, _Y_valid = train_test_split(_X_train, _Y_train, test_size=0.3, random_state=0)\n\n        # モデルの作成～予測\n        model.fit(_X_train, _Y_train)\n        _Y_pred = model.predict(_X_valid)\n\n        # MSEを算出\n        _mse = mean_squared_error(_Y_valid, _Y_pred)\n\n        # 最小のmseを保管\n        if _mse < _best_mse:\n            _best_mse = _mse\n            _best_mse_name = _name\n\n    print(model.__class__.__name__, \":\", ','.join(_best_mse_name), \": MSE=\", _best_mse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # 説明変数名のリスト\n# feature_names = ['pageviews_28','pageviews_29','hits_72', 'pageviews_50', 'hits_38', 'pageviews_21', 'hits_41',\n# 'pageviews_34', 'source_mall.googleplex.com', 'referralPath_/','metro_Chicago IL',\n# 'region_Illinois', 'city_Chicago', 'channelGrouping_Referral', 'hits_24', 'hits_35',\n# 'pageviews_14','networkDomain_(not set)', 'pageviews_16', 'newVisits_1', 'hits_17',\n# 'isTrueDirect_True', 'country_United States', 'subContinent_Northern America',\n# 'city_Sunnyvale', 'hits_19', 'city_San Jose', 'medium_referral', 'continent_Americas',\n# 'metro_San Francisco-Oakland-San Jose CA', 'pageviews_1', 'bounces_1', 'hits_1',\n# 'region_California', 'operatingSystem_Macintosh']\n\n# # モデルのリストを用意\n# model_list = []\n# # model_list.append(LinearRegression())\n# # model_list.append(Lasso())\n# # model_list.append(Ridge())\n# # model_list.append(DecisionTreeRegressor())\n# model_list.append(RandomForestRegressor(n_estimators=100, random_state=0))\n# model_list.append(xgb.XGBRegressor())\n# model_list.append(lgb.LGBMClassifier())\n\n# for model in model_list:\n#     get_best_features(X, df_concat2['transactionRevenue'], feature_names, model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y:目的変数に該当する列\nY = np.array(df_concat2['transactionRevenue'])\n# X:説明変数に該当する列\nX2 = np.array(df_concat2[['pageviews_28','pageviews_29','hits_72', 'pageviews_50', 'hits_38', 'pageviews_21', 'hits_41',\n'pageviews_34', 'source_mall.googleplex.com', 'referralPath_/','metro_Chicago IL',\n'region_Illinois', 'city_Chicago', 'channelGrouping_Referral', 'hits_24', 'hits_35']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X2_train, X2_test, Y_train, Y_test = train_test_split(X2, Y, test_size=0.3, random_state=0)\nX2_train, X2_valid, Y_train, Y_valid = train_test_split(X2_train, Y_train, test_size=0.3, random_state=0)\n\nrandomforest_model = RandomForestRegressor(n_estimators=10, random_state=0)\nrandomforest_model.fit(X2_train, Y_train)\nY_pred = randomforest_model.predict(X2_valid)\n\nrandomforest_model_mse = mean_squared_error(Y_valid, Y_pred)\nprint('MSE(ランダムフォレスト):', randomforest_model_mse)\n\n\nXGboost_model = xgb.XGBRegressor()\nXGboost_model.fit(X2_train, Y_train)\nY_pred = XGboost_model.predict(X2_valid)\n\nXGboost_model_mse = mean_squared_error(Y_valid, Y_pred)\nprint('MSE(XGboost):', XGboost_model_mse)\n\n\nlightgbm_model = lgb.LGBMRegressor()\nlightgbm_model.fit(X2_train, Y_train)\nY_pred = lightgbm_model.predict(X2_valid)\n\nlightgbm_model_mse = mean_squared_error(Y_valid, Y_pred)\nprint('MSE(lightgbm):', lightgbm_model_mse)\n\n\nclf = Perceptron()\nclf.fit(X2_train, Y_train)\nY_pred = clf.predict(X2_valid)\nclf.score(X2_train, Y_train)\n\nclf_mse = mean_squared_error(Y_valid, Y_pred)\nprint('MSE(perceptron):', clf_mse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y:目的変数に該当する列\nY = np.array(df_concat2['transactionRevenue'])\n# X:説明変数に該当する列\nX3 = np.array(df_concat2[['pageviews_28','pageviews_29','hits_72', 'pageviews_50', 'hits_38', 'pageviews_21', 'hits_41',\n'pageviews_34', 'source_mall.googleplex.com']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X3_train, X3_test, Y_train, Y_test = train_test_split(X3, Y, test_size=0.3, random_state=0)\nX3_train, X3_valid, Y_train, Y_valid = train_test_split(X3_train, Y_train, test_size=0.3, random_state=0)\n\nrandomforest_model = RandomForestRegressor(n_estimators=10, random_state=0)\nrandomforest_model.fit(X3_train, Y_train)\nY_pred = randomforest_model.predict(X3_valid)\n\nrandomforest_model_mse = mean_squared_error(Y_valid, Y_pred)\nprint('MSE(ランダムフォレスト):', randomforest_model_mse)\n\n\nXGboost_model = xgb.XGBRegressor()\nXGboost_model.fit(X3_train, Y_train)\nY_pred = XGboost_model.predict(X3_valid)\n\nXGboost_model_mse = mean_squared_error(Y_valid, Y_pred)\nprint('MSE(XGboost):', XGboost_model_mse)\n\n\nlightgbm_model = lgb.LGBMRegressor()\nlightgbm_model.fit(X3_train, Y_train)\nY_pred = lightgbm_model.predict(X3_valid)\n\nlightgbm_model_mse = mean_squared_error(Y_valid, Y_pred)\nprint('MSE(lightgbm):', lightgbm_model_mse)\n\nclf = Perceptron()\nclf.fit(X3_train, Y_train)\nY_pred = clf.predict(X3_valid)\nclf.score(X3_train, Y_train)\n\nclf_mse = mean_squared_error(Y_valid, Y_pred)\nprint('MSE(perceptron):', clf_mse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat2.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df3_concat.astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"df_browser = pd.get_dummies(df2['browser'].apply(lambda x: json.loads(x)))\ndf_concat2 = pd.concat([df_concat, df_browser], axis=1)\ndf_concat.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"df3_concat.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df3_concat = df2_concat[[latitude_not available in demo dataset',\n       'longitude_not available in demo dataset',\n       'networkLocation_not available in demo dataset', 'visits', 'hits',\n       'pageviews', 'bounces', 'newVisits', 'transactionRevenue',\n       'campaign', 'source', 'medium', 'keyword',\n       'adwordsClickInfo.criteriaParameters', 'isTrueDirect',\n       'referralPath', 'adwordsClickInfo.page', 'adwordsClickInfo.slot',\n       'adwordsClickInfo.gclId', 'adwordsClickInfo.adNetworkType',\n       'adwordsClickInfo.isVideoAd', 'adContent'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df3_concat.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\ndf2_concat.plot(\n    kind='hist', # グラフ種別\n    subplots=True, # 項目ごとに分割\n    figsize=(10, 10), # 描画する面積\n    layout=(25, 25) # レイアウト（行数, 列数）\n)\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat['transactionRevenue']=df_concat['transactionRevenue'].fillna(df_concat['transactionRevenue'].median())\n#df_concat['campaign']=df_concat['campaign'].astype(int)\ndf_concat['transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat['source']=df_concat['source'].fillna(df_concat['source'].median())\ndf_concat['source']=df_concat['source'].astype(int)\ndf_concat['source']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat['transactionRevenue']=df_concat['transactionRevenue'].fillna(df_concat['transactionRevenue'].median())\ndf_concat['transactionRevenue']=df_concat['transactionRevenue'].astype(int)\ndf_concat['transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat.fillna(df_concat.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 正規化用にdatasetの一部を抜き出し\ndf3 = df2[[\"date\", \"visitId\", \"visitNumber\", \"visitStartTime\"]]\ndf3.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# z-score normalization\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\ndf3_std = sc.fit_transform(df3)\ndf3_std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nfrom pandas.io.json import json_normalize\ndf2 = df1.copy()\ndf_device = pd.get_dummies(json_normalize(df['device'].apply(lambda x: json.loads(x))))\ndf_geo = pd.get_dummies(json_normalize(df['geoNetwork'].apply(lambda x: json.loads(x))))\ndf_totals = json_normalize(df['totals'].apply(lambda x: json.loads(x)))\ndf_traffice = pd.get_dummies(json_normalize(df['trafficSource'].apply(lambda x: json.loads(x))))\ndf_concat = pd.concat([df2, df_device, df_geo, df_totals, df_traffice], axis=1)\ndf_concat.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\ndef json_to_dict(json_str):\n    json_str_ = json_str.replace(\"'\", '\"')\n    obj = json.loads(json_str_)\n#     for dimension in obj:\n#         for key, value in dimension.items():\n#             print(f'{key}={value}')\n    return obj\ndf2 = df.copy()\ndf2['customDimensions_'] = df2['customDimensions'].apply(lambda json_str: json_to_dict(json_str))\ndf2.head()\nareas = [customDimension[0]['value'] if len(customDimension) > 0 else 'None' for customDimension in df2['customDimensions_'].values]\ndf2['Area'] = areas\ndf2 = pd.get_dummies(df2, columns=['Area'])\ndf2.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}