{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=pd.read_csv('../input/ga-customer-revenue-prediction/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sample = df.iloc[:1000, :]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nfrom pandas.io.json import json_normalize\ndf2 = df_sample.copy().iloc[:1000, :]\n\ndf_device = json_normalize(df2['device'].apply(lambda x: json.loads(x)))\ndf_geo = json_normalize(df2['geoNetwork'].apply(lambda x: json.loads(x)))\ndf_totals = json_normalize(df2['totals'].apply(lambda x: json.loads(x)))\ndf_traffice = json_normalize(df2['trafficSource'].apply(lambda x: json.loads(x)))\ndf_concat = pd.concat([df2, df_device, df_geo, df_totals, df_traffice], axis=1)\nfeatures = pd.get_dummies(df_concat.drop(columns=['transactionRevenue', 'device', 'geoNetwork', 'totals', 'trafficSource', 'sessionId', 'fullVisitorId']))\nlabels = df_concat['transactionRevenue'].fillna(0)\ndf_concat2 = pd.concat([features, labels], axis=1)\ndf_concat2.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat2['transactionRevenue'] = df_concat2['transactionRevenue'].values.astype(int)\ncor_ = df_concat2.corr().abs()\ncor_ = cor_.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_cor = cor_[['transactionRevenue']].sort_values('transactionRevenue', ascending=False)\nsorted_cor[\n    sorted_cor['transactionRevenue'] > 0.05\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_concat.to_csv('/kaggle/working/sample.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 正規化用にdatasetの一部を抜き出し\ndf2 = df1[[\"date\", \"visitId\", \"visitNumber\", \"visitStartTime\"]]\ndf2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# z-score normalization\nfrom sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\ndf2_std = sc.fit_transform(df2)\ndf2_std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import Lasso\nfrom sklearn.linear_model import Ridge\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nimport itertools\n\ndef get_best_features(x, y, feature_names, model):\n    # すべての説明変数名の組み合わせを入れるリスト\n    _name_list = []\n\n    # 指定された長さの組み合わせを作成する\n    for i in range(1, len(feature_names)+1):\n        for sublist in itertools.permutations(feature_names, i):\n            _name_list.append(list(sublist))\n\n    # 最良のMSEを入れる変数（大きい値を入れておく）\n    _best_mse = 999999999999\n    _best_mse_name = ''\n\n    # 総当りで比較する\n    for _name in _name_list:\n        _x = np.array(x[_name])\n\n        # データセットの分割。X4を使っていることに注意\n        _x_train, _x_test, _y_train, _y_test = train_test_split(_x, y, test_size=0.3, random_state=0)\n        _x_train, _x_valid, _y_train, _y_valid = train_test_split(_x_train, _y_train, test_size=0.3, random_state=0)\n\n        # モデルの作成～予測\n        model.fit(_x_train, _y_train)\n        _y_pred = model.predict(_x_valid)\n\n        # MSEを算出\n        _mse = mean_squared_error(_y_valid, _y_pred)\n\n        # 最小のmseを保管\n        if _mse < _best_mse:\n            _best_mse = _mse\n            _best_mse_name = _name\n\n    print(model.__class__.__name__, \":\", ','.join(_best_mse_name), \": MSE=\", _best_mse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 説明変数名のリスト\nfeature_names = ['date', 'sessionId', 'visitId', 'visitNumber', 'visitStartTime', 'channelGrouping_(Other)', 'channelGrouping_Affiliates']\n#\tvisitId\tvisitNumber\tvisitStartTime\tchannelGrouping_(Other)\tchannelGrouping_Affiliates\tchannelGrouping_Direct\tchannelGrouping_Display\tchannelGrouping_Organic Search\tchannelGrouping_Paid Search\tchannelGrouping_Referral\tchannelGrouping_Social\tsocialEngagementType_Not Socially Engaged\n\n# モデルのリストを用意\nmodel_list = []\nmodel_list.append(LinearRegression())\nmodel_list.append(Lasso())\nmodel_list.append(Ridge())\nmodel_list.append(DecisionTreeRegressor())\nmodel_list.append(RandomForestRegressor(n_estimators=100, random_state=0))\n\nfor model in model_list:\n    get_best_features(df1, df1['fullVisitorld'], feature_names, model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\ndef json_to_dict(json_str):\n    json_str_ = json_str.replace(\"'\", '\"', \"'\")\n    obj = json.loads(json_str_)\n\ndf2['hits_'] = df2['hits'].apply(lambda json_str: json_to_dict(json_str))\ndf2.head()\n#browser = [device['key']['value'] if len(device) > 0 else 'None' for device in df2['device_'].values]\n##df2['Browser'] = browser\n#df2 = pd.get_dummies(df2, columns=['Browser'])\n#df2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\ndef json_to_dict(json_str):\n    json_str_ = json_str.replace(\"'\", '\"', \"'\")\n    obj = json.loads(json_str_)\n#    for dimension in obj:\n#         for key, value in dimension.items():\n#            print(f'{key}={value}')\n#    return obj\ndf2 = df1.copy()\ndf2['device_'] = df2['device'].apply(lambda json_str: json_to_dict(json_str))\ndf2.head()\nbrowsers = [browser[0]['value'] if len(customDimension) > 0 else 'None' for device in df2['device_'].values]\ndf2['Browsers'] = browsers\ndf2 = pd.get_dummies(df2, columns=['Browsers'])\ndf2.head()\n\n#{\"browser\": \"Chrome\", \"browserVersion\": \"not available in demo dataset\", \"browserSize\": \"not available in demo dataset\", \"operatingSystem\": \"Windows\", \"operatingSystemVersion\": \"not available in demo dataset\", \"isMobile\": false, \"mobileDeviceBranding\": \"n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\ndef json_to_dict(json_str):\n    json_str_ = json_str.replace(\"'\", '\"')\n    obj = json.loads(json_str_)\n#     for dimension in obj:\n#         for key, value in dimension.items():\n#             print(f'{key}={value}')\n    return obj\ndf2 = df.copy()\ndf2['customDimensions_'] = df2['customDimensions'].apply(lambda json_str: json_to_dict(json_str))\ndf2.head()\nareas = [customDimension[0]['value'] if len(customDimension) > 0 else 'None' for customDimension in df2['customDimensions_'].values]\ndf2['Area'] = areas\ndf2 = pd.get_dummies(df2, columns=['Area'])\ndf2.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Y:目的変数に該当する列\nY = np.array(dataset3['fullVisitorId'])\n# X:説明変数に該当する列\nX = np.array(dataset3[['date', 'visitNumber', 'visitStartTime', 'visitId']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}