{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pandas.io.json import json_normalize\nimport json\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\nimport datetime\nimport seaborn as sns\nfrom bayes_opt import BayesianOptimization\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport gc\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold,train_test_split\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e26b1fedd56621df437e4f26bf3d8b66186373d7"},"cell_type":"code","source":"def add_time_features(df):\n    df['date'] = pd.to_datetime(df['date'], format='%Y%m%d', errors='ignore')\n    df['year'] = df['date'].apply(lambda x: x.year)\n    df['month'] = df['date'].apply(lambda x: x.month)\n    df['day'] = df['date'].apply(lambda x: x.day)\n    df['weekday'] = df['date'].apply(lambda x: x.weekday())\n    df['visitStartTime_'] = pd.to_datetime(df['visitStartTime'],unit=\"s\")\n    df['visitStartTime_year'] = df['visitStartTime_'].apply(lambda x: x.year)\n    df['visitStartTime_month'] = df['visitStartTime_'].apply(lambda x: x.month)\n    df['visitStartTime_day'] = df['visitStartTime_'].apply(lambda x: x.day)\n    df['visitStartTime_weekday'] = df['visitStartTime_'].apply(lambda x: x.weekday())\n    return df\ndate_features = [#\"year\",\"month\",\"day\",\"weekday\",'visitStartTime_year',\n    \"visitStartTime_month\",\"visitStartTime_day\",\"visitStartTime_weekday\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66ff09ee11c9b0c3d4ea1b19eb1e058b98d67212"},"cell_type":"code","source":"%%time\ntrain_df = load_df(\"../input/train.csv\")\ntest_df = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"68f57a713fc130288581d0ddd1538f5fba47f7c9"},"cell_type":"markdown","source":"# Glimpse of data"},{"metadata":{"_uuid":"bb777c478183b6296f83446abdc71b51eb59eff9"},"cell_type":"markdown","source":"## Samples of train data"},{"metadata":{"trusted":true,"_uuid":"38667ec5283438e1be6e44fe957a24d85cce628b"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8166e604313bed85782755c8dd515c7fe830659"},"cell_type":"markdown","source":"## Data type"},{"metadata":{"trusted":true,"_uuid":"0c14b0232151799f29e82daed466b99c3fa27a3a"},"cell_type":"code","source":"train_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72840a582108454d8c515292b3afed1220dbce1f"},"cell_type":"code","source":"test_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e0994b2afa280bcf105d55e55561291b1d4ee2c"},"cell_type":"code","source":"pd.value_counts(train_df.dtypes).plot(kind=\"bar\")\nplt.title(\"type of train data\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f02e437a881fcd50004224f51780806639a0ddce"},"cell_type":"code","source":"def bar_plot(column,**args):\n    pd.value_counts(train_df[column]).plot(kind=\"bar\",**args)\n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"052ee9f6c5629fa0395f2561c3161b62b168fdef"},"cell_type":"markdown","source":"# Remove constant column"},{"metadata":{"trusted":true,"_uuid":"0a4ae348043ba5ffa9cff2fd17340ba3651b0a4d"},"cell_type":"code","source":"constant_column = [col for col in train_df.columns if len(train_df[col].unique()) == 1]\nprint(list(constant_column))\ntrain_df.drop(columns=constant_column,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1698cf8b76c03b962d3b95dd74fa44999eb5b02d"},"cell_type":"code","source":"num_col = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", \n           'totals.bounces',  'totals.newVisits']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4cd4c8e6b06ec4df075e18ba39d80e4830d9fad"},"cell_type":"code","source":"for col in num_col:\n    train_df[col] = train_df[col].fillna(\"0\").astype(\"int32\")\n    test_df[col] = test_df[col].fillna(\"0\").astype(\"int32\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06becf3c2f491eb2d0121b5b6516ea6e795095e8"},"cell_type":"code","source":"train_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f6dd6181fb410c18329f25bfafc1568bd5b52e3"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"689489d948f918536725340f1e7521b3de2e1817"},"cell_type":"markdown","source":"# Univariant viariable analysis"},{"metadata":{"_uuid":"30061f23ab443a2843fcbad9edef94d316fbfa26"},"cell_type":"markdown","source":"## ChannelGrouping\n```\nThe channel via which the user came to the Store.\n```"},{"metadata":{"trusted":true,"_uuid":"f674f5cb848f29c3e4be5d63a5de177c346e1c49"},"cell_type":"code","source":"bar_plot(\"channelGrouping\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"490dc00ccca0dd0d6153b82d4c9e4eb954cc482f"},"cell_type":"markdown","source":"## device.browser"},{"metadata":{"trusted":true,"_uuid":"a5aea53067692ed7d07cb07963525865c995806b"},"cell_type":"code","source":"bar_plot(\"device.browser\",figsize=(12,10))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"746552a996fedc2ecca4cb6c7f2c437e2af67b63"},"cell_type":"markdown","source":"## device.isMobile"},{"metadata":{"trusted":true,"_uuid":"081e8d9241c175f6e118d96462c7371fd75afb55"},"cell_type":"code","source":"bar_plot(\"device.isMobile\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fe096f88913c87f19395f9b7b70265a51694b4d"},"cell_type":"markdown","source":"## device.deviceCategory"},{"metadata":{"trusted":true,"_uuid":"bec145c1db4946027e0b4898fb938c1d5260cd1a"},"cell_type":"code","source":"bar_plot(\"device.deviceCategory\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d94e8ebf03df8bf59c14911fb4594b779b00861f"},"cell_type":"markdown","source":"# Feature engineering"},{"metadata":{"trusted":true,"_uuid":"f2316578cb3a4b341553c4282b01b8428ee6acde"},"cell_type":"code","source":"new_features = [\"hits_per_pageviews\"]\nnew_category_features = [\"is_high_hits\"]\ndef feature_engineering(df):\n    line = 4\n    df['hits_per_pageviews'] = (df[\"totals.hits\"]/(df[\"totals.pageviews\"])).apply(lambda x: 0 if np.isinf(x) else x)\n    df['is_high_hits'] = np.logical_or(train_df[\"totals.hits\"]>line,train_df[\"totals.pageviews\"]>line).astype(np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e47cd8590dd0d36d591c84136fd571ee767f7746"},"cell_type":"code","source":"feature_engineering(train_df)\nfeature_engineering(test_df)\nadd_time_features(train_df)\n_ = add_time_features(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd9b603d0b6790ee3a2e7f54fb558c8ae296d862"},"cell_type":"code","source":"category_features = [\"channelGrouping\", \"device.browser\", \n            \"device.deviceCategory\", \"device.operatingSystem\", \n            \"geoNetwork.city\", \"geoNetwork.continent\", \n            \"geoNetwork.country\", \"geoNetwork.metro\",\n            \"geoNetwork.networkDomain\", \"geoNetwork.region\", \n            \"geoNetwork.subContinent\",\n            #\"trafficSource.adContent\", \n            #\"trafficSource.adwordsClickInfo.adNetworkType\", \n            #\"trafficSource.adwordsClickInfo.gclId\", \n            #\"trafficSource.adwordsClickInfo.page\", \n            #\"trafficSource.adwordsClickInfo.slot\",\n            #\"trafficSource.campaign\",\n            #\"trafficSource.keyword\", \n            \"trafficSource.medium\", \n            #\"trafficSource.referralPath\", \n            \"trafficSource.source\",\n\n            #'trafficSource.adwordsClickInfo.isVideoAd',\n            'trafficSource.isTrueDirect',\n            #\"filtered_keyword\"\n            ] + date_features\ntarget = 'totals.transactionRevenue'\nuseless_col = [\"trafficSource.adContent\", \n              \"trafficSource.adwordsClickInfo.adNetworkType\", \n              \"trafficSource.adwordsClickInfo.page\",\n              \"trafficSource.adwordsClickInfo.slot\",\n              \"trafficSource.campaign\",\n              \"trafficSource.referralPath\",\n              'trafficSource.adwordsClickInfo.isVideoAd',\n              \"trafficSource.adwordsClickInfo.gclId\",\n              \"trafficSource.keyword\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dccd51f87cf106fefe2a7ce34b5b58326999a404"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cba355111a7060e71d42c833eeb9429d29342579"},"cell_type":"markdown","source":"## Useless features"},{"metadata":{"trusted":true,"_uuid":"fb1960f1d0c7f201de3608f5c54697c494bef0d6"},"cell_type":"code","source":"useless_df = train_df[useless_col]\nuseless_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7d718e472fa93066d708eba5401762348b16d75"},"cell_type":"code","source":"for col in useless_col:\n    print(\"-\"*10,col,\"-\"*10)\n    print(\"unique value numbers:\",len(useless_df[col].unique()))\n    print(\"null rate:\",useless_df[col].isna().sum()/len(useless_df[col]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dbdb53682076b55d8999b5290664c72b4386d047"},"cell_type":"code","source":"for col in category_features:\n    print(\"-\"*10,col,\"-\"*10)\n    print(\"unique value numbers:\",len(train_df[col].unique()))\n    print(\"null rate:\",train_df[col].isna().sum()/len(train_df[col]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f122d3f6b07cb0c3f68a88d82bea8497f6c6c6d3"},"cell_type":"code","source":"train_df[target] = train_df[target].fillna(\"0\").astype(\"int32\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3afab89dff0a6a466db03ff917d81d86d14f1bd9"},"cell_type":"code","source":"all_features = category_features+num_col+new_features+new_category_features\nall_features\n# dev_df = train_df[train_df['date']<=pd.to_datetime('20170531', format='%Y%m%d')]\n# val_df = train_df[train_df['date']>pd.to_datetime('20170531', format='%Y%m%d')]\n\n# dev_x = dev_df[all_features]\n# dev_y = dev_df[target]\n# val_x = val_df[all_features]\n# val_y = val_df[target]\n# test_x = test_df[all_features]\n# for col in category_features:\n#     print(\"transform column {}\".format(col))\n#     lbe = LabelEncoder()\n#     lbe.fit(pd.concat([train_df[col],test_x[col]]).astype(\"str\"))\n#     dev_x[col] = lbe.transform(dev_x[col].astype(\"str\"))\n#     val_x[col] = lbe.transform(val_x[col].astype(\"str\"))\n#     test_x[col] = lbe.transform(test_x[col].astype(\"str\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a184b2df75ef7f2c3287cd739ffca7ef8c3a6b4"},"cell_type":"markdown","source":"# Hits and Pageviews"},{"metadata":{"_uuid":"dcf621eb22cdabba54c0add2cf5666723e43d157"},"cell_type":"markdown","source":"## totals.hits"},{"metadata":{"trusted":true,"_uuid":"c51a62d5a0fe5db7910951d0e0a0c89a2b101f04"},"cell_type":"code","source":"train_df[\"totals.hits\"].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d195d867d5f7741f7df78d02c2a938e5d95282a"},"cell_type":"code","source":"sns.distplot(train_df[\"totals.hits\"],kde=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"afba8ddefcef16633a3524144718a64dcde19b85"},"cell_type":"markdown","source":"Most of hits are less than 4."},{"metadata":{"_uuid":"308ebc2fb21bce126151cea429bea9165b254168"},"cell_type":"markdown","source":"## totals.pageviews"},{"metadata":{"trusted":true,"_uuid":"1253af14f1a99b91a737ef3ad6d2b35364c7c6fd"},"cell_type":"code","source":"train_df[\"totals.pageviews\"].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fae8013ab39510c2bf02d76606d6d9805084bbe0"},"cell_type":"code","source":"sns.distplot(train_df[\"totals.pageviews\"],kde=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"437b8b35dc8bd52441d88469dae63e35f1fcd9c9"},"cell_type":"markdown","source":"Pageviews has same distribution with hits."},{"metadata":{"trusted":true,"_uuid":"6aaa5c83d7ba49b940cd4814500258e72cf19069"},"cell_type":"code","source":"sns.jointplot(\"totals.pageviews\",\"totals.hits\",data=train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2672f36a45b9f58845451f7debe661df53575355"},"cell_type":"code","source":"sns.jointplot(\"totals.pageviews\",target,data=train_df)\nsns.jointplot(\"totals.hits\",target,data=train_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a0aeed454a33d305bbe90f0e116ea56110deedc4"},"cell_type":"markdown","source":"**Filter out high hits or pageviews record**"},{"metadata":{"trusted":true,"_uuid":"9ec82a0186f8d242e3361de5d85812546e9bf7b8"},"cell_type":"code","source":"line = 4\nhigh_hits_pageviews_df = train_df[np.logical_or(train_df[\"totals.hits\"]>line,train_df[\"totals.pageviews\"]>line)]\nlow_hits_pageviews_df = train_df[np.logical_and(train_df[\"totals.hits\"]<=line,train_df[\"totals.pageviews\"]<=line)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1636d43928bace26937daf72674086e69bf72fd"},"cell_type":"code","source":"print(\"high rate :\",high_hits_pageviews_df.shape[0]/train_df.shape[0])\nprint(\"low rate :\",low_hits_pageviews_df.shape[0]/train_df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"111f4801d7f39b2d538d77ca97a98e9e4f3cebfe"},"cell_type":"code","source":"high_hits_pageviews_df[target].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cff0aa8dfe126af9e2de23fa5c7465cdf7286c79"},"cell_type":"code","source":"low_hits_pageviews_df[target].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba84697d19fe4bc5c30caeb692207e3bd4438b1d"},"cell_type":"code","source":"fig,axes = plt.subplots(nrows=1,ncols=2,figsize=(16,8))\nsns.distplot(high_hits_pageviews_df[target],kde=False,ax=axes[0])\naxes[0].set_title(\"distribution of high hits transactionRevenue\")\n\nsns.distplot(low_hits_pageviews_df[target],kde=False,ax=axes[1])\naxes[1].set_title(\"distribution of low hits transactionRevenue\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a31c9c9584384179a922b2ed3172e781fbfa0d49"},"cell_type":"code","source":"print(\"zero rate of transactionRevenue:\",(train_df[target]==0).sum()/train_df.shape[0])\nprint(\"zero rate of high hits transactionRevenue:\",(high_hits_pageviews_df[target]==0).sum()/ high_hits_pageviews_df.shape[0])\nprint(\"zero rate of low hits transactionRevenue:\",(low_hits_pageviews_df[target]==0).sum()/ low_hits_pageviews_df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91356b9450dcab4543a4f60979fb80f3c3ca9a1d"},"cell_type":"markdown","source":"high hits and pageviews has low zero rate, low hits and pageviews has high zero rate\n\n**how about hits per pageviews? **"},{"metadata":{"trusted":true,"_uuid":"ef09ce25e8f4c62fa510144d236a04458a1c9638"},"cell_type":"code","source":"train_df[\"hits_per_pageviews\"].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a502386827f45f0848f90b8bd424dc1468e3faed"},"cell_type":"code","source":"sns.jointplot(\"hits_per_pageviews\",target,data=train_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22fdf0348ada21affb493d400d8231da48d3df9e"},"cell_type":"markdown","source":"**High hits_per_pageviews has 0 revenue. Maybe I can use this feature.**"},{"metadata":{"_uuid":"63d3c799e001510a0506c9b6072f5566ecdb4424"},"cell_type":"markdown","source":"# visitStartTime"},{"metadata":{"trusted":true,"_uuid":"9fc2b3f075a32a56bb150ff758c691bc1de23e00"},"cell_type":"code","source":"visitStartTime_df = train_df[[\"visitStartTime\",'visitStartTime_year',\"visitStartTime_month\",\"visitStartTime_day\",\"visitStartTime_weekday\",target]]\nvisitStartTime_df[\"visitStartTime\"] = pd.to_datetime(visitStartTime_df[\"visitStartTime\"],unit=\"s\")\nvisitStartTime_df[\"visitStartDate\"] = visitStartTime_df[\"visitStartTime\"].apply(lambda x: x.date())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b9e91eb16fbad2972fc3724e287b0e43538cf4c"},"cell_type":"code","source":"def plot_dist_date(col,kind=\"bar\"):\n    fig,axes = plt.subplots(nrows=1,ncols=2,figsize=(12,6))\n    visitStartTime_df.groupby(col)[target].agg([\"sum\"]).plot(kind=kind,title=\"sum of transactionRevenue:\"+col,ax=axes[0])\n    visitStartTime_df.groupby(col)[target].agg([\"count\"]).plot(kind=kind,title=\"count of transactionRevenue:\"+col,ax=axes[1])\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9eface64b570413d2a07f066c3f31a48c7bd02b9"},"cell_type":"code","source":"plot_dist_date(\"visitStartDate\",kind=\"line\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"_uuid":"a0fd705cfcd978f677166d49445724bb6d7f3de5"},"cell_type":"markdown","source":"When I convert visitStartTime to numerical type, the feature has high importance. It can not make sence. I try to remove this feature. LB score decrease.\n\n**What happen?**"},{"metadata":{"trusted":true,"_uuid":"082e1606e57e313facd18c4a5e7a87de6ad778ca"},"cell_type":"code","source":"test_visitStartTime_df = test_df[[\"visitStartTime\",'visitStartTime_year',\"visitStartTime_month\",\"visitStartTime_day\",\"visitStartTime_weekday\"]]\ntest_visitStartTime_df[\"visitStartTime\"] = pd.to_datetime(test_visitStartTime_df[\"visitStartTime\"],unit=\"s\")\ntest_visitStartTime_df[\"visitStartDate\"] = test_visitStartTime_df[\"visitStartTime\"].apply(lambda x: x.date())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19a61d3dc6987bb705722dd219cc3e813bdd1350"},"cell_type":"code","source":"test_visitStartTime_df.groupby(\"visitStartDate\")[\"visitStartTime\"].agg(\"count\").plot(figsize=(8,6),title=\"count of test\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c8dce8968409c1022662e8fbaa38df14f41f800f"},"cell_type":"markdown","source":"train_data: (2016-08,2017-08)\ntest_data: (2017-08,2018-05)\n\nThe count distribution of test data is different from train data\n* The year feature may be useless"},{"metadata":{"trusted":true,"_uuid":"6e734297cfd02ea5965efb0dc719d2ad36685fc7"},"cell_type":"code","source":"plot_dist_date(\"visitStartTime_year\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db180d71d71219f6751ebf08aac1aa26ae81ada7"},"cell_type":"code","source":"plot_dist_date(\"visitStartTime_month\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"748b99e947b95db098e3774215540d2e323b8230"},"cell_type":"code","source":"plot_dist_date(\"visitStartTime_day\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d9a2e1b0a09d2e94ebe71ec2600a319b48a3b2b"},"cell_type":"markdown","source":"Transaction count in day is uniform distribution, unless 31, because of many months no 31 days.\n\ntransaction revenue is not uniform distribution, **visitStartTime_day may be a useful feature .**"},{"metadata":{"trusted":true,"_uuid":"ea48e96b5809666fa3b10675aed0e8c53e66bc25"},"cell_type":"code","source":"plot_dist_date(\"visitStartTime_weekday\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2b3fb059a4a75267de1a7f508d46f5dc675a6ed"},"cell_type":"markdown","source":"# trafficSource.keyword\nTransaction revenue must related to goods. The keyword may have relation to goods.\n\ncontinue to analyse"},{"metadata":{"trusted":true,"_uuid":"ece4d30451e6067bb6c4b570fc8ba802c8dd4bef"},"cell_type":"code","source":"col = \"trafficSource.cleanedkeyword\"\ntrain_df[col] = train_df[\"trafficSource.keyword\"].apply(lambda x :x if isinstance(x,float) and np.isnan(x) else x.lower()).apply(lambda x :x if isinstance(x,float) and np.isnan(x) else x.replace(\"+\", \"\"))\ntest_df[col] = test_df[\"trafficSource.keyword\"].apply(lambda x :x if isinstance(x,float) and np.isnan(x) else x.lower())\nprint(\"-\"*10,\"train\",\"-\"*10)\nprint(\"unique value numbers:\",len(train_df[col].unique()))\nprint(\"null rate:\",train_df[col].isna().sum()/len(train_df[col]))\nprint(\"-\"*10,\"test\",\"-\"*10)\nprint(\"unique value numbers:\",len(test_df[col].unique()))\nprint(\"null rate:\",test_df[col].isna().sum()/len(test_df[col]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b5df8eb6fdcad4da6f9f62d994afef8cf7cf094"},"cell_type":"code","source":"pd.value_counts(train_df[col]).sort_values(ascending=False)[0:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dfdc7761d70960bed2ef6f5a578b37ef4a0d4c5"},"cell_type":"code","source":"pd.value_counts(test_df[col]).sort_values(ascending=False)[0:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"daf8b1468fe39a3edc6f40347f11b5382adb9930"},"cell_type":"code","source":"train_df.groupby(col)[target].agg(\"sum\").sort_values(ascending=False)[0:29].apply(lambda x: np.log1p(x)).plot(kind=\"bar\",figsize=(20,5),title=\"sum revenue of keyword\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9fe69d3a44e04263ca1c4bdc421305a174811c29"},"cell_type":"markdown","source":"There are only 3 keywords has none zero revenues. \n\nDoes these keywords occurs in test data? \n\nHow do I encode this feature?"},{"metadata":{"trusted":true,"_uuid":"bc1ceff8c6086512dc463e73874cf1beb2636ebb"},"cell_type":"code","source":"none_zero_keywords= set(train_df.groupby(col)[target].agg(\"sum\").sort_values(ascending=False)[0:28].index)\ntest_keywords_set = set(test_df[col].unique())\nintersection_keyword = none_zero_keywords.intersection(test_keywords_set)\nprint(\"len:\",len(intersection_keyword))\nintersection_keyword","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f2af823f2a94147f13694e70cd35c8a81e2dfa5"},"cell_type":"code","source":"train_df.groupby(col)[target].agg(\"sum\").sort_values(ascending=False)[0:29].apply(lambda x: np.log1p(x)).plot(kind=\"bar\",figsize=(20,5),title=\"zero revenue of keyword\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"95641740f35b761f8e099ed90159e800af50cb11"},"cell_type":"code","source":"def add_keyword_feature(df):\n    col_name =\"filtered_keyword\"\n    sets = intersection_keyword.difference({'(automatic matching)',\n     '(not provided)',\n     '(remarketing/content targeting)'})\n    df[col_name] = df[col].apply(lambda x: x if x in sets else \"other\")\nadd_keyword_feature(train_df)\nadd_keyword_feature(test_df)\n# no improvement ,something wrong","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94e7ca5a3af6679d2398c12169bd748436769c2d"},"cell_type":"code","source":"none_zero_keywords.difference(test_keywords_set)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c7f26f7d2a48f6445b41a3780202a2f5d948fa2"},"cell_type":"code","source":"train_df.groupby(col)[target].agg(\"count\").sort_values(ascending=False)[0:40].apply(lambda x: np.log1p(x)).plot(kind=\"bar\",figsize=(20,5),title=\"count revenue of keyword\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a87084a10cd87ba7b6dab6d0d481afb4dfb3c4e"},"cell_type":"markdown","source":"# LGBM"},{"metadata":{"trusted":true,"_uuid":"1eff1cb188e4362e62ca1af3bcf47d52520a9221"},"cell_type":"code","source":"train_x = train_df[all_features]\ntrain_y = train_df[target]\ntest_x = test_df[all_features]\nfor col in category_features:\n    print(\"transform column {}\".format(col))\n    lbe = LabelEncoder()\n    lbe.fit(pd.concat([train_df[col],test_x[col]]).astype(\"str\"))\n    train_x[col] = lbe.transform(train_x[col].astype(\"str\"))\n    test_x[col] = lbe.transform(test_x[col].astype(\"str\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5dc50f674c3f37f33d64c92a989ea84b12f9db4"},"cell_type":"code","source":"def lgb_eval(num_leaves,max_depth,lambda_l2,lambda_l1,min_child_samples,bagging_fraction,feature_fraction):\n    params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\", \n    \"num_leaves\" : int(num_leaves),\n    \"max_depth\" : int(max_depth),\n    \"lambda_l2\" : lambda_l2,\n    \"lambda_l1\" : lambda_l1,\n    \"num_threads\" : 4,\n    \"min_child_samples\" : int(min_child_samples),\n    \"learning_rate\" : 0.03,\n    \"bagging_fraction\" : bagging_fraction,\n    \"feature_fraction\" : feature_fraction,\n    \"subsample_freq\" : 5,\n    \"bagging_seed\" : 42,\n    \"verbosity\" : -1\n    }\n    lgtrain = lgb.Dataset(train_x, label=np.log1p(train_y.apply(lambda x : 0 if x < 0 else x)),categorical_feature=category_features)\n    cv_result = lgb.cv(params,\n                       lgtrain,\n                       10000,\n                       categorical_feature=category_features,\n                       early_stopping_rounds=100,\n                       stratified=False,\n                       nfold=5)\n    return -cv_result['rmse-mean'][-1]\n\ndef lgb_train(num_leaves,max_depth,lambda_l2,lambda_l1,min_child_samples,bagging_fraction,feature_fraction):\n    params = {\n    \"objective\" : \"regression\",\n    \"metric\" : \"rmse\", \n    \"num_leaves\" : int(num_leaves),\n    \"max_depth\" : int(max_depth),\n    \"lambda_l2\" : lambda_l2,\n    \"lambda_l1\" : lambda_l1,\n    \"num_threads\" : 4,\n    \"min_child_samples\" : int(min_child_samples),\n    \"learning_rate\" : 0.01,\n    \"bagging_fraction\" : bagging_fraction,\n    \"feature_fraction\" : feature_fraction,\n    \"subsample_freq\" : 5,\n    \"bagging_seed\" : 42,\n    \"verbosity\" : -1\n    }\n    t_x,v_x,t_y,v_y = train_test_split(train_x,train_y,test_size=0.2)\n    lgtrain = lgb.Dataset(t_x, label=np.log1p(t_y.apply(lambda x : 0 if x < 0 else x)),categorical_feature=category_features)\n    lgvalid = lgb.Dataset(v_x, label=np.log1p(v_y.apply(lambda x : 0 if x < 0 else x)),categorical_feature=category_features)\n    model = lgb.train(params, lgtrain, 2000, valid_sets=[lgvalid], early_stopping_rounds=100, verbose_eval=100)\n    pred_test_y = model.predict(test_x, num_iteration=model.best_iteration)\n    return pred_test_y, model\n    \ndef param_tuning(init_points,num_iter,**args):\n    lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (25, 50),\n                                                'max_depth': (5, 15),\n                                                'lambda_l2': (0.0, 0.05),\n                                                'lambda_l1': (0.0, 0.05),\n                                                'bagging_fraction': (0.5, 0.8),\n                                                'feature_fraction': (0.5, 0.8),\n                                                'min_child_samples': (20, 50),\n                                                })\n\n    lgbBO.maximize(init_points=init_points, n_iter=num_iter,**args)\n    return lgbBO","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"469945a0d5a4dc077a5d1cf8755ed2f0c8d9fee9"},"cell_type":"code","source":"result = param_tuning(5,15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b829484e2038c8ff3b41da7edc3124a4a274f9dc"},"cell_type":"code","source":"result.res['max']['max_params']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53b102ac473dc12d02955765b4a1d4e7c584c3a0"},"cell_type":"code","source":"prediction1,model1 = lgb_train(**result.res['max']['max_params'])\nprediction2,model2 = lgb_train(**result.res['max']['max_params'])\nprediction3,model3 = lgb_train(**result.res['max']['max_params'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd26976f4b4caf64b7d3fc1aeab853a0163045a1"},"cell_type":"code","source":"# param = {'num_leaves': 45.61216380347129,\n#  'max_depth': 11.578579827303919,\n#  'lambda_l2': 0.0107663924764632,\n#  'lambda_l1': 0.046541310399201855,\n#  'bagging_fraction': 0.7851516324443661,\n#  'feature_fraction': 0.7944881085591733,\n#  'min_child_samples': 28.5601473698899}\n# prediction,model = lgb_train(**param)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbbde4af07e985a5074725e738142b9fa91797d3"},"cell_type":"code","source":"test_df['PredictedLogRevenue'] = (np.expm1(prediction1)+np.expm1(prediction2)+np.expm1(prediction3))/3\nsubmit = test_df[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum()['PredictedLogRevenue'].apply(np.log1p).fillna(0).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73640192fb9672ecaf5b296bd53f5cee09827bbc"},"cell_type":"code","source":"submit.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41142e9811daf67c73b94bd3397cb2e01c68694"},"cell_type":"code","source":"lgb.plot_importance(model, figsize=(15, 10),height=0.8)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59b3a7385aa18986a77d205ed5759dd0173cd90e"},"cell_type":"code","source":"lgb.plot_importance(model, figsize=(15, 10),height=0.8,importance_type=\"gain\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e067a9f1465306033f7c5999d4da615e86612985"},"cell_type":"markdown","source":"## If this kernel is useful to you, please give a vote. This is my first depth EDA kernel. Your vote is my inspiration."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}