{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os\nimport json\nimport xgboost as xgb\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nfrom sklearn import model_selection, preprocessing, metrics\nimport matplotlib.pylab as plt\nimport seaborn as sns\ncolor = sns.color_palette()\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.offline as offline\noffline.init_notebook_mode()\nimport plotly.tools as tls\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e08fe7f8ec7419ba476cd52664140ed6ae736a8c","trusted":true},"cell_type":"code","source":"train_df = load_df()\ntest_df = load_df(\"../input/test.csv\")\ntrain_df.shape,test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e458c9388056ac6f9336fa6aab2a1c7622aebc2","trusted":true},"cell_type":"code","source":"train_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].fillna(0).astype(float)\ntrain_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].apply(lambda x: np.log(x) if x > 0 else x)\n\ntrain_df.drop(['trafficSource.campaignCode'], axis =1,inplace = True)\ntrain_df.shape,test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a848f93eef6804db003238b51a7c9a3576331e53"},"cell_type":"code","source":"trn_len = train_df.shape[0]\ndf = pd.concat([train_df, test_df])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8264cf02ea355cedcd6f93c89a61afb12b59a624"},"cell_type":"code","source":"columns = [col for col in train_df.columns if train_df[col].nunique() > 1]\n\ndf = df[columns]\ndf.shape\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"547a078a8677ee6298da5227210e26754d65d7a6","trusted":true},"cell_type":"code","source":"import datetime\n\ndf['date'] = df['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\ndf['date'] = pd.to_datetime(df['date'])\ndf['month'] = df['date'].dt.month\ndf['dayofweek'] = df['date'].dt.dayofweek\ndf['weekday'] = df['date'].dt.weekday","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c4e6ee7d07b9bc675ab31d7bb05496bd94a951b","trusted":true},"cell_type":"code","source":"cat_cols = [\"channelGrouping\", \"device.browser\", \"device.deviceCategory\", \"device.operatingSystem\", \n            \"geoNetwork.city\", \"geoNetwork.continent\", \"geoNetwork.country\", \"geoNetwork.metro\",\n            \"geoNetwork.networkDomain\", \"geoNetwork.region\", \"geoNetwork.subContinent\", \"trafficSource.adContent\", \n            \"trafficSource.adwordsClickInfo.adNetworkType\", \"trafficSource.adwordsClickInfo.gclId\", \n            \"trafficSource.adwordsClickInfo.page\", \"trafficSource.adwordsClickInfo.slot\", \"trafficSource.campaign\",\n            \"trafficSource.keyword\", \"trafficSource.medium\", \"trafficSource.referralPath\", \"trafficSource.source\"]\n\nnum_cols = [\"totals.hits\", \"totals.pageviews\" ,\"month\" ,\"dayofweek\" ,\"weekday\"]\ntarget = \"totals.transactionRevenue\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fd229e94df0051a8cd6dfdb3f4002a2863b4870"},"cell_type":"code","source":"for col in cat_cols:\n    lbl = preprocessing.LabelEncoder()\n    df[col] = lbl.fit_transform(list(df[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79d5f2af74dcb66d267f328809f9830554fb4bed"},"cell_type":"code","source":"for col in num_cols:\n    df[col] = df[col].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4885f7e8120c6c9c0d43170f85131ada98cc38f5"},"cell_type":"code","source":"train_df = df[:trn_len]\ntest_df = df[trn_len:]\ntrain_df.shape,test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c108fad2391626c5d1cb7d0ff73454bc2ccc93e"},"cell_type":"code","source":"train_id = train_df[\"fullVisitorId\"].values\ntest_id = test_df[\"fullVisitorId\"].values","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"5ccffe85601c556200d3cc3c27f32339401e6e1f"},"cell_type":"code","source":"tra_df = train_df[train_df['date']<=datetime.date(2017,4,31)]\nval_df = train_df[train_df['date']>datetime.date(2017,4,31)]\n\ntra_y = tra_df[\"totals.transactionRevenue\"].values\nval_y = val_df[\"totals.transactionRevenue\"].values\n\ntra_X = tra_df[cat_cols + num_cols]\nval_X = val_df[cat_cols + num_cols]\ntest_X = test_df[cat_cols + num_cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2de028b138c7e91b0ba88742f349e0b9a4a47c7"},"cell_type":"code","source":"param = {'num_leaves':40,\n         'min_data_in_leaf': 15, \n         'objective':'regression',\n         'max_depth': 8,\n         'learning_rate':0.02,\n         \"min_child_samples\":10,\n        # \"boosting\":\"rf\",\n         \"feature_fraction\":0.8,\n         \"bagging_freq\":3,\n         \"bagging_fraction\":0.9 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 1,\n         \"verbosity\": -1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50e6873badcb315af2a4796629defdb4a494e768"},"cell_type":"code","source":"def run_lgb(params ,train_X, train_y, val_X, val_y, test_X):\n        \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    model = lgb.train(params, lgtrain, 5000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    return pred_test_y, model\n\n\npred_test, model = run_lgb(param,tra_X, tra_y, val_X, val_y, test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05a6f9b4d3e779a8e13e54dd8009cbaf8cd8dabb"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"fullVisitorId\":test_id})\npred_test[pred_test<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(pred_test)\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8a58b15633b426ff7aa264ad116ae9f5a844d78d"},"cell_type":"code","source":"features = cat_cols+num_cols\nfeature_importance_df = pd.DataFrame()\nfeature_importance_df[\"feature\"] = features\nfeature_importance_df[\"importance\"] = model.feature_importance()\n    \ncols = feature_importance_df[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n    by=\"importance\", ascending=False)[:1000].index\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,10))\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}