{"cells":[{"metadata":{"trusted":true,"_uuid":"b86b726138ef0ff4b25cc855709a8ed6a39cd752"},"cell_type":"code","source":"from IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { width:75% !important; }</style>\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25a3053d3e979f8188b2cac627c76eba4fd499db"},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nfrom sklearn import preprocessing\ndef load_df(csv_path='./train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [\"{}.{}\".format(column, subcolumn) for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    df = df.loc[:, (df != df.iloc[0]).any()] \n    print(\"Loaded {}. Shape: {}\".format(os.path.basename(csv_path), df.shape))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f5a401349b1b95ed476c9a1f46cc00fdf9f5a23"},"cell_type":"code","source":"train = load_df('../input/train.csv')\ntrain = train.drop(['trafficSource.campaignCode', 'sessionId'], axis = 1) # Уберем столбец с одним не NaN значением и ID. транзакции\ntest = load_df('../input/test.csv')\ntest = test.drop(['sessionId'], axis = 1)\n\n# Преобразуем даты\ntrain['date'] = pd.to_datetime(train['date'], format = '%Y%m%d')\ntrain['visitStartTime_'] = pd.to_datetime(train['visitStartTime'], unit='s')\ntest['date'] = pd.to_datetime(test['date'], format = '%Y%m%d')\ntest['visitStartTime_'] = pd.to_datetime(test['visitStartTime'], unit='s')\n# Преобразуем тип целевой переменной\ntrain[\"totals.transactionRevenue\"] = train[\"totals.transactionRevenue\"].astype('float')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bfdf7c4422caaa7f9f0d31bc3a5c1815ed609e8c"},"cell_type":"markdown","source":"if bounds then revenue = 0"},{"metadata":{"trusted":true,"_uuid":"f7ec544c7297c1f8e46f50b8258e0c873375e9cf"},"cell_type":"code","source":"# добавим переменных\ntrain['hits_by_pageviews'] = train['totals.hits'].astype(float) / train['totals.pageviews'].astype(float)\ntest['hits_by_pageviews'] = test['totals.hits'].astype(float) / test['totals.pageviews'].astype(float)\n\ntrain['day_of_week'] = train['visitStartTime_'].dt.day_name()\ntest['day_of_week'] = test['visitStartTime_'].dt.day_name()\ntrain['hour'] = train['visitStartTime_'].dt.hour\ntest['hour'] = test['visitStartTime_'].dt.hour\ntrain['day'] = train['visitStartTime_'].dt.day\ntest['day'] = test['visitStartTime_'].dt.day","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e72871cd777f4f7d540c1dfe499e9e3dbabac84e"},"cell_type":"code","source":"train[\"totals.transactionRevenue\"].fillna(0, inplace=True)\ntrain_y = train[\"totals.transactionRevenue\"].values\ntrain_id = train[\"fullVisitorId\"].values\ntest_id = test[\"fullVisitorId\"].values\n\n\n# label encode the categorical variables and convert the numerical variables to float\ncat_cols = [\"channelGrouping\", \"device.browser\", \n            \"device.deviceCategory\", \"device.operatingSystem\", \n            \"geoNetwork.city\", \"geoNetwork.continent\", \n            \"geoNetwork.country\", \"geoNetwork.metro\",\n            \"geoNetwork.networkDomain\", \"geoNetwork.region\", \n            \"geoNetwork.subContinent\", \"trafficSource.adContent\", \n            \"trafficSource.adwordsClickInfo.adNetworkType\", \n            \"trafficSource.adwordsClickInfo.gclId\", \n            \"trafficSource.adwordsClickInfo.page\", \n            \"trafficSource.adwordsClickInfo.slot\", \"trafficSource.campaign\",\n            \"trafficSource.keyword\", \"trafficSource.medium\", \n            \"trafficSource.referralPath\", \"trafficSource.source\",\n            'trafficSource.adwordsClickInfo.isVideoAd', 'trafficSource.isTrueDirect',\n            'day_of_week', 'hour', 'day', 'device.isMobile']\nfor col in cat_cols:\n    print(col)\n    lbl = preprocessing.LabelEncoder()\n    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n    train[col] = lbl.transform(list(train[col].values.astype('str')))\n    test[col] = lbl.transform(list(test[col].values.astype('str')))\n\n\nnum_cols = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", 'totals.bounces',  'totals.newVisits', 'hits_by_pageviews', 'visitStartTime']    \nfor col in num_cols:\n    train[col] = train[col].astype(float)\n    test[col] = test[col].astype(float)\n\n\n    \n# Split the train dataset into development and valid based on time \ndev_df = train[train['date']<='2017-05-31']\nval_df = train[train['date']>'2017-05-31']\ndev_y = np.log1p(dev_df[\"totals.transactionRevenue\"].values)\nval_y = np.log1p(val_df[\"totals.transactionRevenue\"].values)\n\ndev_X = dev_df[cat_cols + num_cols] \nval_X = val_df[cat_cols + num_cols] \ntest_X = test[cat_cols + num_cols] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7de013eb0929f2f42c912a9f38d4643244ebcdaf"},"cell_type":"code","source":"import lightgbm as lgb\ndef run_lgb(train_X, train_y, val_X, val_y, test_X):\n    params = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\", \n        \"num_leaves\" : 30,\n        \"min_child_samples\" : 100,\n        \"learning_rate\" : 0.1,\n        \"bagging_fraction\" : 0.7,\n        \"feature_fraction\" : 0.5,\n        \"bagging_frequency\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1\n    }\n    \n    lgtrain = lgb.Dataset(train_X, label=train_y)\n    lgval = lgb.Dataset(val_X, label=val_y)\n    model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)\n    \n    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)\n    pred_val_y = model.predict(val_X, num_iteration=model.best_iteration)\n    return pred_test_y, model, pred_val_y\n\n# Training the model #\npred_test, model, pred_val = run_lgb(dev_X, dev_y, val_X, val_y, test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2c8c1c9328c1f2815356c82664648ed2f69ad7e"},"cell_type":"code","source":"from sklearn import metrics\npred_val[pred_val<0] = 0\nval_pred_df = pd.DataFrame({\"fullVisitorId\":val_df[\"fullVisitorId\"].values})\nval_pred_df[\"transactionRevenue\"] = val_df[\"totals.transactionRevenue\"].values\nval_pred_df[\"PredictedRevenue\"] = np.expm1(pred_val)\n#print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))\nval_pred_df = val_pred_df.groupby(\"fullVisitorId\")[\"transactionRevenue\", \"PredictedRevenue\"].sum().reset_index()\nprint(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e99df54482ca17b21d3bd499f34c710946ddb8d8"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"fullVisitorId\":test_id})\npred_test[pred_test<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(pred_test)\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"baseline_lgb.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed61664ac82ab65d0243ff6afc22bcd526c1921a"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig, ax = plt.subplots(figsize=(12,18))\nlgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax)\nax.grid(False)\nplt.title(\"LightGBM - Feature Importance\", fontsize=15)\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}