{"cells":[{"metadata":{"trusted":true,"_uuid":"8c27d06e85772133ade81e28478c6f3f75ff60bd"},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nfrom pandas.io.json import json_normalize\nimport json\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df=load_df('../input/train.csv')\ntest_df=load_df('../input/test.csv')\nprint(train_df.head(),test_df.head())\ntrain_df_copy=train_df\ntest_df_copy=test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92ecebe12623ba2ca427c5007a17945f8cb9fdab"},"cell_type":"code","source":"train_df=train_df_copy\ntest_df=test_df_copy\ndrop_column=[]\nfor col in train_df.columns:\n    if len(train_df[col].unique())==1:\n        drop_column.append(col)\nfor col in test_df.columns:\n    if len(test_df[col].unique())==1:\n        if col not in drop_column:\n            drop_column.append(col)\nprint(drop_column)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5e566420353cd1daa8446a03dec2fe9b521c5a3"},"cell_type":"code","source":"train_df=train_df.drop(drop_column,axis=1)\ntest_df=test_df.drop(drop_column,axis=1)\nprint(train_df.shape,test_df.shape)\n#print(train_df.isna().any())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d33195b0714b8c4e09b2f4df7461317ce25b8234"},"cell_type":"code","source":"print([col for col in train_df.columns if col not in test_df.columns])\n#'trafficSource.campaignCode' column of train data is not test data so  drop it \ntrain_df=train_df.drop('trafficSource.campaignCode',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d0d9be387d5a3733e6772d46f590d707a84b8e6"},"cell_type":"code","source":"train_y=train_df['totals.transactionRevenue'].values\ntrain_id=train_df['fullVisitorId'].values\ntest_id=test_df['fullVisitorId'].values\n# Drop totals.transactionRevenue,fullVisitorId and sessionId(as it is unique for every session)\ntrain_df=train_df.drop(['totals.transactionRevenue','sessionId'],axis=1)\ntest_df=test_df.drop(['sessionId'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89140976062991ba204b7e5f96ee6655a1f0ced3"},"cell_type":"code","source":"for col in train_df:\n    print(col,(train_df[col].isnull().sum()*100)/len(train_df[col]),train_df[col].dtype,len(test_df[col].unique()))\n    if (train_df[col].isnull().sum()*100)/len(train_df[col])>65:\n        train_df=train_df.drop(col,axis=1)\n        test_df=test_df.drop(col,axis=1)\n#train_df.isnull().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83d38bf628db0d3668cd043dbaeb041a260c02d4"},"cell_type":"code","source":"cat_data_columns=[]\nfor col in train_df.columns:\n    if train_df[col].dtype=='object':\n        cat_data_columns.append(col)\n#print(cat_data_columns)\nfrom sklearn.preprocessing import LabelEncoder\nfor col in cat_data_columns:\n    lblen=LabelEncoder()\n    lblen.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col]=lblen.transform(train_df[col].values.astype('str'))\n    test_df[col]=lblen.transform(test_df[col].values.astype('str'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"381706006a0fe7775ddf84f455bd9a87351b0f3c"},"cell_type":"code","source":"train_df_copy[\"totals.transactionRevenue\"].fillna(0, inplace=True)\nprint(train_df_copy[\"totals.transactionRevenue\"].isna().any())\ntrain_y=train_df_copy['totals.transactionRevenue'].values\nprint(train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b240df27097d4c057e48891e127dc06618759eb7"},"cell_type":"code","source":"#train_df=train_df.drop('totals.transactionRevenue',axis=1)\ntrain_df.insert(0,\"totals.transactionRevenue\",train_y)\ntrain_df[\"totals.transactionRevenue\"] = train_df[\"totals.transactionRevenue\"].astype('float')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3345a0a337f53f2c11a901637d9fd85d80359e7"},"cell_type":"code","source":"#print(train_df['date'])\ndev_df = train_df\nval_df = train_df[train_df['date']>20170531]\ntrain_y = np.log1p(dev_df[\"totals.transactionRevenue\"].values.astype(float))\nval_y = np.log1p(val_df[\"totals.transactionRevenue\"].values.astype(float))\n\ndev_df_x=dev_df.drop(['totals.transactionRevenue','fullVisitorId'],axis=1)\nval_df_x=val_df.drop(['totals.transactionRevenue','fullVisitorId'],axis=1)\ntrain_x=dev_df_x\nval_x=val_df_x\ntest_x=test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"195a67e1c84651830296ed4ae7a4482b8da57f3d"},"cell_type":"code","source":"train_x.info()\nval_x.info()\nprint(len(train_y))\nlen(val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"090c76f90fd04761765ae933983ceebc824aa062"},"cell_type":"code","source":"from sklearn import model_selection, preprocessing, metrics\nimport lightgbm as lgb\ntrain_set=lgb.Dataset(train_x, label=train_y)\nval_set=lgb.Dataset(val_x,label=val_y)\nparams = {\n        \"objective\" : \"regression\",\n        \"metric\" : \"rmse\", \n        \"num_leaves\" : 30,\n        \"min_child_samples\" : 100,\n        \"learning_rate\" : 0.1,\n        \"bagging_fraction\" : 0.7,\n        \"feature_fraction\" : 0.5,\n        \"bagging_frequency\" : 5,\n        \"bagging_seed\" : 2018,\n        \"verbosity\" : -1\n    }\nlgb_model=lgb.train(params,train_set,500,valid_sets=[val_set],early_stopping_rounds=100, verbose_eval=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6848bc3ad07a772d527277601a45e8ba7eb35c5d"},"cell_type":"code","source":"pred_test = lgb_model.predict(test_x, num_iteration=lgb_model.best_iteration)\npred_val = lgb_model.predict(val_x, num_iteration=lgb_model.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d35041f59e32520a6bcb4d2913485e0c64a0f9e"},"cell_type":"code","source":"#from sklearn import metrics\npred_val[pred_val<0] = 0\nval_pred_df = pd.DataFrame({\"fullVisitorId\":val_df[\"fullVisitorId\"].values})\nval_pred_df[\"transactionRevenue\"] = val_df[\"totals.transactionRevenue\"].values\nval_pred_df[\"PredictedRevenue\"] = np.expm1(pred_val)\n\n#print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df[\"transactionRevenue\"].values), np.log1p(val_pred_df[\"PredictedRevenue\"].values))))\nval_pred_df = val_pred_df.groupby(\"fullVisitorId\")[\"transactionRevenue\",\"PredictedRevenue\"].sum().reset_index()\n#val_pred_df[\"PredictedRevenue\"] = val_pred_df.groupby(\"fullVisitorId\")[\"PredictedRevenue\"].sum().reset_index()\nprint(val_pred_df.head())\nval_pred_col=np.log1p(val_pred_df[\"transactionRevenue\"].values)\nval_actual_col=np.log1p(val_pred_df[\"PredictedRevenue\"].values)\nprint(np.sqrt(metrics.mean_squared_error(val_pred_col,val_actual_col)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"faa917c9c921ffd61baa8c147ff6250266ca2428"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"fullVisitorId\":test_id})\npred_test[pred_test<0] = 0\nsub_df[\"PredictedLogRevenue\"] = np.expm1(pred_test)\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"result_lgb_full.csv\", index=False)\nsub_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}