{"cells":[{"metadata":{"_uuid":"6541566306dc69139bcabe43e6d7a03dde307a62"},"cell_type":"markdown","source":"# Testing BaseLine Models\n\nThe notebook includes testing different base models on the dataset \n\n1. [Defining Models](#def_models)\n2. [Defining Training and Evaluation Methods](#t_evals)\n3. Models: <br>\n    3.1.   [Keras](#kr)<br>\n    3.2.   [ XGBoost ](#another_cell)<br>\n    3.3.  [LightGBM](#lgbm)<br>\n4. [Submission](#sub)<br>\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\nimport time\nimport pickle\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.ensemble import RandomForestClassifier\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation,Dropout, BatchNormalization\nfrom sklearn.model_selection import KFold\nimport xgboost\nfrom xgboost import plot_importance\nimport lightgbm as lgb\nfrom sklearn.model_selection import  train_test_split\nfrom sklearn.model_selection import GridSearchCV   #Performing grid search\nfrom sklearn.model_selection import validation_curve\nimport gc\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8eca02b0829a440d7f3298cc7b04c2c9b435cf6d"},"cell_type":"code","source":"#Loading Data \nfile_train = \"../input/train_v2.csv\"\nfile_test = \"../input/test_v2.csv\"\nchunk_size = 10000\n\ndef load_data(file,chunk_size,nrows_load=None,test_data=False):\n    df_res = pd.DataFrame()\n    df_reader = pd.read_csv(file,\n                            dtype={ 'date': str, 'fullVisitorId': str},\n                            chunksize=10000)\n    \n    for cidx, df in enumerate(df_reader):\n        df.reset_index(drop=True, inplace=True)   \n        process_df(df,test_data)\n        df_res = pd.concat([df_res,df ], axis=0).reset_index(drop=True)\n        del df #free memory\n        gc.collect()\n        #print every 20 iterations\n        if cidx % 20 == 0:\n            print('{}: rows loaded: {}'.format(cidx, df_res.shape[0]))\n        if nrows_load:\n            if res.shape[0] >= nrows_load:\n                break\n    return df_res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e43630a92700da4879e025838e98a154545a5937","scrolled":true},"cell_type":"code","source":"#every column as key and the important features to extract from each column\n\ndef parse_json(x,s):\n    res = json.loads(x)\n    try:\n        return res[s]\n    except:\n        return float('NaN') \n\ndef process_df(df,test_data):\n    #process date \n    df['days'] = df['date'].str[-2:]\n    df['days'] = df['days'].astype(int)\n    df['month'] = df['date'].str[-4:-2]\n    df['month'] = df['month'].astype(int)\n    df['year'] = df['date'].str[:4]\n    df['year'] = df['year'].astype(int)\n\n    #process json fields\n    process_dict = {\n        'totals':['transactionRevenue','newVisits','pageviews','hits'] ,\n        'trafficSource':['campaign','source','medium'] ,\n        'device':['browser'],\n        'geoNetwork': ['country','city','continent','region','subContinent']\n    }\n \n    #add new columns from json in df\n    for c,l in process_dict.items():\n        for it in l:\n            df[it] = df[c].apply(lambda x : parse_json(x,it))\n    \n    #process custom dimensions\n    #df['customDimensions_index'] = df['customDimensions'].apply(lambda x : parse_json(x,'index'))\n    #df['customDimensions_val'] = df['customDimensions'].apply(lambda x : parse_json(x,'value'))\n    \n    \n    #labelencoding for continuous data\n    cols = ['country','campaign','source','medium','continent','city','region','socialEngagementType','browser'\n             ,'channelGrouping','subContinent','date']\n    labelencoder_X=LabelEncoder()\n    for c in cols:\n        df.loc[:,c] = labelencoder_X.fit_transform(df.loc[:,c])\n        \n    \n    #Dealing with missing values\n    #transactionsRevenue and NewVisits:  nans ->  0\n    df['transactionRevenue'].fillna(0,inplace=True)\n    df['newVisits'].fillna(0,inplace=True)\n    df['pageviews'].fillna(0,inplace=True)\n    \n    \n    #Casting Str columns to int\n    df['transactionRevenue'] = df['transactionRevenue'].astype('float32')\n    df['newVisits']= df['newVisits'].astype('uint16')\n    df['pageviews'] = df['pageviews'].astype('uint16')\n    df['hits'] = df['hits'].astype('uint32')\n    #df['index'] = df['index'].astype('uint32')\n    \n    #remove json field columns and some unwanted columns\n    #(some removed for saving memory)\n    \n    rm_col = ['subContinent',\n             'channelGrouping','date','continent','customDimensions','fullVisitorId']\n    if test_data:\n        rm_col = rm_col[:-1]\n    df.drop(list(process_dict.keys()) + rm_col, axis=1,inplace=True)\n    \n#load and process\ndf = load_data(file_train,chunk_size)\ndf_test =load_data(file_test,chunk_size,test_data=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a49a3ad8bcbc09d9b527cce4fd938292f3e2e325"},"cell_type":"code","source":"#percent not zero \nper = sum(df['transactionRevenue'] > 0) / len(df['transactionRevenue'])\nprint('Percentage of transactions greater than 0 :   {} %'.format(per*100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7071cdeb4146097ec49cb8cfb11ebd886f64a0f0"},"cell_type":"code","source":"#Setting up Data\nX = df[df.columns[df.columns != 'transactionRevenue']]\nY = np.log1p(df['transactionRevenue'])\nX_train, X_val, y_train, y_val = train_test_split(X, Y, test_size=0.4, random_state=42)\n\nX_test = df_test[df_test.columns[df_test.columns != 'transactionRevenue']]\nY_test = np.log1p(df_test['transactionRevenue'])\n\n#Handling the imbalanced dataset and looking at how it effects training\n\n#Sampling \n#Using all the 1% non zero transactions revenue with an equal num of rows from the zero results\nindices_nonzero = np.where (y_train > 0)\nindices_zero = np.where (y_train == 0)\nnum_nonzero = len(indices_nonzero[0])\nnum_zero = len(indices_zero[0])\nprint('Number of non-zero transactions revenues = ', num_nonzero)\nprint('Number of zero transactions revenues = ', num_zero)\n\n#Creating a sample dataset containing 50% of rows with non-zero transactions \nall_indx = list(indices_zero[0][0:num_nonzero]) + list(indices_nonzero[0])\nX_sample = X_train.iloc[all_indx,:]\ny_sample = y_train.iloc[all_indx]\n#split train validation \nX_train_sample, X_val_sample, y_train_sample, y_val_sample = train_test_split(X_sample, y_sample, test_size=0.45, random_state=42)\nprint('Sample X_Train shape ', X_train_sample.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"515f4db426ecb7c8162ef25477daee154957f67c"},"cell_type":"markdown","source":"# Defining Models\n<a id='def_models'></a>"},{"metadata":{"trusted":true,"_uuid":"cbfe788f345a7e9eaed43b075dfc46c93bce3819"},"cell_type":"code","source":"def create_model_xgboost(X_train,y_train,X_val=None,y_val=None):\n    params = {'objective': 'reg:linear',\n              'eval_metric': 'rmse',\n              'min_child_weight':1.5,    \n              'eta': 0.01,\n              'max_depth': 4,\n              'subsample': 0.7,\n              'colsample_bytree': 0.6,\n              'reg_alpha':1,\n              'reg_lambda':0.45,\n              'random_state': 42,\n              'silent': True}\n    \n    xgb_train_data = xgboost.DMatrix(X_train, y_train)\n    if not X_val is None:\n        xgb_val_data = xgboost.DMatrix(X_val, y_val)\n        evals=[(xgb_train_data, 'train'), (xgb_val_data, 'valid')]\n    else:\n        evals=[(xgb_train_data, 'train')]\n    model = xgboost.train(params, xgb_train_data, \n                      num_boost_round=1200, \n                      evals= evals,\n                      early_stopping_rounds=50, \n                      verbose_eval=300) \n    return model\n\n#Model  definition\ndef create_model_nn(in_dim,layer_size=250):\n    model = Sequential()\n    model.add(Dense(layer_size,activation='relu',input_dim=in_dim))\n    model.add(BatchNormalization())\n    model.add(Dense(layer_size,activation='relu'))\n    model.add(Dense(layer_size,activation='relu'))\n    model.add(Dense(1, activation='linear'))\n    adam = optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)\n    model.compile(optimizer=adam,loss='mse')    \n    return model\n\n\ndef create_model_lgbm(X_train,y_train,X_val=None,y_val=None):\n    dtrain = lgb.Dataset(X_train,label=y_train)\n    dval = lgb.Dataset(X_val,label=y_val)\n\n    param_up = {\"objective\" : \"regression\", \"metric\" : \"rmse\", \n               \"max_depth\": 2, \"min_child_samples\": 20, \n               \"reg_alpha\": 1.5, \"reg_lambda\": 1.5,\n               \"num_leaves\" : 15, \"learning_rate\" : 0.1, \n               \"subsample\" : 1, \"colsample_bytree\" : 1, \n               \"verbosity\": -1,'data_random_seed':4}\n    if not X_val is None:\n        valid_sets = (dtrain,dval)\n        valid_names = ['train','valid']\n    else:\n        valid_sets = (dtrain)\n        valid_names = ['train']\n    model = lgb.train(param_up,dtrain,num_boost_round=5000,valid_sets=valid_sets,valid_names=['train','valid'],verbose_eval=300,\n                     early_stopping_rounds=50)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a4e77cacc7ae4539b48f1cd16464f6871d12e84"},"cell_type":"markdown","source":"# Defining Training and Evaluation Methods\n<a id='t_evals'></a>"},{"metadata":{"trusted":true,"_uuid":"a4f5dbbf2c5a4bd56fdedd7bfcf60ca9d42ba04c"},"cell_type":"code","source":"\n#Fitting and Training\ndef fit_train(x,y,X_val,y_val,layer_size=64,mod = 'nn'):    \n    if mod =='nn':\n        model = create_model_nn(x.shape[1],layer_size)\n        history = model.fit(x, y, epochs=3, batch_size=128,validation_data=(X_val,y_val),verbose=1)\n        print('Mean RMSE : ',np.sqrt(history.history['val_loss']).mean())\n    elif mod =='xgboost':\n         model = create_model_xgboost(x,y,X_val,y_val)\n    elif mod =='lgbm':\n        model = create_model_lgbm(x,y,X_val,y_val)\n    else:\n        raise Exception ('invalid model')\n    return model\n\n\ndef calc_rmse(pred,y):\n    diff =  pred - y\n    RMSE = ((diff ** 2).mean()) ** .5\n    print('RMSE : ',RMSE)\n\n#Evaluation\ndef eval_set(model,x,y,mod='nn'):\n        if mod == 'nn':\n            p = model.predict(x,batch_size=64,verbose=1)\n            pred = p[:,0]\n        elif mod == 'xgboost':\n            dx = xgboost.DMatrix(x)\n            pred = model.predict(dx,ntree_limit=model.best_ntree_limit)\n        elif mod == 'lgbm':\n            pred = model.predict(x, num_iteration=model.best_iteration)\n        else:\n            raise Exception ('invalid model')\n        calc_rmse(pred,y)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91d26e8fb73fd5ebf9c8bba9b594d5b436f01a26"},"cell_type":"markdown","source":"1. # Keras NN Sequential model\n<a id='kr'></a>"},{"metadata":{"trusted":true,"_uuid":"a90d7bb9730fa58626634787e08436951ccab8ad","scrolled":false},"cell_type":"code","source":"from keras import backend as K\nimport tensorflow as tf\nfrom keras import optimizers\nimport keras as k\n\n\n#Training\nprint('Training on training/val Dataset')\nmodel_training = fit_train(X_train,y_train,X_val,y_val)\nprint('-'*40)\nprint('Training on Sample Dataset')\nmodel_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500)\nprint('-'*40)\nprint('Training on Full training Dataset')\nmodel_full = create_model_nn(X.shape[1])\nmodel_full.fit(X, Y, epochs=3, batch_size=128,verbose=1)\n\nprint('-'*40)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49145bc20b422cf5fccf466487d567909524f313"},"cell_type":"markdown","source":" # XGBOOST Model \n<a id='another_cell'></a>"},{"metadata":{"trusted":true,"_uuid":"930c7997886bfd78bdf116cdf3a4cf294a01924f"},"cell_type":"code","source":"#Training\nprint('Training on training/val Dataset')\nmodel_training = fit_train(X_train,y_train,X_val,y_val,mod='xgboost')\nprint('-'*40)\nprint('Training on Sample Dataset')\nmodel_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500,mod='xgboost')\nprint('-'*40)\nprint('Training on Full training Dataset')\nmodel_full = create_model_xgboost(X, Y)\n\nprint('Full Training set -- Using Sample training model')\neval_set(model_sample,X_train,y_train,mod='xgboost')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"04629459fff30546b280d30dd3a3134e0b09848e"},"cell_type":"markdown","source":"# LightGBM\n<a id='lgbm'></a>"},{"metadata":{"trusted":true,"_uuid":"70b91ea76dd58d136f4c3426f7bf4a945682ff59","scrolled":true},"cell_type":"code","source":"#Training\nprint('Training on training/val Dataset')\nmodel_training = fit_train(X_train,y_train,X_val,y_val,mod='lgbm')\nprint('-'*40)\nprint('Training on Sample Dataset')\nmodel_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500,mod='lgbm')\nprint('-'*40)\nprint('Training on Full training Dataset')\nmodel_full = create_model_lgbm(X, Y)\n\nprint('Full Training set -- Using Sample training model')\neval_set(model_sample,X_train,y_train,mod='lgbm')\n\nprint('-'*40)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"654d9acd81fb7c10fe6649f6c90a39fe035851b5"},"cell_type":"markdown","source":"# Submission\n<a id='sub'></a>"},{"metadata":{"trusted":true,"_uuid":"a3757b95c1ab769caa3bd761627027574ce3b4d7"},"cell_type":"code","source":"#Baseline Predictions\ndf_test['predictions'] = model_full.predict(df_test.loc[:,df_test.columns[1:]],num_iteration=model_full.best_iteration)\ndf_test.loc[df_test['predictions']< 0 ,'predictions']= 0\n#set up dataframe for submission\nsub_df = pd.DataFrame({'fullVisitorId':df_test['fullVisitorId'] , 'PredictedLogRevenue': np.expm1(df_test['predictions'])})\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"baseline_lgb_submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}