{"cells":[{"metadata":{"_uuid":"15486f9d4b1007112c01ea907938fa42ad57e2ba"},"cell_type":"markdown","source":"## Keras NN baseline\n\nIn this kernel I show a simple NN with categorical embeddings in Keras. If you want to read more about feature processing, please refer to my previous [kernel](https://www.kaggle.com/artgor/eda-on-basic-data-and-lgb-in-progress)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport json\nimport bq_helper\nfrom pandas.io.json import json_normalize\nimport seaborn as sns \nimport matplotlib.pyplot as plt\n%matplotlib inline\nplt.style.use('ggplot')\nimport lightgbm as lgb\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import TimeSeriesSplit, KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.models import Model, load_model\nfrom keras.layers import Input, Dropout, Dense, Embedding, SpatialDropout1D, concatenate, BatchNormalization, Flatten\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.losses import mean_squared_error as mse_loss\n\nfrom keras import optimizers\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"23cb99127e2aaebe64da839312507cebc1415952"},"cell_type":"markdown","source":"### Loading data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/julian3833/1-quick-start-read-csv-and-flatten-json-fields\n\ndef load_df(csv_path='../input/train.csv', JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']):\n\n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'})\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7b79f46f3c01f090fba491120a42f71faf873b0"},"cell_type":"code","source":"%%time\ntrain = load_df(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f2ae1015f4200773f29e6c68ed9f27cdc1f0588"},"cell_type":"code","source":"%%time\ntest = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"690255b6e749b7fd72e9bfba20fac6a1feec56e9"},"cell_type":"markdown","source":"### Processing data\n\nThere is a lot of processing, so I hide it by default, you can see it by opening the cells below."},{"metadata":{"trusted":true,"_uuid":"c9baa6c1df42dc9c89cfb97d56d150abff7d820e","_kg_hide-input":true},"cell_type":"code","source":"train['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\ntest['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\ntrain['trafficSource.isTrueDirect'].fillna(False, inplace=True)\ntest['trafficSource.isTrueDirect'].fillna(False, inplace=True)\n\ntrain['date'] = pd.to_datetime(train['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))\ntest['date'] = pd.to_datetime(test['date'].apply(lambda x: str(x)[:4] + '-' + str(x)[4:6] + '-' + str(x)[6:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fbcd43339a4ae57365bd46b2079583ce570715a","_kg_hide-input":true},"cell_type":"code","source":"cols_to_drop = [col for col in train.columns if train[col].nunique(dropna=False) == 1]\ntrain.drop(cols_to_drop, axis=1, inplace=True)\ntest.drop([col for col in cols_to_drop if col in test.columns], axis=1, inplace=True)\n\n#only one not null value\ntrain.drop(['trafficSource.campaignCode'], axis=1, inplace=True)\n\nfor col in ['visitNumber', 'totals.hits', 'totals.pageviews', 'totals.transactionRevenue']:\n    train[col] = train[col].astype(float)\n    \ntrain['totals.transactionRevenue'] = train['totals.transactionRevenue'].fillna(0)\ntrain['totals.transactionRevenue'] = np.log1p(train['totals.transactionRevenue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e84b536ab19676d25663fa2e087d7c0e770863dc","_kg_hide-input":true},"cell_type":"code","source":"train['totals.bounces'] = train['totals.bounces'].fillna(0)\ntrain['totals.newVisits'] = train['totals.newVisits'].fillna(0)\ntrain['totals.pageviews'] = train['totals.pageviews'].fillna(0)\ntrain['totals.transactionRevenue'] = train['totals.transactionRevenue'].fillna(0)\ntrain['trafficSource.adContent'] = train['trafficSource.adContent'].fillna(0)\ntrain['trafficSource.keyword'] = train['trafficSource.keyword'].fillna(0)\ntrain['trafficSource.adwordsClickInfo.adNetworkType'] = train['trafficSource.adwordsClickInfo.adNetworkType'].fillna(0)\ntrain['trafficSource.adwordsClickInfo.gclId'] = train['trafficSource.adwordsClickInfo.gclId'].fillna(0)\ntrain['trafficSource.adwordsClickInfo.page'] = train['trafficSource.adwordsClickInfo.page'].fillna(0)\ntrain['trafficSource.adwordsClickInfo.slot'] = train['trafficSource.adwordsClickInfo.slot'].fillna(0)\n\ntest['totals.bounces'] = test['totals.bounces'].fillna(0)\ntest['totals.newVisits'] = test['totals.newVisits'].fillna(0)\ntest['totals.pageviews'] = test['totals.pageviews'].fillna(0)\ntest['trafficSource.adContent'] = test['trafficSource.adContent'].fillna(0)\ntest['trafficSource.keyword'] = test['trafficSource.keyword'].fillna(0)\ntest['trafficSource.adwordsClickInfo.adNetworkType'] = test['trafficSource.adwordsClickInfo.adNetworkType'].fillna(0)\ntest['trafficSource.adwordsClickInfo.gclId'] = test['trafficSource.adwordsClickInfo.gclId'].fillna(0)\ntest['trafficSource.adwordsClickInfo.page'] = test['trafficSource.adwordsClickInfo.page'].fillna(0)\ntest['trafficSource.adwordsClickInfo.slot'] = test['trafficSource.adwordsClickInfo.slot'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"557b9e5641a68d6d137ad01a9aa343044305250e","_kg_hide-input":true},"cell_type":"code","source":"train['month'] = train['date'].dt.month\ntrain['day'] = train['date'].dt.day\ntrain['weekday'] = train['date'].dt.weekday\ntrain['weekofyear'] = train['date'].dt.weekofyear\n\ntest['month'] = test['date'].dt.month\ntest['day'] = test['date'].dt.day\ntest['weekday'] = test['date'].dt.weekday\ntest['weekofyear'] = test['date'].dt.weekofyear\n\ntrain['browser_category'] = train['device.browser'] + '_' + train['device.deviceCategory']\ntrain['browser_operatingSystem'] = train['device.browser'] + '_' + train['device.operatingSystem']\n\ntest['browser_category'] = test['device.browser'] + '_' + test['device.deviceCategory']\ntest['browser_operatingSystem'] = test['device.browser'] + '_' + test['device.operatingSystem']\n\ntrain['source_country'] = train['trafficSource.source'] + '_' + train['geoNetwork.country']\ntest['source_country'] = test['trafficSource.source'] + '_' + test['geoNetwork.country']\n                                                                   \ntrain['visitNumber'] = np.log1p(train['visitNumber'])\ntest['visitNumber'] = np.log1p(test['visitNumber'])\n\ntrain['totals.hits'] = np.log1p(train['totals.hits'])\ntest['totals.hits'] = np.log1p(test['totals.hits'].astype(int))\n\ntrain['totals.pageviews'] = np.log1p(train['totals.pageviews'].fillna(0))\ntest['totals.pageviews'] = np.log1p(test['totals.pageviews'].astype(float).fillna(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"970e94b09c07b1a448a9a712e326863af3dc2433"},"cell_type":"code","source":"num_cols = ['visitNumber', 'totals.hits', 'totals.pageviews', 'month_unique_user_count', 'day_unique_user_count', 'mean_hits_per_day'\n           'sum_pageviews_per_network_domain', 'sum_hits_per_network_domain', 'count_hits_per_network_domain', 'sum_hits_per_region',\n           'sum_hits_per_day', 'count_pageviews_per_network_domain', 'mean_pageviews_per_network_domain', 'weekday_unique_user_count',\n           'sum_pageviews_per_region', 'count_pageviews_per_region', 'mean_pageviews_per_region', 'user_pageviews_count', 'user_hits_count',\n           'count_hits_per_region', 'mean_hits_per_region', 'user_pageviews_sum', 'user_hits_sum', 'user_pageviews_sum_to_mean',\n            'user_hits_sum_to_mean', 'user_pageviews_to_region', 'user_hits_to_region', 'mean_pageviews_per_network_domain',\n           'mean_hits_per_network_domain','totals.bounces', 'totals.newVisits']\nnum_cols = [col for col in num_cols if col in train.columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0814f33f0a2b7e9d9a6df3b5a5fa1eb6d121b014"},"cell_type":"code","source":"no_use = [\"date\", \"fullVisitorId\", \"sessionId\", \"visitId\", \"visitStartTime\", 'totals.transactionRevenue', 'trafficSource.referralPath']\ncat_cols = [col for col in train.columns if col not in num_cols and col not in no_use]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65f0549598cd09a8e7d8e94d4026240d73e26f62","scrolled":true},"cell_type":"code","source":"max_values = {}\nfor col in cat_cols:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n    train[col] = lbl.transform(list(train[col].values.astype('str')))\n    test[col] = lbl.transform(list(test[col].values.astype('str')))\n    max_values[col] = max(train[col].max(), test[col].max())  + 2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"824d2d3ff1670ed976aa3cdd1f573b8f00f7d691"},"cell_type":"markdown","source":"### Neural net\n\nThe main idea which allows us to efficiently use neural nets for tabular data is categorical embeddings. Basically this means using encodings for categorical features like we do it for text data.\n\nYou can read more in this fastai [article](http://www.fast.ai/2018/04/29/categorical-embeddings/) or in this [one](https://medium.com/@satnalikamayank12/on-learning-embeddings-for-categorical-data-using-keras-165ff2773fc9) with an example of Keras implentation.\n\nTo encode variables we need two numbers:\n- length - it will be equal to cardinality + 2 (to be slightly higher than it);\n- dimensionality - it is usually calculated as `(cardinality + 1) // 2` and usually is capped at 50 or we could have huge values;"},{"metadata":{"trusted":true,"_uuid":"0e4f7873a407d04913e21b9ae9788066d4f46849","_kg_hide-output":true},"cell_type":"code","source":"# printing because I'm too lazy to write everything by hand. Open output to see.\nfor col in cat_cols:\n    n = col.replace('.', '_')\n    print(f'{n} = Input(shape=[1], name=\"{col}\")')\n    print(f'emb_{n} = Embedding({max_values[col]}, {(np.min(max_values[col]+1)//2, 50)})({col})')\n    print(',', n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cec50542da500384c688bfb9f9b1305e92bcb251"},"cell_type":"markdown","source":"#### Model definition"},{"metadata":{"trusted":true,"_uuid":"8cccf1b8244f42fee8c0b5798bedc2f522dfa973","scrolled":true,"_kg_hide-input":true},"cell_type":"code","source":"def model(dense_dim_1=128, dense_dim_2=64, dense_dim_3=32, dense_dim_4=16, \ndropout1=0.2, dropout2=0.1, dropout3=0.1, dropout4=0.1, lr=0.0001):\n\n    #Inputs\n    channelGrouping = Input(shape=[1], name=\"channelGrouping\")\n    device_browser = Input(shape=[1], name=\"device.browser\")\n    device_deviceCategory = Input(shape=[1], name=\"device.deviceCategory\")\n    device_operatingSystem = Input(shape=[1], name=\"device.operatingSystem\")\n    day = Input(shape=[1], name=\"day\")\n    geoNetwork_city = Input(shape=[1], name=\"geoNetwork.city\")\n    geoNetwork_continent = Input(shape=[1], name=\"geoNetwork.continent\")\n    geoNetwork_country = Input(shape=[1], name=\"geoNetwork.country\")\n    geoNetwork_metro = Input(shape=[1], name=\"geoNetwork.metro\")\n    geoNetwork_networkDomain = Input(shape=[1], name=\"geoNetwork.networkDomain\")\n    geoNetwork_region = Input(shape=[1], name=\"geoNetwork.region\")\n    geoNetwork_subContinent = Input(shape=[1], name=\"geoNetwork.subContinent\")\n    trafficSource_adContent = Input(shape=[1], name=\"trafficSource.adContent\")\n    trafficSource_adwordsClickInfo_adNetworkType = Input(shape=[1], name=\"trafficSource.adwordsClickInfo.adNetworkType\")\n    trafficSource_adwordsClickInfo_gclId = Input(shape=[1], name=\"trafficSource.adwordsClickInfo.gclId\")\n    trafficSource_adwordsClickInfo_isVideoAd = Input(shape=[1], name=\"trafficSource.adwordsClickInfo.isVideoAd\")\n    trafficSource_adwordsClickInfo_page = Input(shape=[1], name=\"trafficSource.adwordsClickInfo.page\")\n    trafficSource_adwordsClickInfo_slot = Input(shape=[1], name=\"trafficSource.adwordsClickInfo.slot\")\n    trafficSource_campaign = Input(shape=[1], name=\"trafficSource.campaign\")\n    trafficSource_keyword = Input(shape=[1], name=\"trafficSource.keyword\")\n    trafficSource_medium = Input(shape=[1], name=\"trafficSource.medium\")\n    trafficSource_source = Input(shape=[1], name=\"trafficSource.source\")\n    month = Input(shape=[1], name=\"month\")\n    weekday = Input(shape=[1], name=\"weekday\")\n    weekofyear = Input(shape=[1], name=\"weekofyear\")\n    browser_category = Input(shape=[1], name=\"browser_category\")\n    browser_operatingSystem = Input(shape=[1], name=\"browser_operatingSystem\")\n    source_country = Input(shape=[1], name=\"source_country\")\n\n    totals_pageviews = Input(shape=[1], name=\"totals.pageviews\")\n    totals_hits = Input(shape=[1], name=\"totals.hits\")\n    visitNumber = Input(shape=[1], name=\"visitNumber\")\n    \n    \n    #Embeddings layers\n\n    emb_channelGrouping = Embedding(9, 5)(channelGrouping)\n    emb_device_browser = Embedding(130, 50)(device_browser)\n    emb_device_deviceCategory = Embedding(4, 3)(device_deviceCategory)\n    emb_device_operatingSystem = Embedding(25, 13)(device_operatingSystem)\n    emb_day = Embedding(32, 16)(day)\n    emb_geoNetwork_city = Embedding(957, 50)(geoNetwork_city)\n    emb_geoNetwork_continent = Embedding(7, 4)(geoNetwork_continent)\n    emb_geoNetwork_country = Embedding(229, 50)(geoNetwork_country)\n    emb_geoNetwork_metro = Embedding(124, 50)(geoNetwork_metro)\n    emb_geoNetwork_networkDomain = Embedding(41983, 50)(geoNetwork_networkDomain)\n    emb_geoNetwork_region = Embedding(484, 50)(geoNetwork_region)\n    emb_geoNetwork_subContinent = Embedding(24, 12)(geoNetwork_subContinent)\n    emb_trafficSource_adContent = Embedding(78, 39)(trafficSource_adContent)\n    emb_trafficSource_adwordsClickInfo_adNetworkType = Embedding(5, 3)(trafficSource_adwordsClickInfo_adNetworkType)\n    emb_trafficSource_adwordsClickInfo_gclId = Embedding(59010, 50)(trafficSource_adwordsClickInfo_gclId)\n    emb_trafficSource_adwordsClickInfo_isVideoAd = Embedding(3, 3)(trafficSource_adwordsClickInfo_isVideoAd)\n    emb_trafficSource_adwordsClickInfo_page = Embedding(13, 7)(trafficSource_adwordsClickInfo_page)\n    emb_trafficSource_adwordsClickInfo_slot = Embedding(5, 3)(trafficSource_adwordsClickInfo_slot)\n    emb_trafficSource_campaign = Embedding(36, 18)(trafficSource_campaign)\n    emb_trafficSource_keyword = Embedding(5394, 50)(trafficSource_keyword)\n    emb_trafficSource_medium = Embedding(8, 4)(trafficSource_medium)\n    emb_trafficSource_source = Embedding(501, 50)(trafficSource_source)\n    emb_month = Embedding(13, 7)(month)\n    emb_weekday = Embedding(8, 4)(weekday)\n    emb_weekofyear = Embedding(53, 27)(weekofyear)\n    emb_browser_category = Embedding(175, 50)(browser_category)\n    emb_browser_operatingSystem = Embedding(215, 50)(browser_operatingSystem)\n    emb_source_country = Embedding(4480, 50)(source_country)\n\n    concat_emb1 = concatenate([\n           Flatten() (emb_channelGrouping),\n            Flatten() (emb_device_deviceCategory),\n            Flatten() (emb_device_operatingSystem),\n            Flatten() (emb_day),\n            Flatten() (emb_geoNetwork_continent),\n            Flatten() (emb_geoNetwork_subContinent),\n            Flatten() (emb_trafficSource_adContent),\n            Flatten() (emb_trafficSource_adwordsClickInfo_adNetworkType),\n            Flatten() (emb_trafficSource_adwordsClickInfo_isVideoAd),\n            Flatten() (emb_trafficSource_adwordsClickInfo_page),\n            Flatten() (emb_trafficSource_adwordsClickInfo_slot),\n            Flatten() (emb_trafficSource_campaign),\n            Flatten() (emb_trafficSource_medium),\n            Flatten() (emb_month),\n            Flatten() (emb_weekday),\n            Flatten() (emb_weekofyear),\n            Flatten() (emb_geoNetwork_region)\n    ])\n    \n    categ = Dropout(dropout1)(Dense(dense_dim_1,activation='relu') (concat_emb1))\n    categ = BatchNormalization()(categ)\n    categ = Dropout(dropout2)(Dense(dense_dim_2,activation='relu') (categ))\n    \n    concat_emb2 = concatenate([\n           Flatten() (emb_browser_category), \n            Flatten() (emb_browser_operatingSystem), \n            Flatten() (emb_source_country), \n            Flatten() (emb_device_browser), \n            Flatten() (emb_geoNetwork_city), \n            Flatten() (emb_trafficSource_source), \n            Flatten() (emb_trafficSource_keyword), \n            Flatten() (emb_trafficSource_adwordsClickInfo_gclId), \n            Flatten() (emb_geoNetwork_networkDomain), \n            Flatten() (emb_geoNetwork_country), \n            Flatten() (emb_geoNetwork_metro), \n            Flatten() (emb_geoNetwork_region)\n    ])\n    categ2 = Dropout(dropout1)(Dense(dense_dim_1* 2,activation='relu') (concat_emb2))\n    categ2 = BatchNormalization()(categ2)\n    categ2 = Dropout(dropout2)(Dense(dense_dim_2* 2,activation='relu') (categ2))\n    \n    #main layer\n    main_l = concatenate([\n          categ\n        , categ2\n        , totals_pageviews\n        , totals_hits\n        , visitNumber\n    ])\n    \n    main_l = Dropout(dropout3)(Dense(dense_dim_3,activation='relu') (main_l))\n    main_l = BatchNormalization()(main_l)\n    main_l = Dropout(dropout4)(Dense(dense_dim_4,activation='relu') (main_l))\n    \n    #output\n    output = Dense(1) (main_l)\n\n    model = Model([channelGrouping,\n                   device_browser,\n                   device_deviceCategory,\n                   device_operatingSystem,\n                   geoNetwork_city,\n                   geoNetwork_continent,\n                   geoNetwork_country,\n                   geoNetwork_metro,\n                   geoNetwork_networkDomain,\n                   geoNetwork_region,\n                   geoNetwork_subContinent,\n                   trafficSource_adContent,\n                   trafficSource_adwordsClickInfo_adNetworkType,\n                   trafficSource_adwordsClickInfo_gclId,\n                   trafficSource_adwordsClickInfo_isVideoAd,\n                   trafficSource_adwordsClickInfo_page,\n                   trafficSource_adwordsClickInfo_slot,\n                   trafficSource_campaign,\n                   trafficSource_keyword,\n                   trafficSource_medium,\n                   trafficSource_source,\n                   month,\n                   day,\n                   weekday,\n                   weekofyear,\n                   browser_category,\n                   browser_operatingSystem,\n                   source_country,\n                   totals_pageviews, totals_hits, visitNumber], output)\n    #model = Model([**params], output)\n    model.compile(optimizer = Adam(lr=lr),\n                  loss= mse_loss,\n                  metrics=[root_mean_squared_error])\n    return model\n\ndef root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))\n#model=model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8acc6670adac6a70fa42480cd637b351bce5a202"},"cell_type":"code","source":"train = train.sort_values('date')\nX = train.drop(no_use, axis=1)\ny = train['totals.transactionRevenue']\nX_test = test.drop([col for col in no_use if col in test.columns], axis=1)\nn_fold = 10\nfolds = KFold(n_splits=n_fold, shuffle=False, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b07433e92a119c5efb2a8633c04b624848ffd017"},"cell_type":"code","source":"# converting data to format which can be used by Keras\ndef get_keras_data(df, num_cols, cat_cols):\n    cols = num_cols + cat_cols\n\n    X = {col: np.array(df[col]) for col in cols}\n    # print(\"Data ready for Vectorization\")\n    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ecbdc1fbeca9c74ea3da09a9887b9dffff02372"},"cell_type":"code","source":"X_test_keras = get_keras_data(X_test, num_cols, cat_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d0c1f6c21534d149712d51a063b9eff669fbfca"},"cell_type":"code","source":"def train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, reduce_lr=False, patience=3):\n    \"\"\"\n    Helper function to train model. Also I noticed that ReduceLROnPlateau is rarely\n    useful, so added an option to turn it off.\n    \"\"\"\n    \n    early_stopping = EarlyStopping(patience=patience, verbose=1)\n    model_checkpoint = ModelCheckpoint(\"model.hdf5\",\n                                       save_best_only=True, verbose=1, monitor='val_root_mean_squared_error', mode='min')\n    if reduce_lr:\n        reduce_lr = ReduceLROnPlateau(factor=0.1, patience=2, min_lr=0.000005, verbose=1)\n        hist = keras_model.fit(X_t, y_train, batch_size=batch_size, epochs=epochs,\n                            validation_data=(X_v, y_valid), verbose=False,\n                            callbacks=[early_stopping, model_checkpoint, reduce_lr])\n    \n    else:\n        hist = keras_model.fit(X_t, y_train, batch_size=batch_size, epochs=epochs,\n                            validation_data=(X_v, y_valid), verbose=False,\n                            callbacks=[early_stopping, model_checkpoint])\n\n    keras_model = load_model(\"model.hdf5\", custom_objects={'root_mean_squared_error': root_mean_squared_error})\n    \n    return keras_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b43be59c19d85408f1ff54383e1738a87e107b86"},"cell_type":"code","source":"def root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"df56625c6634e5164e6bdf04d7b01bc114ced741"},"cell_type":"code","source":"oof = np.zeros(len(train))\npredictions = np.zeros(len(test))\nbatch_size = 2 ** 10\nepochs = 100\nscores = []\nfor fold_n, (train_index, valid_index) in enumerate(folds.split(X)):\n    print('Fold:', fold_n)\n    X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]\n    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n    X_t = get_keras_data(X_train, num_cols, cat_cols)\n    X_v = get_keras_data(X_valid, num_cols, cat_cols)\n    \n    keras_model = model(dense_dim_1=64, dense_dim_2=64, dense_dim_3=32, dense_dim_4=8, \n                        dropout1=0.05, dropout2=0.1, dropout3=0.1, dropout4=0.05, lr=0.0001)\n    mod = train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, reduce_lr=False, patience=3)\n    oof[valid_index] = mod.predict(X_v).reshape(-1,)\n    \n    y_pred = mod.predict(X_test_keras).reshape(-1,)\n    predictions += y_pred\n    \n    y_pred_valid = mod.predict(X_v)\n    scores.append(mean_squared_error(y_valid, y_pred_valid) ** 0.5)\n    print('Validation score: {}.'.format(scores[-1]))\n    print('*'* 50)\n\npredictions /= n_fold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f50fba283c145228cd74f523a476b14975f7eab"},"cell_type":"code","source":"3.7187107900597733 ** 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b5ff927b8abd953fbdb8c17fc96a80ef6ee9af6"},"cell_type":"code","source":"print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f4a108fd1ca13fe9c7532a1e343ba3b9418c838"},"cell_type":"code","source":"submission = test[['fullVisitorId']].copy()\nsubmission.loc[:, 'PredictedLogRevenue'] = predictions\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].apply(lambda x : 0.0 if x < 0 else x)\nsubmission[\"PredictedLogRevenue\"] = submission[\"PredictedLogRevenue\"].fillna(0.0)\ngrouped_test = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId').sum().reset_index()\ngrouped_test.to_csv(f'nn_cv{np.mean(scores):.4f}_std_{np.std(scores):.4f}_prediction.csv', index=False)\noof_df = pd.DataFrame({\"fullVisitorId\": train[\"fullVisitorId\"], \"PredictedLogRevenue\": oof})\noof_df.to_csv(f'nn_cv{np.mean(scores):.4f}_std_{np.std(scores):.4f}_oof.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0987f7ba8b2f1b5aa6e5b287c212cf100080e268"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb1806bd03a8e2c9f32d736cc4baa03ca4e842cc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2ce40fcb5fb837443b4c987425ecd1a8a4866a2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fac31cc189640419743978a74ac16d84a6728659"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}