{"cells":[{"metadata":{},"cell_type":"markdown","source":"# TensorFlow for tabular data (with categorical variables)\n\nThis notebook applies TensorFlow / Keras tecniques for tabular data as described in:\n\n* https://github.com/lmassaron/deep_learning_for_tabular_data\n* https://www.kaggle.com/lucamassaron/deep-learning-for-tabular-data\n\nand as presented at various meetups and Google DevFests:\n* https://www.youtube.com/watch?v=nQgUt_uADSE\n\nIn particular, in the code you will find different data pipelines drafted for binary, high cardinality nominal, low cardinality nominal, ordinal, dates.\n\nThe code is fully commented for you to explore and experiment, and, more important, it will be regularly updated during the competition with furthermore feature creation and more performing neural architectures."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing core libraries\nimport numpy as np\nimport pandas as pd\nfrom time import time\nimport pprint\nimport joblib\n\n# Model selection\nfrom sklearn.model_selection import StratifiedKFold\n\n# Metrics\nfrom sklearn.metrics import roc_auc_score, average_precision_score\nfrom sklearn.metrics import make_scorer\n\n# Data transformation pipelines\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import RobustScaler, StandardScaler\n\n# Graphics\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TensorFlow \nimport tensorflow as tf\nfrom keras import backend as K\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.optimizers import Adam, Nadam\nfrom keras.layers import Input, Embedding, Reshape, GlobalAveragePooling1D\nfrom keras.layers import Flatten, concatenate, Concatenate, Lambda, Dropout, SpatialDropout1D\nfrom keras.layers import Reshape, MaxPooling1D,BatchNormalization, AveragePooling1D, Conv1D\nfrom keras.layers import Activation, LeakyReLU\nfrom keras.optimizers import SGD, Adam, Nadam\nfrom keras.models import Model, load_model\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom keras.regularizers import l2, l1_l2\nfrom keras.losses import binary_crossentropy\nfrom keras.utils import get_custom_objects\nfrom keras.layers import Activation, LeakyReLU\nfrom keras.models import load_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading the data\nX = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\")\nXt = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Separating target and ids\ny = X.target.values\nid_train = X.id\nid_test = Xt.id\n\nX.drop(['id', 'target'], axis=1, inplace=True)\nXt.drop(['id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Classifying variables in binary, high and low cardinality nominal, ordinal and dates\nbinary_vars = [c for c in X.columns if 'bin_' in c]\n\nnominal_vars = [c for c in X.columns if 'nom_' in c]\nhigh_cardinality = [c for c in nominal_vars if len(X[c].unique()) > 16]\nlow_cardinality = [c for c in nominal_vars if len(X[c].unique()) <= 16]\n\nordinal_vars = [c for c in X.columns if 'ord_' in c]\n\ntime_vars = ['day', 'month']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some feature engineering\nX['ord_5_1'] = X['ord_5'].apply(lambda x: x[0] if type(x) == str else np.nan)\nX['ord_5_2'] = X['ord_5'].apply(lambda x: x[1] if type(x) == str else np.nan)\nXt['ord_5_1'] = Xt['ord_5'].apply(lambda x: x[0] if type(x) == str else np.nan)\nXt['ord_5_2'] = Xt['ord_5'].apply(lambda x: x[1] if type(x) == str else np.nan)\n\nordinal_vars += ['ord_5_1', 'ord_5_2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting ordinal labels into ordered values\nordinals = {\n    'ord_1' : {\n        'Novice' : 0,\n        'Contributor' : 1,\n        'Expert' : 2,\n        'Master' : 3,\n        'Grandmaster' : 4\n    },\n    'ord_2' : {\n        'Freezing' : 0,\n        'Cold' : 1,\n        'Warm' : 2,\n        'Hot' : 3,\n        'Boiling Hot' : 4,\n        'Lava Hot' : 5\n    }\n}\n\ndef return_order(X, Xt, var_name):\n    mode = X[var_name].mode()[0]\n    el = sorted(set(X[var_name].fillna(mode).unique())|set(Xt[var_name].fillna(mode).unique()))\n    return {v:e for e, v in enumerate(el)}\n\nfor mapped_var in ordinal_vars:\n    if mapped_var not in ordinals:\n        mapped_values = return_order(X, Xt, mapped_var)\n        X[mapped_var].replace(mapped_values, inplace=True)\n        Xt[mapped_var].replace(mapped_values, inplace=True)\n    else:\n        X[mapped_var].replace(ordinals[mapped_var], inplace=True)\n        Xt[mapped_var].replace(ordinals[mapped_var], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating a list of numpy values from high cardinality variables\nX_cat, Xt_cat = list(), list()\ncategorical_counts = dict()\n\nfor hc in binary_vars+nominal_vars+ordinal_vars+time_vars:\n    # Finding out the levels in each high cardinality variable\n    levels = set(X[hc].astype(str).fillna(\"NAN\").unique())|set(Xt[hc].astype(str).fillna(\"NAN\").unique())\n    levels = np.array(list(levels))\n    # Counting the levels\n    categorical_counts[hc] = len(levels)\n    # Converting the levels into numeric values\n    le = LabelEncoder()\n    le.fit(np.ravel(levels.reshape(-1,1)))\n    X_cat.append(le.transform(X[hc].astype(str).fillna(\"NAN\").values))\n    Xt_cat.append(le.transform(Xt[hc].astype(str).fillna(\"NAN\").values))\n\nprint(\"Countings for high cardinality variables:\")\nprint(categorical_counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Enconding frequencies instead of labels (so we have some numeric variables)\n\ndef frequency_encoding(column, df, df_test=None):\n    frequencies = df[column].value_counts().reset_index()\n    df_values = df[[column]].merge(frequencies, how='left', \n                                   left_on=column, right_on='index').iloc[:,-1].values\n    if df_test is not None:\n        df_test_values = df_test[[column]].merge(frequencies, how='left', \n                                                 left_on=column, right_on='index').fillna(1).iloc[:,-1].values\n    else:\n        df_test_values = None\n    return df_values, df_test_values\n\nfreq_encoded = list()\n\nfor column in X.columns:\n    train_values, test_values = frequency_encoding(column, X, Xt)\n    X[column+'_counts'] = train_values\n    Xt[column+'_counts'] = test_values\n    freq_encoded.append(column+'_counts')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Target encoding of selected variables\nimport category_encoders as cat_encs\n\ncat_feat_to_encode = binary_vars + ordinal_vars + nominal_vars + time_vars\nsmoothing = 0.3\n\nenc_x = np.zeros(X[cat_feat_to_encode].shape)\n\nfor tr_idx, oof_idx in StratifiedKFold(n_splits=3, random_state=42, shuffle=True).split(X, y):\n    encoder = cat_encs.TargetEncoder(cols=cat_feat_to_encode, smoothing=smoothing)\n    \n    encoder.fit(X[cat_feat_to_encode].iloc[tr_idx], y[tr_idx])\n    enc_x[oof_idx, :] = encoder.transform(X[cat_feat_to_encode].iloc[oof_idx], y[oof_idx])\n    \nencoder.fit(X[cat_feat_to_encode], y)\nenc_xt = encoder.transform(Xt[cat_feat_to_encode]).values\n\ntarget_encoded = list()\n\nfor idx, new_var in enumerate(cat_feat_to_encode):\n    new_var = new_var + '_enc'\n    X[new_var] = enc_x[:,idx]\n    Xt[new_var] = enc_xt[:, idx]\n    target_encoded.append(new_var)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The values are normalized using the Standard Scaler\nssc = StandardScaler()\nselection = freq_encoded + target_encoded + ordinal_vars + time_vars\nX_ohe = ssc.fit_transform(X[selection].fillna(X[selection].median()))\nXt_ohe = ssc.transform(Xt[selection].fillna(X[selection].median()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Adding the GELU and LEAKY RELU functions as custom objects \n# (see: https://datascience.stackexchange.com/questions/49522/what-is-gelu-activation)\n\ndef gelu(x):\n    return 0.5 * x * (1 + tf.tanh(tf.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))\n\nget_custom_objects().update({'gelu': Activation(gelu)})\n\n# Add leaky-relu so we can use it as a string\nget_custom_objects().update({'leaky-relu': Activation(LeakyReLU(alpha=0.2))})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parametric DNN architecture\n\ndef tabular_dnn(numeric_variables, categorical_variables, categorical_counts,\n                feature_selection_dropout=0.2, categorical_dropout=0.1,\n                first_dense = 256, second_dense = 256, dense_dropout = 0.2, \n                activation_type=gelu):\n    \n    # Numeric inputs pipeline\n    numerical_inputs = Input(shape=(numeric_variables,))\n    numerical_normalization = BatchNormalization()(numerical_inputs)\n    numerical_feature_selection = Dropout(feature_selection_dropout)(numerical_normalization)\n\n    # Categorical inputs pipeline\n    categorical_inputs = []\n    categorical_embeddings = []\n    for category in categorical_variables:\n        categorical_inputs.append(Input(shape=[1], name=category))\n        category_counts = categorical_counts[category]\n        categorical_embeddings.append(\n            Embedding(category_counts+1, \n                      min(int(category_counts/1.5 + 1), 3), \n                      name = category + \"_embed\")(categorical_inputs[-1]))\n\n    categorical_logits = Concatenate(name = \"categorical_conc\")([Flatten()(SpatialDropout1D(categorical_dropout)(cat_emb)) \n                                                                 for cat_emb in categorical_embeddings])\n\n    # Fully connected layers\n    x = concatenate([numerical_feature_selection, categorical_logits])\n    x = BatchNormalization()(x)\n    \n    x = Dense(first_dense, activation=activation_type)(x)\n    x = BatchNormalization()(x)\n    x = Dropout(dense_dropout)(x)\n    \n    x = Dense(second_dense, activation=activation_type)(x)\n    x = Dropout(dense_dropout)(x)\n    x = BatchNormalization()(x)\n    \n    # Sigmoid final activation\n    output = Dense(1, activation=\"sigmoid\")(x)\n    \n    # Composing the model -> input list of numeric and each high cardinality variable\n    model = Model([numerical_inputs] + categorical_inputs, output)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Useful functions for training DNNs\n\ndef auroc(y_true, y_pred):\n    try:\n        return tf.py_function(roc_auc_score, (y_true, y_pred), tf.double)\n    except:\n        return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)\n\nget_custom_objects().update({'auroc': auroc})  \n\ndef mAP(y_true, y_pred):\n    try:\n        return tf.py_function(average_precision_score, (y_true, y_pred), tf.double)\n    except:\n        return tf.py_func(average_precision_score, (y_true, y_pred), tf.double)\n    \nget_custom_objects().update({'mAP': mAP})\n\ndef compile_model(model, loss, metrics, optimizer):\n    model.compile(loss=loss, metrics=metrics, optimizer=optimizer)\n    return model\n\ndef plot_keras_history(history, measures):\n    \"\"\"\n    history: Keras training history\n    measures = list of names of measures\n    \"\"\"\n    rows = len(measures) // 2 + len(measures) % 2\n    fig, panels = plt.subplots(rows, 2, figsize=(15, 5))\n    plt.subplots_adjust(top = 0.99, bottom=0.01, hspace=0.4, wspace=0.2)\n    try:\n        panels = [item for sublist in panels for item in sublist]\n    except:\n        pass\n    for k, measure in enumerate(measures):\n        panel = panels[k]\n        panel.set_title(measure + ' history')\n        panel.plot(history.epoch, history.history[measure], label=\"Train \"+measure)\n        panel.plot(history.epoch, history.history[\"val_\"+measure], label=\"Validation \"+measure)\n        panel.set(xlabel='epochs', ylabel=measure)\n        panel.legend()\n        \n    plt.show(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train/validation batch generator\ndef batch_generator(X_ohe, X_cat, y, cv=5, batch_size=64, random_state=None):\n    '''\n    Returns a batch from X, y\n    random_state allows determinism\n    different scikit-learn CV strategies are possible\n    '''\n    folds = len(y) // batch_size\n    if isinstance(cv, int):\n        kf = StratifiedKFold(n_splits=cv, \n                              shuffle=True, \n                              random_state=random_state)\n    else:\n        kf = cv\n    \n    while True:\n        for _, batch_index in kf.split(X_ohe, y):\n            numeric_input = X_ohe[batch_index].astype(np.float32)\n            categorical_input = [X_cat[i][batch_index] for i in range(len(X_cat))]\n            target = y[batch_index]\n            yield [numeric_input] + categorical_input, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Global training settings\nSEED = 42\nFOLDS = 20\nMAX_EPOCHS = 100\nBATCH_SIZE = 1024 * 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Defining callbacks\nmeasure_to_monitor = 'val_auroc' \nmodality = 'max'\n\nearly_stopping = EarlyStopping(monitor=measure_to_monitor, \n                               mode=modality, \n                               patience=5, \n                               verbose=0)\n\nmodel_checkpoint = ModelCheckpoint('best.model', \n                                   monitor=measure_to_monitor, \n                                   mode=modality, \n                                   save_best_only=True, \n                                   verbose=0)\n\nmodel_reduce_lr = ReduceLROnPlateau(monitor=measure_to_monitor,\n                                    mode=modality,\n                                    factor=0.25,\n                                    patience=2, \n                                    min_lr=1e-6, \n                                    verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Defining model\nmodel_params = {\n    \"numeric_variables\" : X_ohe.shape[1], \n    \"categorical_variables\" : categorical_counts.keys(),\n    \"categorical_counts\" : categorical_counts, \n    \"feature_selection_dropout\" : 0.0,\n    \"categorical_dropout\" : 0.3,\n    \"first_dense\" : 512,\n    \"second_dense\" : 512,\n    \"dense_dropout\" : 0.3,\n    \"activation_type\" : 'relu'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting the CV strategy\nskf = StratifiedKFold(n_splits=FOLDS, \n                      shuffle=True, \n                      random_state=SEED)\n\n# CV Iteration: we store best epochs, oof and cv testv prediciton\nroc_auc = list()\naverage_precision = list()\noof = np.zeros(len(X))\nbest_iteration = list()\ncv_test_preds = np.zeros(len(Xt))\n\nfor fold, (train_idx, test_idx) in enumerate(skf.split(X, y)):\n    \n    # Re-instantiating the model\n    model = compile_model(tabular_dnn(**model_params), \n                          binary_crossentropy, \n                          [auroc, mAP], \n                          Adam(learning_rate=0.0001))\n    \n    # Creating the train and validation sets\n    X_cv_ohe = X_ohe[train_idx].astype(np.float32)\n    X_cv_cat = [X_cat[i][train_idx] for i in range(len(X_cat))]\n    y_cv = y[train_idx]\n    X_oof_ohe = X_ohe[test_idx].astype(np.float32)\n    X_oof_cat = [X_cat[i][test_idx] for i in range(len(X_cat))]\n    y_oof = y[test_idx]\n    \n    # Instantiating the train and validation generators\n    train_batch = batch_generator(X_cv_ohe,\n                                  X_cv_cat,\n                                  y_cv,\n                                  batch_size=BATCH_SIZE,\n                                  random_state=SEED)\n    \n    val_batch = batch_generator(X_oof_ohe,\n                                X_oof_cat,\n                                y_oof,\n                                batch_size=BATCH_SIZE,\n                                random_state=SEED)\n    \n    train_steps = len(y_cv) // BATCH_SIZE \n    validation_steps = len(y_oof) // BATCH_SIZE\n    \n    # Training\n    history = model.fit_generator(train_batch,\n                                  validation_data=val_batch,\n                                  epochs=MAX_EPOCHS,\n                                  steps_per_epoch=train_steps,\n                                  validation_steps=validation_steps,\n                                  callbacks=[model_checkpoint, early_stopping, model_reduce_lr],\n                                  #class_weight=[1.0, (np.sum(y_cv==0) / np.sum(y_cv==1))],\n                                  verbose=1)\n    \n    # Reporting\n    print(\"\\nFOLD %i\" % fold)\n    plot_keras_history(history, measures=['auroc', 'loss'])\n    \n    # OOF prediction\n    best_iteration.append(np.argmax(history.history['val_auroc']) + 1)\n    \n    model = load_model('best.model')\n    \n    preds = model.predict([X_oof_ohe] + X_oof_cat,\n                          verbose=1,\n                          batch_size=1024).flatten()\n\n    oof[test_idx] = preds\n\n    roc_auc.append(roc_auc_score(y_true=y_oof, y_score=preds))\n    \n    average_precision.append(average_precision_score(y_true=y_oof, y_score=preds))\n    \n    # CV test prediction\n    cv_test_preds += model.predict([Xt_ohe] + Xt_cat,\n                                   verbose=1,\n                                   batch_size=1024).flatten() / FOLDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Storing results to disk\noof = pd.DataFrame({'id':id_train, 'dnn_oof': oof})\noof.to_csv(\"oof.csv\", index=False)\n\nsubmission = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/sample_submission.csv\")\nsubmission.target = cv_test_preds\nsubmission.to_csv(\"./dnn_cv_submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reporting from training\nprint(\"Average cv roc auc score %0.3f ± %0.3f\" % (np.mean(roc_auc), np.std(roc_auc)))\nprint(\"Average cv roc average precision %0.3f ± %0.3f\" % (np.mean(average_precision), np.std(average_precision)))\n\nprint(\"Roc auc score OOF %0.3f\" % roc_auc_score(y_true=y, y_score=oof.dnn_oof))\nprint(\"Average precision OOF %0.3f\" % average_precision_score(y_true=y, y_score=oof.dnn_oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We now train on all the examples, using a rule of thumb for the number of iterations\ntrain_batch = batch_generator(X_ohe, X_cat, y,\n                              batch_size=BATCH_SIZE,\n                              random_state=SEED)\n\ntrain_steps = len(y) // BATCH_SIZE\n\nmodel = compile_model(tabular_dnn(**model_params), \n                      binary_crossentropy, \n                      [auroc, mAP], \n                      Adam(learning_rate=0.0001))\n\nhistory = model.fit_generator(train_batch,\n                              epochs=int(np.median(best_iteration)),\n                              steps_per_epoch=train_steps,\n                              #class_weight=[1.0, (np.sum(y==0) / np.sum(y==1))],\n                              verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predicting and final submission\ntest_preds = model.predict([Xt_ohe] + Xt_cat,\n                           verbose=1,\n                           batch_size=1024).flatten()\n\nsubmission = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/sample_submission.csv\")\nsubmission.target = test_preds\nsubmission.to_csv(\"./submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Blend\nsubmission.target = (submission.target + pd.read_csv(\"./dnn_cv_submission.csv\").target) / 2\nsubmission.to_csv(\"./blend_submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}