{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Load Libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_in = '../input/cat-in-the-dat-ii/'\nX_train= pd.read_csv(path_in+'train.csv')\nX_test= pd.read_csv(path_in+'test.csv')\nsamp_subm = pd.read_csv(path_in+'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test =pd.concat([X_train, X_test],sort=False).reset_index()\ny_train = X_train['target']\ntrain_test = train_test.drop(['index','id','target'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_bin = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\nfeatures_low_nom = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']\nfeatures_hi_nom = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\nfeatures_ord = ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4']\nfeatures_hi_ord = ['ord_5']\nfeatures_cyc = ['day', 'month']\n\nnumerics = ['float16', 'float32', 'float64']\ncategoricals=['int8','int16', 'int32', 'int64', ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Features to numeric"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom category_encoders import WOEEncoder\nfrom sklearn import preprocessing\nimport string \n\n\ndef Convert_to_numeric(df):\n      \n    #Ordinal features\n    map_ord = {'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4,\n           'Freezing': 0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot': 4, 'Lava Hot':5}\n    \n    scii_letters_list=list(string.ascii_letters)\n    map_ord_hex= dict(zip(scii_letters_list,range(0, len(scii_letters_list))))\n\n    \n    df['ord_0'] = df['ord_0']\n    df['ord_1'] = df['ord_1'].replace(map_ord)\n    df['ord_2'] = df['ord_2'].replace(map_ord)\n    df['ord_3'] = df['ord_3'].replace(map_ord_hex)\n    df['ord_4'] = df['ord_4'].replace(map_ord_hex)\n        \n    df[features_ord] = df[features_ord].fillna(df[features_ord].mean())\n    \n    StandardScaler_Encoder = preprocessing.StandardScaler()\n    df[features_ord] = StandardScaler_Encoder.fit_transform(df[features_ord].astype(float))    \n    \n\n   #Binary, Low nominal and time features WOE encoder.\n    n_splits=5\n    WOE_features=features_bin+features_low_nom+features_cyc\n    for col in WOE_features:\n        df[f'{col}_Encode']=0\n        for tr_idx, tst_idx in StratifiedKFold(n_splits=n_splits, random_state=2020, shuffle=True).split(df[:600000], y_train):\n            WOE_encoder = WOEEncoder(cols=col)        \n            WOE_encoder.fit(df[:600000].iloc[tr_idx, :], y_train.iloc[tr_idx])\n            col_df=WOE_encoder.transform(df)[col]/n_splits\n            df[f'{col}_Encode']= df[f'{col}_Encode']+col_df       \n    df = df.drop(WOE_features, axis = 1)          \n    \n    \n    #High Nominal Features Label encoder.\n    Label_col=features_hi_nom+features_hi_ord\n    for col in Label_col:\n        Label_Encoder = preprocessing.LabelEncoder()\n        df[col] = Label_Encoder.fit_transform(df[col].fillna(\"-1\").astype(str).values)\n\n\n\n        \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_test=Convert_to_numeric(train_test)\n\ntrain_test = reduce_mem_usage(train_test,use_float16=True)\n\nX_train=train_test[:len(X_train)]\nX_test=train_test[-len(X_test):]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#converting data to list format to match the network structure\ndef preproc(X_train, X_val, X_test):\n\n    input_list_train = []\n    input_list_val = []\n    input_list_test = []    \n    \n    \n    #the cols to be embedded: rescaling to range [0, # values)\n    for c in X_train.select_dtypes(include=categoricals):\n        raw_vals = np.unique(X_train[c])\n        val_map = {}\n        for i in range(len(raw_vals)):\n            val_map[raw_vals[i]] = i       \n        input_list_train.append(X_train[c].map(val_map).values)\n        input_list_val.append(X_val[c].map(val_map).fillna(0).values)\n        input_list_test.append(X_test[c].map(val_map).fillna(0).values)\n     \n    #the rest of the columns\n    other_cols = X_train.select_dtypes(include=numerics).columns.tolist()\n    input_list_train.append(X_train[other_cols].values)\n    input_list_val.append(X_val[other_cols].values)\n    input_list_test.append(X_test[other_cols].values)\n    \n    return input_list_train, input_list_val, input_list_test ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Keras embeddings based on\n#https://github.com/mmortazavi/EntityEmbedding-Working_Example\n    \nimport tensorflow as tf\nimport keras.backend as K\nfrom keras.models import Sequential,Model  \nfrom keras.models import Model as KerasModel\nfrom keras.layers import Input, Dense, Activation, Reshape, BatchNormalization\nfrom keras.layers import Concatenate, Dropout, SpatialDropout1D\nfrom keras.layers.embeddings import Embedding\nfrom tensorflow.keras import callbacks, utils \nfrom keras.optimizers import Adam\nfrom tensorflow.python.keras.optimizers import TFOptimizer\nfrom sklearn.metrics import roc_auc_score\n\n\ndef make_model():\n\n    input_models=[]\n    output_embeddings=[]\n    numerics = ['float16', 'float32', 'float64']\n    categoricals=['int8','int16', 'int32', 'int64']\n\n    for categorical_var in X_train.select_dtypes(include=categoricals):\n    \n        #Name of the categorical variable that will be used in the Keras Embedding layer\n        cat_emb_name= categorical_var.replace(\" \", \"\")+'_Embedding'\n        # Define the embedding_size\n        no_of_unique_cat  = X_train[categorical_var].nunique()\n        embedding_size = int(min(np.ceil((no_of_unique_cat)/2), 50 ))\n  \n        #One Embedding Layer for each categorical variable\n        input_model = Input(shape=(1,))\n        output_model = Embedding(no_of_unique_cat, embedding_size, name=cat_emb_name)(input_model)\n        output_model = SpatialDropout1D(0.3)(output_model)\n        output_model = Reshape(target_shape=(embedding_size,))(output_model)    \n  \n        #Appending all the categorical inputs\n        input_models.append(input_model)\n  \n        #Appending all the embeddings\n        output_embeddings.append(output_model)\n        \n    shape_numeric=len(X_train.select_dtypes(include=numerics).columns.tolist())\n    #Other non-categorical data columns (numerical). \n    #I define single another network for the other columns and add them to our models list.\n    input_numeric = Input(shape=(shape_numeric,))\n    embedding_numeric = BatchNormalization()(input_numeric) \n    \n    input_models.append(input_numeric)\n    output_embeddings.append(embedding_numeric)\n\n    #At the end we concatenate altogther and add other Dense layers\n    output = Concatenate()(output_embeddings)\n    output = BatchNormalization()(output)\n    \n    output = Dense(317,activation='relu',kernel_initializer=\"uniform\")(output) \n    output= Dropout(0.3)(output) # To reduce ovwefiting\n    output = BatchNormalization()(output)\n    \n    output = Dense(150,activation='relu',kernel_initializer=\"uniform\")(output) \n    output= Dropout(0.2)(output) # To reduce ovwefiting\n    output = BatchNormalization()(output)\n    \n    output = Dense(30,activation='relu')(output) \n    output= Dropout(0.1)(output) # To reduce ovwefiting\n    output = BatchNormalization()(output)\n    \n       \n    output = Dense(1, activation='sigmoid')(output)\n\n    model = Model(inputs=input_models, outputs=output)\n    \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pydot_ng\nimport pydot_ng as pydot\nfrom keras.utils import plot_model\n\nmodel=make_model()\nplot_model(model, show_shapes=True, show_layer_names=True, to_file='model.png')\nfrom IPython.display import Image\nImage(retina=True, filename='model.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom time import time\nfrom sklearn.decomposition import PCA\nfrom catboost import CatBoostClassifier\n\n\n\ncls =CatBoostClassifier(eval_metric='AUC',\n                        loss_function='CrossEntropy',\n                        learning_rate=0.05,\n                        l2_leaf_reg=3)\n\nkeras_oof = np.zeros(len(X_train))\ncls_oof = np.zeros(len(X_train))\nkeras_preds = np.zeros(len(X_test))\ncls_preds = np.zeros(len(X_test))\n\nNFOLDS = 10\n\nfolds = StratifiedKFold(n_splits=NFOLDS, shuffle=False)\n\n\ntraining_start_time = time()\nfor fold, (train_idx, valid_idx) in enumerate(folds.split(X_train, y_train)):\n    \n    train_x, train_y = X_train.iloc[train_idx], y_train.iloc[train_idx]\n    valid_x, valid_y = X_train.iloc[valid_idx], y_train.iloc[valid_idx]\n    \n    X_train_list,X_val_list,X_test_list = preproc(train_x,valid_x, X_test)#Transfor to arrays to be accepted by \n\n    \n    start_time = time()\n    print(f'Training on fold {fold+1}')\n    \n    K.clear_session()\n    model = make_model()\n    adam = Adam(learning_rate=1e-4)\n    model.compile(loss='binary_crossentropy',  optimizer=adam ,metrics=[tf.keras.metrics.AUC()])\n\n    \n    es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.001, patience=6,\n                                 verbose=1, mode='min', baseline=None, restore_best_weights=True)\n\n    rl = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1,min_delta=0.001,\n                                      patience=2, min_lr=1e-6, mode='min', verbose=0)\n     \n   \n    history  =  model.fit(X_train_list,train_y, validation_data=(X_val_list,valid_y) , epochs =  20,\n                          batch_size = 1024, callbacks=[es,rl],verbose= 0)\n        \n    \n    keras_fold_val= model.predict(X_val_list).ravel()\n    keras_oof[valid_idx] += keras_fold_val/folds.n_splits\n    keras_fold_auc=roc_auc_score(valid_y, keras_fold_val)\n    keras_preds += model.predict(X_test_list).ravel()/folds.n_splits        \n        \n#######################CatBooost########################################\n\n    layer_name = 'batch_normalization_2'\n    intermediate_layer_model = Model(inputs=model.input,outputs=model.get_layer(layer_name).output)    \n    \n    X_train_k = pd.DataFrame(intermediate_layer_model.predict(X_train_list))\n    X_val_k = pd.DataFrame(intermediate_layer_model.predict(X_val_list))\n    X_test_k = pd.DataFrame(intermediate_layer_model.predict(X_test_list))   \n    \n    cls.fit(X_train_k, train_y, eval_set=(X_val_k, valid_y),early_stopping_rounds=50,verbose=100,plot=False)\n\n    cls_fold_val=cls.predict_proba(X_val_k)[:,1]\n    cls_oof[valid_idx] += cls_fold_val/folds.n_splits\n    cls_fold_AUC=roc_auc_score(valid_y, cls_fold_val)\n    cls_preds +=cls.predict_proba(X_test_k)[:,1]/folds.n_splits\n    \n###########################Fold Results#########################\n        \n    print(\"\\n\")\n    print('-' * 30)  \n    print('Fold {} - Keras_OOF = {}'.format(fold + 1,keras_fold_auc ))\n    print('Fold {} - CatBoost_OOF = {}'.format(fold + 1,cls_fold_AUC))\n    print('-' * 30)\n    print(\"\\n\")\n    \n    \n#################Validation Results##############################\nkeras_val_auc=roc_auc_score(y_train, keras_oof)\ncls_val_auc=roc_auc_score(y_train, cls_oof)\nmix_val_auc=roc_auc_score(y_train, (keras_oof+cls_oof)/2)\n\n\nprint('-' * 30)  \nprint('# Final_Keras_OOF = {}'.format(keras_val_auc))\nprint('# Final_CatBoost_OOF = {}'.format(cls_val_auc))\nprint('# Final_Mix_OOF = {}'.format(mix_val_auc))\nprint('-' * 30)\nprint(\"\\n\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions=(keras_preds+cls_preds)/2\n\nnum = samp_subm.id\noutput = pd.DataFrame({'id': num,\n                       'target': predictions})\noutput.to_csv('keras-Embedding-catboost.csv', index=False)\noutput.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}