{"cells":[{"metadata":{},"cell_type":"markdown","source":"# (Embeddings,Target + Keras) + (OHE,Target + Logit)\n\nIdeas:\n* Replace missing values with constant\n* Add number of missing values in row as a feature\n* Apply StandardScaler to created feature\n* Apply Target to features that have many unique values\n* Apply entity embedding layers for other features + Keras\n* Apply OHE for other features + Logit\n* Blend Logit and Keras"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport warnings\nwarnings.simplefilter('ignore')\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv', index_col='id')\ntest = pd.read_csv('../input/cat-in-the-dat-ii/test.csv', index_col='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train.head(3).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def summary(df):\n    summary = pd.DataFrame(df.dtypes, columns=['dtypes'])\n    summary = summary.reset_index()\n    summary['Name'] = summary['index']\n    summary = summary[['Name', 'dtypes']]\n    summary['Missing'] = df.isnull().sum().values    \n    summary['Uniques'] = df.nunique().values\n    summary['First Value'] = df.loc[0].values\n    summary['Second Value'] = df.loc[1].values\n    summary['Third Value'] = df.loc[2].values\n    return summary\n\n\nsummary(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Handle missing values"},{"metadata":{},"cell_type":"markdown","source":"Add number of missing values in row as a feature"},{"metadata":{"trusted":false},"cell_type":"code","source":"train['missing_count'] = train.isnull().sum(axis=1)\ntest['missing_count'] = test.isnull().sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Replace missing values with constants"},{"metadata":{"trusted":false},"cell_type":"code","source":"missing_number = -99999\nmissing_string = 'MISSING_STRING'","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"numerical_features = [\n    'bin_0', 'bin_1', 'bin_2',\n    'ord_0',\n    'day', 'month'\n]\n\nstring_features = [\n    'bin_3', 'bin_4',\n    'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5',\n    'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def impute(train, test, columns, value):\n    for column in columns:\n        train[column] = train[column].fillna(value)\n        test[column] = test[column].fillna(value)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"impute(train, test, numerical_features, missing_number)\nimpute(train, test, string_features, missing_string)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature engineering"},{"metadata":{},"cell_type":"markdown","source":"Split 'ord_5' preserving missing values"},{"metadata":{"trusted":false},"cell_type":"code","source":"train['ord_5_1'] = train['ord_5'].str[0]\ntrain['ord_5_2'] = train['ord_5'].str[1]\n\ntrain.loc[train['ord_5'] == missing_string, 'ord_5_1'] = missing_string\ntrain.loc[train['ord_5'] == missing_string, 'ord_5_2'] = missing_string\n\ntrain = train.drop('ord_5', axis=1)\n\n\ntest['ord_5_1'] = test['ord_5'].str[0]\ntest['ord_5_2'] = test['ord_5'].str[1]\n\ntest.loc[test['ord_5'] == missing_string, 'ord_5_1'] = missing_string\ntest.loc[test['ord_5'] == missing_string, 'ord_5_2'] = missing_string\n\ntest = test.drop('ord_5', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"simple_features = [\n    'missing_count'\n]\n\noe_features = [\n    'bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4',\n    'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4',\n    'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5_1', 'ord_5_2',\n    'day', 'month'\n]\n\nohe_features = oe_features\n\ntarget_features = [\n    'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Extract target variable"},{"metadata":{"trusted":false},"cell_type":"code","source":"y_train = train['target'].copy()\nx_train = train.drop('target', axis=1)\ndel train\n\nx_test = test.copy()\ndel test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Standard scaler"},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n\nscaler = StandardScaler()\nsimple_x_train = scaler.fit_transform(x_train[simple_features])\nsimple_x_test = scaler.transform(x_test[simple_features])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## OHE"},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\n\n\nohe = OneHotEncoder(dtype='uint16', handle_unknown=\"ignore\")\nohe_x_train = ohe.fit_transform(x_train[ohe_features])\nohe_x_test = ohe.transform(x_test[ohe_features])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ordinal encoder"},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.preprocessing import OrdinalEncoder\n\n\noe = OrdinalEncoder()\noe_x_train = oe.fit_transform(x_train[oe_features])\noe_x_test = oe.transform(x_test[oe_features])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Target encoder"},{"metadata":{"trusted":false},"cell_type":"code","source":"from category_encoders import TargetEncoder\nfrom sklearn.model_selection import StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def transform(transformer, x_train, y_train, cv):\n    oof = pd.DataFrame(index=x_train.index, columns=x_train.columns)\n    for train_idx, valid_idx in cv.split(x_train, y_train):\n        x_train_train = x_train.loc[train_idx]\n        y_train_train = y_train.loc[train_idx]\n        x_train_valid = x_train.loc[valid_idx]\n        transformer.fit(x_train_train, y_train_train)\n        oof_part = transformer.transform(x_train_valid)\n        oof.loc[valid_idx] = oof_part\n    return oof","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"target = TargetEncoder(drop_invariant=True, smoothing=0.2)\n\ncv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\ntarget_x_train = transform(target, x_train[target_features], y_train, cv).astype('float')\n\ntarget.fit(x_train[target_features], y_train)\ntarget_x_test = target.transform(x_test[target_features]).astype('float')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge for Logit"},{"metadata":{"trusted":false},"cell_type":"code","source":"import scipy\n\n\nx_train = scipy.sparse.hstack([ohe_x_train, simple_x_train, target_x_train]).tocsr()\nx_test = scipy.sparse.hstack([ohe_x_test, simple_x_test, target_x_test]).tocsr()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Logistic regression"},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\n\nlogit = LogisticRegression(C=0.54321, solver='lbfgs', max_iter=10000)\nlogit.fit(x_train, y_train)\ny_pred_logit = logit.predict_proba(x_test)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge for Keras"},{"metadata":{"trusted":false},"cell_type":"code","source":"x_train = np.concatenate((oe_x_train, simple_x_train, target_x_train), axis=1)\nx_test = np.concatenate((oe_x_test, simple_x_test, target_x_test), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"categorial_part = oe_x_train.shape[1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Keras"},{"metadata":{"trusted":false},"cell_type":"code","source":"import tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\n\ndef auc(y_true, y_pred):\n    def fallback_auc(y_true, y_pred):\n        try:\n            return roc_auc_score(y_true, y_pred)\n        except:\n            return 0.5\n    return tf.py_function(fallback_auc, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def make_model(data, categorial_part):\n    \n    inputs = []\n    \n    categorial_outputs = []\n    for idx in range(categorial_part):\n        n_unique = np.unique(data[:,idx]).shape[0]\n        n_embeddings = int(min(np.ceil(n_unique / 2), 50))\n        inp = tf.keras.layers.Input(shape=(1,))\n        inputs.append(inp)\n        x = tf.keras.layers.Embedding(n_unique + 1, n_embeddings)(inp)\n        x = tf.keras.layers.SpatialDropout1D(0.3)(x)\n        x = tf.keras.layers.Reshape((n_embeddings,))(x)\n        categorial_outputs.append(x)\n    \n    x1 = tf.keras.layers.Concatenate()(categorial_outputs)\n    x1 = tf.keras.layers.BatchNormalization()(x1)\n    \n    inp = tf.keras.layers.Input(shape=(data.shape[1] - categorial_part,))\n    inputs.append(inp)\n    x2 = tf.keras.layers.BatchNormalization()(inp)\n    \n    x = tf.keras.layers.Concatenate()([x1, x2])\n    x = tf.keras.layers.Dense(128, activation='relu')(x)\n    x = tf.keras.layers.Dropout(0.3)(x)\n    \n    y = tf.keras.layers.Dense(1, activation='sigmoid', name='dense_output')(x)\n    \n    print('Expected number of inputs:', len(inputs))\n    model = tf.keras.Model(inputs=inputs, outputs=y)\n    model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(learning_rate=3e-5), metrics=['accuracy', auc])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def make_inputs(data, categorial_part):\n    inputs = []\n    for idx in range(categorial_part):\n        inputs.append(data[:, idx])\n    inputs.append(data[:, categorial_part:])\n    return inputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport tensorflow.keras.backend as K\n\n\nn_splits = 50\n\ntrained_estimators = []\nhistories = []\nscores = []\n\ncv = KFold(n_splits=n_splits, random_state=42)\nfor train_idx, valid_idx in cv.split(x_train, y_train):\n    \n    x_train_train = x_train[train_idx]\n    y_train_train = y_train[train_idx]\n    x_train_valid = x_train[valid_idx]\n    y_train_valid = y_train[valid_idx]\n    \n    K.clear_session()\n    \n    estimator = make_model(x_train, categorial_part)\n    \n    es = tf.keras.callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=10,\n                                          verbose=1, mode='max', restore_best_weights=True)\n    \n    rl = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=3, min_lr=1e-6, mode='max', verbose=1)\n    \n    history = estimator.fit(make_inputs(x_train_train, categorial_part), y_train_train, batch_size=1024, epochs=100, callbacks=[es, rl],\n                            validation_data=(make_inputs(x_train_valid, categorial_part), y_train_valid))\n    trained_estimators.append(estimator)\n    histories.append(history)\n    \n    oof_part = estimator.predict(make_inputs(x_train_valid, categorial_part))\n    score = roc_auc_score(y_train_valid, oof_part)\n    print('Fold score:', score)\n    scores.append(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print('Mean score:', np.mean(scores))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize"},{"metadata":{"trusted":false},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nfig, axs = plt.subplots(3, 2, figsize=(18,18))\n\n# AUC\nfor h in histories:\n    axs[0,0].plot(h.history['auc'], color='g')\naxs[0,0].set_title('Model AUC - Train')\naxs[0,0].set_ylabel('AUC')\naxs[0,0].set_xlabel('Epoch')\n\nfor h in histories:\n    axs[0,1].plot(h.history['val_auc'], color='b')\naxs[0,1].set_title('Model AUC - Test')\naxs[0,1].set_ylabel('AUC')\naxs[0,1].set_xlabel('Epoch')\n\n# accuracy\nfor h in histories:\n    axs[1,0].plot(h.history['accuracy'], color='g')\naxs[1,0].set_title('Model accuracy - Train')\naxs[1,0].set_ylabel('Accuracy')\naxs[1,0].set_xlabel('Epoch')\n\nfor h in histories:\n    axs[1,1].plot(h.history['val_accuracy'], color='b')\naxs[1,1].set_title('Model accuracy - Test')\naxs[1,1].set_ylabel('Accuracy')\naxs[1,1].set_xlabel('Epoch')\n\n# loss\nfor h in histories:\n    axs[2,0].plot(h.history['loss'], color='g')\naxs[2,0].set_title('Model loss - Train')\naxs[2,0].set_ylabel('Loss')\naxs[2,0].set_xlabel('Epoch')\n\nfor h in histories:\n    axs[2,1].plot(h.history['val_loss'], color='b')\naxs[2,1].set_title('Model loss - Test')\naxs[2,1].set_ylabel('Loss')\naxs[2,1].set_xlabel('Epoch')\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict"},{"metadata":{"trusted":false},"cell_type":"code","source":"len(trained_estimators)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_pred = np.zeros(x_test.shape[0])\nx_test_inputs = make_inputs(x_test, categorial_part)\nfor estimator in trained_estimators:\n    y_pred += estimator.predict(x_test_inputs).reshape(-1) / len(trained_estimators)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_pred_keras = y_pred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Blend Logit and Keras"},{"metadata":{"trusted":false},"cell_type":"code","source":"y_pred = np.add(y_pred_logit, y_pred_keras) / 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submit predictions"},{"metadata":{"trusted":false},"cell_type":"code","source":"submission = pd.read_csv('../input/cat-in-the-dat-ii/sample_submission.csv', index_col='id')\nsubmission['target'] = y_pred\nsubmission.to_csv('logit_keras.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}