{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport joblib\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics, preprocessing\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import utils","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def auc(y_true, y_pred):\n    def fallback_auc(y_true, y_pred):\n        try:\n            return metrics.roc_auc_score(y_true, y_pred)\n        except:\n            return 0.5\n    return tf.py_function(fallback_auc, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(data, catcols):    \n    inputs = []\n    outputs = []\n    for c in catcols:\n        num_unique_values = int(data[c].nunique())\n        embed_dim = int(min(np.ceil((num_unique_values)/2), 50))\n        inp = layers.Input(shape=(1,))\n        out = layers.Embedding(num_unique_values + 1, embed_dim, name=c)(inp)\n        out = layers.SpatialDropout1D(0.3)(out)\n        out = layers.Reshape(target_shape=(embed_dim, ))(out)\n        inputs.append(inp)\n        outputs.append(out)\n    \n    x = layers.Concatenate()(outputs)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Dense(300, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Dense(300, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.BatchNormalization()(x)\n    \n    y = layers.Dense(2, activation=\"softmax\")(x)\n\n    model = Model(inputs=inputs, outputs=y)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\ntest = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\nsample = pd.read_csv(\"../input/cat-in-the-dat-ii/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"target\"] = -1\ndata = pd.concat([train, test]).reset_index(drop=True)\n\nfeatures = [x for x in train.columns if x not in [\"id\", \"target\"]]\n\nfor feat in features:\n    lbl_enc = preprocessing.LabelEncoder()\n    data[feat] = lbl_enc.fit_transform(data[feat].fillna(\"-1\").astype(str).values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = data[data.target != -1].reset_index(drop=True)\ntest = data[data.target == -1].reset_index(drop=True)\ntest_data = [test.loc[:, features].values[:, k] for k in range(test.loc[:, features].values.shape[1])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_preds = np.zeros((len(train)))\ntest_preds = np.zeros((len(test)))\n\nskf = StratifiedKFold(n_splits=50)\nfor train_index, test_index in skf.split(train, train.target.values):\n    X_train, X_test = train.iloc[train_index, :], train.iloc[test_index, :]\n    X_train = X_train.reset_index(drop=True)\n    X_test = X_test.reset_index(drop=True)\n    y_train, y_test = X_train.target.values, X_test.target.values\n    model = create_model(data, features)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[auc])\n    X_train = [X_train.loc[:, features].values[:, k] for k in range(X_train.loc[:, features].values.shape[1])]\n    X_test = [X_test.loc[:, features].values[:, k] for k in range(X_test.loc[:, features].values.shape[1])]\n    \n    es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5,\n                                 verbose=1, mode='max', baseline=None, restore_best_weights=True)\n\n    rlr = callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.5,\n                                      patience=3, min_lr=1e-6, mode='max', verbose=1)\n    \n    model.fit(X_train,\n              utils.to_categorical(y_train),\n              validation_data=(X_test, utils.to_categorical(y_test)),\n              verbose=1,\n              batch_size=1024,\n              callbacks=[es, rlr],\n              epochs=100\n             )\n    valid_fold_preds = model.predict(X_test)[:, 1]\n    test_fold_preds = model.predict(test_data)[:, 1]\n    oof_preds[test_index] = valid_fold_preds.ravel()\n    test_preds += test_fold_preds.ravel()\n    print(metrics.roc_auc_score(y_test, valid_fold_preds))\n    K.clear_session()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Overall AUC={}\".format(metrics.roc_auc_score(train.target.values, oof_preds)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds /= 50\ntest_ids = test.id.values\nprint(\"Saving submission file\")\nsubmission = pd.DataFrame.from_dict({\n    'id': test_ids,\n    'target': test_preds\n})\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}