{"cells":[{"metadata":{},"cell_type":"markdown","source":"**This kernel uses Deepfm model from deepctr package**\n\n**Deepfm :** [Deepfm using deepctr](https://deepctr-doc.readthedocs.io/en/v0.6.3/deepctr.models.deepfm.html)\n* Guo H, Tang R, Ye Y, et al. Deepfm: a factorization-machine based neural network for ctr prediction[J]. arXiv preprint arXiv:1703.04247, 2017.(https://arxiv.org/abs/1703.04247)"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-warn-conflicts -q deepctr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import log_loss, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom deepctr.feature_column import SparseFeat, SparseFeat, DenseFeat, get_feature_names\nfrom tensorflow.keras.models import Model, load_model\nfrom deepctr.models import DeepFM\nfrom tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, Callback\nfrom sklearn.preprocessing import OrdinalEncoder, LabelEncoder, OneHotEncoder\nfrom tensorflow.keras.utils import get_custom_objects\nfrom tensorflow.keras.optimizers import Adam,RMSprop\nfrom tensorflow.keras.layers import Activation\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import utils\nimport tensorflow.keras as keras\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport warnings\nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\ntest = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['target'] = -1\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.concat([train, test]).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['null'] = data.isna().sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sparse_features = [feat for feat in train.columns if feat not in ['id','target']]\n\ndata[sparse_features] = data[sparse_features].fillna('-1', )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for feat in sparse_features:\n    lbe = LabelEncoder()\n    data[feat] = lbe.fit_transform(data[feat].fillna('-1').astype(str).values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = data[data.target != -1].reset_index(drop=True)\ntest  = data[data.target == -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique()) for feat in sparse_features]\n\ndnn_feature_columns = fixlen_feature_columns\nlinear_feature_columns = fixlen_feature_columns\n\nfeature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def auc(y_true, y_pred):\n    def fallback_auc(y_true, y_pred):\n        try:\n            return roc_auc_score(y_true, y_pred)\n        except:\n            return 0.5\n    return tf.py_function(fallback_auc, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def focal_loss(gamma=2., alpha=.25):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(K.epsilon()+pt_1))-K.mean((1-alpha) * K.pow( pt_0, gamma) * K.log(1. - pt_0 + K.epsilon()))\n    return focal_loss_fixed\n\nget_custom_objects().update({'focal_loss_fn': focal_loss()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def custom_gelu(x):\n    return 0.5 * x * (1 + tf.tanh(tf.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))\n\nget_custom_objects().update({'custom_gelu': Activation(custom_gelu)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Mish(Activation):\n    '''\n    Mish Activation Function.\n    .. math::\n        mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + e^{x}))\n    Shape:\n        - Input: Arbitrary. Use the keyword argument `input_shape`\n        (tuple of integers, does not include the samples axis)\n        when using this layer as the first layer in a model.\n        - Output: Same shape as the input.\n    Examples:\n        >>> X = Activation('Mish', name=\"conv1_act\")(X_input)\n    '''\n\n    def __init__(self, activation, **kwargs):\n        super(Mish, self).__init__(activation, **kwargs)\n        self.__name__ = 'Mish'\n\n\ndef mish(inputs):\n    return inputs * tf.math.tanh(tf.math.softplus(inputs))\n\n\nget_custom_objects().update({'Mish': Mish(mish)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class WarmUpLearningRateScheduler(tf.keras.callbacks.Callback):\n    \"\"\"Warmup learning rate scheduler\n    \"\"\"\n\n    def __init__(self, warmup_batches, init_lr, verbose=0):\n        \"\"\"Constructor for warmup learning rate scheduler\n\n        Arguments:\n            warmup_batches {int} -- Number of batch for warmup.\n            init_lr {float} -- Learning rate after warmup.\n\n        Keyword Arguments:\n            verbose {int} -- 0: quiet, 1: update messages. (default: {0})\n        \"\"\"\n\n        super(WarmUpLearningRateScheduler, self).__init__()\n        self.warmup_batches = warmup_batches\n        self.init_lr = init_lr\n        self.verbose = verbose\n        self.batch_count = 0\n        self.learning_rates = []\n\n    def on_batch_end(self, batch, logs=None):\n        self.batch_count = self.batch_count + 1\n        lr = K.get_value(self.model.optimizer.lr)\n        self.learning_rates.append(lr)\n\n    def on_batch_begin(self, batch, logs=None):\n        if self.batch_count <= self.warmup_batches:\n            lr = self.batch_count*self.init_lr/self.warmup_batches\n            K.set_value(self.model.optimizer.lr, lr)\n            if self.verbose > 0:\n                print('\\nBatch %05d: WarmUpLearningRateScheduler setting learning '\n                      'rate to %s.' % (self.batch_count + 1, lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CyclicLR(keras.callbacks.Callback):\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1 / (2. ** (x - 1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma ** (x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n\n    def clr(self):\n        cycle = np.floor(1 + self.clr_iterations / (2 * self.step_size))\n        x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * self.scale_fn(\n                self.clr_iterations)\n\n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())\n\n    def on_batch_end(self, epoch, logs=None):\n\n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        K.set_value(self.model.optimizer.lr, self.clr())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = ['target']\nN_Splits = 50\nVerbose = 0\nEpochs = 10\nSEED = 2020\nBatch_S_T = 128\nBatch_S_P = 512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_pred_deepfm = np.zeros((len(train), ))\ny_pred_deepfm = np.zeros((len(test), ))\n\n\nskf = StratifiedKFold(n_splits=N_Splits, shuffle=True, random_state=SEED)\nfor fold, (tr_ind, val_ind) in enumerate(skf.split(train, train[target])):\n    X_train, X_val = train[sparse_features].iloc[tr_ind], train[sparse_features].iloc[val_ind]\n    y_train, y_val = train[target].iloc[tr_ind], train[target].iloc[val_ind]\n    train_model_input = {name:X_train[name] for name in feature_names}\n    val_model_input = {name:X_val[name] for name in feature_names}\n    test_model_input = {name:test[name] for name in feature_names}\n    model = DeepFM(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 256), dnn_dropout=0.0, dnn_activation='Mish', dnn_use_bn=False, task='binary')\n    model.compile('adam', loss = 'binary_crossentropy', metrics=[auc], )\n    es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=3, verbose=Verbose, mode='max', baseline=None, restore_best_weights=True)\n    sb = callbacks.ModelCheckpoint('./nn_model.w8', save_weights_only=True, save_best_only=True, verbose=Verbose)\n    clr = CyclicLR(base_lr=1e-7, max_lr = 1e-4, step_size= int(1.0*(test.shape[0])/(Batch_S_T*4)) , mode='exp_range', gamma=1.0, scale_fn=None, scale_mode='cycle')\n    history = model.fit(train_model_input, y_train,\n                        validation_data=(val_model_input, y_val),\n                        batch_size=Batch_S_T, epochs=Epochs, verbose=Verbose,\n                        callbacks=[es, sb, clr],)\n    model.load_weights('./nn_model.w8')\n    val_pred = model.predict(val_model_input, batch_size=Batch_S_P)\n    print(f'validation AUC fold {fold+1} : {round(roc_auc_score(y_val, val_pred), 5)}')\n    oof_pred_deepfm[val_ind] = val_pred.ravel()\n    y_pred_deepfm += model.predict(test_model_input, batch_size=Batch_S_P).ravel() / (N_Splits)\n    K.clear_session()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'OOF AUC : {round(roc_auc_score(train.target.values, oof_pred_deepfm), 5)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_idx = test.id.values\nsubmission = pd.DataFrame.from_dict({\n    'id': test_idx,\n    'target': y_pred_deepfm\n})\nsubmission.to_csv('submission.csv', index=False)\nprint('Submission file saved!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('oof_pred_deepfm.npy',oof_pred_deepfm)\nnp.save('y_pred_deepfm.npy',    y_pred_deepfm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}