{"cells":[{"metadata":{},"cell_type":"markdown","source":"**This kernel uses fgcnn model from deepctr package**\n\n**fgcnn :** [fgcnn using deepctr](https://deepctr-doc.readthedocs.io/en/v0.7.0/deepctr.models.fgcnn.html)\n\ncode forked from https://www.kaggle.com/siavrez/deepfm-model"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-warn-conflicts -q deepctr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from deepctr.inputs import  SparseFeat, DenseFeat, get_feature_names\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.optimizers import Adam,RMSprop\nfrom sklearn.metrics import log_loss, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import utils\nfrom deepctr.models import *\nfrom deepctr.models.fgcnn import FGCNN\nfrom deepctr.models.nffm import NFFM\nimport tensorflow.keras as keras\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport warnings\nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\ntest = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"target\"] = -1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.concat([train, test]).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['null'] = data.isna().sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sparse_features = [feat for feat in train.columns if feat not in ['id','target']]\n\ndata[sparse_features] = data[sparse_features].fillna('-1', )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for feat in sparse_features:\n    lbe = LabelEncoder()\n    data[feat] = lbe.fit_transform(data[feat].fillna(\"-1\").astype(str).values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = data[data.target != -1].reset_index(drop=True)\ntest  = data[data.target == -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique()) for feat in sparse_features]\n\ndnn_feature_columns = fixlen_feature_columns\nlinear_feature_columns = fixlen_feature_columns\n\nfeature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def auc(y_true, y_pred):\n    def fallback_auc(y_true, y_pred):\n        try:\n            return roc_auc_score(y_true, y_pred)\n        except:\n            return 0.5\n    return tf.py_function(fallback_auc, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CyclicLR(keras.callbacks.Callback):\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1 / (2. ** (x - 1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma ** (x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n\n    def clr(self):\n        cycle = np.floor(1 + self.clr_iterations / (2 * self.step_size))\n        x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * self.scale_fn(\n                self.clr_iterations)\n\n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())\n\n    def on_batch_end(self, epoch, logs=None):\n\n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        K.set_value(self.model.optimizer.lr, self.clr())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = ['target']\nN_Splits = 20\nEpochs = 10\nSEED = 2020","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_pred_deepfm = np.zeros((len(train), ))\ny_pred_deepfm = np.zeros((len(test),))\n\n\nskf = StratifiedKFold(n_splits=N_Splits, shuffle=True, random_state=SEED)\nfor fold, (tr_ind, val_ind) in enumerate(skf.split(train, train[target])):\n    X_train, X_val = train[sparse_features].iloc[tr_ind], train[sparse_features].iloc[val_ind]\n    y_train, y_val = train[target].iloc[tr_ind], train[target].iloc[val_ind]\n    train_model_input = {name:X_train[name] for name in feature_names}\n    val_model_input = {name:X_val[name] for name in feature_names}\n    test_model_input = {name:test[name] for name in feature_names}\n    model = NFFM(linear_feature_columns, dnn_feature_columns)\n    model.compile(\"adam\", \"binary_crossentropy\", metrics=[auc], )\n    es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.0001, patience=2, verbose=1, mode='max', baseline=None, restore_best_weights=True)\n    sb = callbacks.ModelCheckpoint('./nn_model.w8', save_weights_only=True, save_best_only=True, verbose=0)\n    clr = CyclicLR(base_lr=0.00001 / 100, max_lr = 0.0001, \n                       step_size= int(1.0*(test.shape[0])/1024) , mode='exp_range',\n                       gamma=1., scale_fn=None, scale_mode='cycle')\n    history = model.fit(train_model_input, y_train,\n                        validation_data=(val_model_input, y_val),\n                        batch_size=512, epochs=Epochs, verbose=1,\n                        callbacks=[es, sb, clr],)\n    model.load_weights('./nn_model.w8')\n    val_pred = model.predict(val_model_input, batch_size=512)\n    print(f\"validation AUC fold {fold+1} : {round(roc_auc_score(y_val, val_pred), 5)}\")\n    oof_pred_deepfm[val_ind] = val_pred.ravel()\n    y_pred_deepfm += model.predict(test_model_input, batch_size=512).ravel() / (N_Splits)\n    K.clear_session()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"OOF AUC : {round(roc_auc_score(train.target.values, oof_pred_deepfm), 5)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_idx = test.id.values\nsubmission = pd.DataFrame.from_dict({\n    'id': test_idx,\n    'target': y_pred_deepfm\n})\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Submission file saved!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('oof_pred_deepfm.npy',oof_pred_deepfm)\nnp.save('y_pred_deepfm.npy',    y_pred_deepfm)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}