{"cells":[{"metadata":{},"cell_type":"markdown","source":"this is an example of using target encoding with keras neural network to solve this problem. if you are interesting in *my solution with logit and cat boost*, see [this notebook](https://www.kaggle.com/fkdplc/ensembling-logisticregression-and-catboost)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport math\nfrom tqdm.notebook import tqdm\nfrom sklearn.exceptions import ConvergenceWarning\nimport warnings\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.simplefilter(action='ignore', category=ConvergenceWarning)\n\nPATH = '/kaggle/input/cat-in-the-dat-ii/'\ntrain = pd.read_csv(PATH + 'train.csv')\ntest = pd.read_csv(PATH + 'test.csv')\n\n# separate target, remove id and target\ntest_ids = test['id']\ntarget = train['target']\ntrain.drop(columns=['id', 'target'], inplace=True)\ntest.drop(columns=['id'], inplace=True)\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import category_encoders as ce\n\nte = ce.TargetEncoder(cols=train.columns.values, smoothing=0.3).fit(train, target)\n\ntrain = te.transform(train)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train, x_test, y_train, y_test = train_test_split(train, target, test_size=0.2, random_state=289)\n\nx_train.shape, x_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"this is the main code of snapshot ensembling method. you can see [my tutorial about snapshot ensembles and cosine annealing](https://www.kaggle.com/fkdplc/snapshot-ensemble-tutorial-with-keras)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import Callback\nfrom keras import backend\nfrom keras.models import load_model\n\n# this callback applies cosine annealing, saves snapshots and allows to load them\nclass SnapshotEnsemble(Callback):\n    \n    __snapshot_name_fmt = \"snapshot_%d.hdf5\"\n    \n    def __init__(self, n_models, n_epochs_per_model, lr_max, verbose=1):\n        \"\"\"\n        n_models -- quantity of models (snapshots)\n        n_epochs_per_model -- quantity of epoch for every model (snapshot)\n        lr_max -- maximum learning rate (snapshot starter)\n        \"\"\"\n        self.n_epochs_per_model = n_epochs_per_model\n        self.n_models = n_models\n        self.n_epochs_total = self.n_models * self.n_epochs_per_model\n        self.lr_max = lr_max\n        self.verbose = verbose\n        self.lrs = []\n \n    # calculate learning rate for epoch\n    def cosine_annealing(self, epoch):\n        cos_inner = (math.pi * (epoch % self.n_epochs_per_model)) / self.n_epochs_per_model\n        return self.lr_max / 2 * (math.cos(cos_inner) + 1)\n\n    # when epoch begins update learning rate\n    def on_epoch_begin(self, epoch, logs={}):\n        # update learning rate\n        lr = self.cosine_annealing(epoch)\n        backend.set_value(self.model.optimizer.lr, lr)\n        # log value\n        self.lrs.append(lr)\n\n    # when epoch ends check if there is a need to save a snapshot\n    def on_epoch_end(self, epoch, logs={}):\n        if (epoch + 1) % self.n_epochs_per_model == 0:\n            # save model to file\n            filename = self.__snapshot_name_fmt % ((epoch + 1) // self.n_epochs_per_model)\n            self.model.save(filename)\n            if self.verbose:\n                print('Epoch %d: snapshot saved to %s' % (epoch, filename))\n                \n    # load all snapshots after training\n    def load_ensemble(self):\n        models = []\n        for i in range(self.n_models):\n            models.append(load_model(self.__snapshot_name_fmt % (i + 1)))\n        return models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Dropout, LeakyReLU\n\nmodel = Sequential()\nmodel.add(Dense(32, input_shape=(train.shape[1], )))\nmodel.add(LeakyReLU())\nmodel.add(Dropout(0.5))\nmodel.add(Dense(16))\nmodel.add(LeakyReLU())\nmodel.add(Dropout(0.5))\nmodel.add(Dense(1, activation='sigmoid'))\n\nmodel.summary()\n\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer='adam',\n    metrics=['acc']\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"se_callback = SnapshotEnsemble(n_models=7, n_epochs_per_model=15, lr_max=.01)\n\nhistory = model.fit(\n    x_train,\n    y_train,\n    epochs=se_callback.n_epochs_total,\n    verbose=1,\n    batch_size=32,\n    callbacks=[se_callback],\n    validation_data=(x_test, y_test)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\n\nh = history.history\nplt.figure(1, figsize=(16, 10))\n\nplt.subplot(121)\nplt.xlabel('epoch')\nplt.ylabel('loss')\nplt.plot(h['loss'], label='training')\nplt.plot(h['val_loss'], label='validation')\nplt.legend()\n\nplt.subplot(122)\nplt.xlabel('epoch')\nplt.ylabel('accuracy')\nplt.plot(h['acc'], label='training')\nplt.plot(h['val_acc'], label='validation')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\n# makes prediction according to given models and given weights\ndef predict(models, data, weights=None):\n    if weights is None:\n        # default weights provide voting equality\n        weights = [1 / (len(models))] * len(models)\n    pred = np.zeros((data.shape[0], ))\n    for i, model in enumerate(models):\n        pred += model.predict(data).flatten() * weights[i]\n    return pred\n\n# returns roc auc for preds and weights\ndef evaluate(preds, weights=None):\n    if weights is None:\n        weights = [1 / len(preds)] * len(preds)\n    y_true = np.zeros((y_test.shape[0], ))\n    for i, pred in enumerate(preds):\n        y_true += pred.flatten() * weights[i]\n    return roc_auc_score(y_test, y_true)\n\n# load list of snapshots\nmodels = se_callback.load_ensemble()\npreds = []\n# evaluate every model as single\nfor i, model in enumerate(models):\n    pred = predict([model], x_test)\n    preds.append(pred)\n    score = evaluate([pred])\n    print(f'model {i + 1}: roc = {score:.4f}')\n\n# evaluate ensemble (with voting equality)\nensemble_score = evaluate(preds)\nprint(f'ensemble: roc = {ensemble_score:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_score = ensemble_score\nbest_weights = None\nno_improvements = 0\nwhile no_improvements < 500: #patience\n    \n    # generate normalized weights\n    new_weights = np.random.uniform(size=(len(models), ))\n    new_weights /= new_weights.sum()\n    \n    # get the score (no extra predictions)\n    new_score = evaluate(preds, new_weights)\n    \n    # check (and save)\n    if new_score > best_score:\n        no_improvements = 0\n        best_score = new_score\n        best_weights = new_weights\n        print(f'improvement: {best_score:.4f}')\n    else:\n        no_improvements += 1\n\n\nprint(f'best weights are {best_weights}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# transform test and predict\ntest = te.transform(test)\npred = predict(models, test, best_weights)\n\nres = pd.DataFrame()\nres['id'] = test_ids\nres['target'] = pred\nres.to_csv('submission.csv', index=False)\nres.head(15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h3>thank you for reading till the end! i hope you liked it!</h3>"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}