{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"target\"] = -1\ndata = pd.concat([train, test]).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [x for x in data.columns if x not in [\"id\", \"target\"]]\nfeature_map = {}\ncnt = 0\nfor feature in features:\n    feature_unique = data[feature].unique()\n    d = dict(zip(feature_unique, range(cnt, len(feature_unique) + cnt)))\n    cnt += len(feature_unique)\n    feature_map[feature] = d","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(cnt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for feature in features:\n    train[feature] = train[feature].map(feature_map[feature])\n    test[feature] = test[feature].map(feature_map[feature])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = train[features]\ntrain_label = train[['target']]\ntest_data = test[features]\nprint(\"train_data.shape = \", train_data.shape)\nprint(\"train_label.shape = \", train_label.shape)\nprint(\"test_data.shape = \", test_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport joblib\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics, preprocessing\nfrom keras import layers\nfrom keras import optimizers\nfrom keras.models import Model, load_model\nfrom keras import callbacks\nfrom keras import backend as K\nfrom keras import utils\nfrom keras.layers import Dense, Dropout, Activation\nfrom keras.optimizers import Adam\nfrom keras.layers import Flatten, Input, Embedding, BatchNormalization\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def auc(y_true, y_pred):\n    def fallback_auc(y_true, y_pred):\n        try:\n            return metrics.roc_auc_score(y_true, y_pred)\n        except:\n            return 0.5\n    return tf.py_function(fallback_auc, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(): \n    inp = layers.Input(shape=(23,))\n    x = layers.Embedding(5725, 10)(inp)\n    x = layers.Flatten()(x)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Dense(500, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.BatchNormalization()(x)\n    \n    x = layers.Dense(300, activation=\"relu\")(x)\n    x = layers.Dropout(0.3)(x)\n    x = layers.BatchNormalization()(x)\n    \n    y = layers.Dense(2, activation=\"softmax\")(x)\n\n    model = Model(inputs=inp, outputs=y)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = create_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_preds = np.zeros((len(train_data)))\ntest_preds = np.zeros((len(test_data)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nskf = StratifiedKFold(n_splits=10)\nk = 0\nfor train_index, test_index in skf.split(train_data, train_label):\n    x_train, x_test = train_data.iloc[train_index, :], train_data.iloc[test_index, :]\n    y_train, y_test = train_label.iloc[train_index, :], train_label.iloc[test_index, :]\n    model = create_model()\n    adam = Adam(learning_rate=0.0001, beta_1=0.9, beta_2=0.999, amsgrad=False)\n    model.compile(loss='binary_crossentropy', optimizer=adam, metrics=[auc])\n    lr = callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.1,\n                                     patience=3, min_lr=1e-6, mode='max', verbose=1)\n    es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5,\n                                 verbose=1, mode='max', baseline=None, restore_best_weights=True)\n    model.fit(x_train,\n              utils.to_categorical(y_train),\n              validation_data=(x_test, utils.to_categorical(y_test)),\n              verbose=1,\n              batch_size=1024,\n              callbacks=[lr, es],\n              epochs=20\n             )\n    valid_fold_preds = model.predict(x_test)[:, 1]\n    test_fold_preds = model.predict(test_data)[:, 1]\n    oof_preds[test_index] = valid_fold_preds.ravel()\n    test_preds += test_fold_preds.ravel()\n    print(\"KFold %d, the best auc is %.4f\" % (k, metrics.roc_auc_score(y_test, valid_fold_preds)))\n    k += 1\n    K.clear_session()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds /= 10\ntest_ids = test.id.values\nprint(\"Saving submission file\")\nsubmission = pd.DataFrame.from_dict({\n    'id': test_ids,\n    'target': test_preds\n})\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}