{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport math\nfrom tqdm.notebook import tqdm\nfrom sklearn.exceptions import ConvergenceWarning\nimport warnings\n\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.simplefilter(action='ignore', category=ConvergenceWarning)\n\nPATH = '/kaggle/input/cat-in-the-dat-ii/'\ntrain = pd.read_csv(PATH + 'train.csv')\ntest = pd.read_csv(PATH + 'test.csv')\n\n# separate target, remove id and target\ntest_ids = test['id']\ntarget = train['target']\ntrain.drop(columns=['id', 'target'], inplace=True)\ntest.drop(columns=['id'], inplace=True)\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import category_encoders as ce\n\nte = ce.TargetEncoder(cols=train.columns.values, smoothing=0.3).fit(train, target)\n\ntrain = te.transform(train)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import GridSearchCV\n\n# split train data\nx_train, x_test, y_train, y_test = train_test_split(\n    train, target,  \n    test_size=0.2, \n    random_state=289\n)\n\n# hyperparams setting\ndef make_search(estimator, params, verbose=1):\n    search = GridSearchCV(estimator, params, cv=5, scoring='roc_auc', verbose=verbose, n_jobs=-1)\n    search.fit(x_train, y_train)\n    results = pd.DataFrame()\n    for k, v in search.cv_results_.items():\n        results[k] = v\n    results = results.sort_values(by='rank_test_score')\n    best_params_row = results[results['rank_test_score'] == 1]\n    mean, std = best_params_row['mean_test_score'].iloc[0], best_params_row['std_test_score'].iloc[0]\n    best_params = best_params_row['params'].iloc[0]\n    if verbose:\n        print('%.4f (%.4f) with params' % (mean, std), best_params)\n    return best_params\n\n# make prediction\ndef predict(estimator, features):\n    return estimator.predict_proba(features)[:, 1]\n\n# calculate auc\ndef auc(estimator):\n    y_pred = predict(estimator, x_test)\n    return roc_auc_score(y_test, y_pred)\n\n# precalculated best params (0.7979)\nbest_params_lr = {\n    'solver': 'lbfgs',\n    'C': 2,\n    'max_iter': 200,\n    'random_state': 289\n}\n\nSEARCH_NOW = False\n\n# this is to reproduce search\nif SEARCH_NOW:\n    params = {\n        'solver': ['lbfgs'],\n        'C': [1, 1.5, 2, 2.5, 3, 3.5, 4, 5],\n        'max_iter': [100, 150, 200, 250, 300],\n        'random_state': [289]\n    }\n    best_params_lr = make_search(LogisticRegression(), params)   \n\n# fit logistic regressor\nprint('training..')\nlr = LogisticRegression()\nlr.set_params(**best_params_lr)\nlr.fit(x_train, y_train)\n\n# check auc\nprint('validation..')\nprint('roc auc = %.4f' % auc(lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catboost import CatBoostClassifier\n\n# precalculated best params (0.7978)\nbest_params_cat = {\n    'max_depth': 2,\n    'n_estimators': 600,\n    'random_state': 289,\n    'verbose': 0\n}\n\nSEARCH_NOW = False\n\n# this is to reproduce search\nif SEARCH_NOW:\n    params = {\n        'max_depth': [2, 3, 4, 5],\n        'n_estimators': [50, 100, 200, 400, 600],\n        'random_state': [289],\n        'verbose': [0]\n    }\n    best_params_cat = make_search(CatBoostClassifier(), params)   \n\n# fit xgb\nprint('training..')\ncat = CatBoostClassifier()\ncat.set_params(**best_params_cat)\ncat.fit(x_train, y_train)\n\n# check auc\nprint('validation..')\nprint('roc auc = %.4f' % auc(cat))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_predict = predict(lr, x_test)\ncat_predict = predict(cat, x_test)\n\nprint('lr auc: %.4f' % roc_auc_score(y_test, lr_predict))\nprint('cat auc: %.4f' % roc_auc_score(y_test, cat_predict))\n\nbest_auc = 0\nbest_off = 0\naucs = []\noffsets = [i * .01 for i in range(101)]\nfor off in tqdm(offsets):\n    off_predict = lr_predict * off + cat_predict * (1 - off)\n    auc = roc_auc_score(y_test, off_predict)\n    aucs.append(auc)\n    if auc > best_auc:\n        best_auc = auc\n        best_off = off\n\nprint('best auc %.4f reached on offset %.2f' % (best_auc, best_off))        \n\nfrom matplotlib import pyplot as plt\nplt.figure(1, figsize=(8, 8))\nplt.xlabel('<- cat ------------- lr ->')\nplt.ylabel('auc')\nplt.plot(offsets, aucs)\nplt.plot([best_off], [best_auc], 'o', color='r', label='best score (%.4f) at offset %.2f' % (best_auc, best_off))\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create submission file\ntest = te.transform(test)\n\nres = pd.DataFrame()\nres['id'] = test_ids\nres['target'] = predict(lr, test) * best_off + predict(cat, test) * (1 - best_off)\nres.to_csv('submission.csv', index=False)\nres.head(20)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}