{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## https://github.com/optuna/optuna","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport optuna\nimport catboost as cb\nimport sklearn\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_cat_features(df):\n    cat_features_list = []\n    object_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()\n    for cf in object_cols:\n        cat_features_list.append(df.columns.get_loc(cf))\n    return cat_features_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/geekbrains-competitive-data-analysis/'\n\ntrain = pd.read_csv(path + \"train.csv\")\ntest = pd.read_csv(path + \"test.csv\")\ncp = pd.read_csv(path + \"client_profile.csv\")\n\nID_COLUMN = 'APPLICATION_NUMBER'\nTARGET = 'TARGET'\n\ntrain_df = train.merge(cp, on=ID_COLUMN, how='left')\ntest_df = test.merge(cp, on=ID_COLUMN, how='left')\n\nX = train_df.drop(TARGET, axis=1).fillna(-1)\ny = train_df[TARGET]\n\ncat_features = get_cat_features(X)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective(trial):\n    train_x, valid_x, train_y, valid_y = train_test_split(X, y, test_size=0.3, random_state=42)\n    \n    param = {\n        \"n_estimators\": 1000,\n        \"eval_metric\": \"AUC\",\n        \"silent\": True,\n        #Tuning\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.1),\n        \"depth\": trial.suggest_int(\"depth\", 3, 3),\n    }\n\n    gbm = cb.CatBoostClassifier(**param)\n\n    gbm.fit(train_x,\n            train_y,\n            cat_features,\n            eval_set=[(valid_x, valid_y)],\n            verbose=0,\n            early_stopping_rounds=None,\n            )\n\n    preds = gbm.predict_proba(valid_x.fillna(-1))[:, 1]\n    roc_auc = roc_auc_score(valid_y, preds)\n    return roc_auc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=10, timeout=600)\n\nprint(\"Number of finished trials: {}\".format(len(study.trials)))\n\nprint(\"Best trial:\")\ntrial = study.best_trial\n\nprint(\"  Value: {}\".format(trial.value))\n\nprint(\"  Params: \")\nfor key, value in trial.params.items():\n    print(\"    {}: {}\".format(key, value))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cb_params = {\n        \"n_estimators\": 1000,\n        \"eval_metric\": \"AUC\",\n        \"silent\": True,\n}\n\ncb_params = {**cb_params, **trial.params}\n\n\nmodel = cb.CatBoostClassifier(**cb_params)\nmodel.fit(X, y, cat_features, plot=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probabilities = model.predict_proba(test_df.fillna(-1))\nsubmission = pd.DataFrame({\n    ID_COLUMN: test[ID_COLUMN],\n    TARGET:     [row[1] for row in probabilities]\n})\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cb_params","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}