{"cells":[{"metadata":{},"cell_type":"markdown","source":"CatBoost. (NO SKF, no feature engineering). \nFork of https://www.kaggle.com/alexandervc/simple-catboost-cat-in-dat-ii\n\nNote: \ncatboost will be speedup-ed by GPU around 20 times. \nyou should use GPU to get result in about 2 minutes.\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/scikit-optimize/scikit-optimize.git --user","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import datetime\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport pprint\n\n# % matplotlib inline\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import make_scorer\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\n\nfrom catboost import CatBoostClassifier\nimport random\n\n# Skopt functions\nfrom skopt import BayesSearchCV\nfrom skopt.callbacks import DeadlineStopper, VerboseCallback\nfrom skopt.space import Real, Integer\nfrom time import time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reporting util for different optimizers\ndef report_perf(optimizer, x, y, title, callbacks=None):\n    \"\"\"\n    A wrapper for measuring time and performances of different optmizers\n\n    optimizer = a sklearn or a skopt optimizer\n    X = the training set\n    y = our target\n    title = a string label for the experiment\n    \"\"\"\n    start = time()\n    if callbacks:\n        optimizer.fit(x, y, callback=callbacks)\n    else:\n        optimizer.fit(x, y)\n    d = pd.DataFrame(optimizer.cv_results_)\n    best_score = optimizer.best_score_\n    best_score_std = d.iloc[optimizer.best_index_].std_test_score\n    best_params = optimizer.best_params_\n    print((\n                  title + \" took %.2f seconds,  candidates checked: %d, best CV score: %.3f \"\n                  + u\"\\u00B1\" + \" %.3f\") % (time() - start,\n                                            len(optimizer.cv_results_[\n                                                    'params']),\n                                            best_score,\n                                            best_score_std))\n    print('Best parameters:')\n    pprint.pprint(best_params)\n    print()\n    return best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RANDOM_STATE = 1242\nrandom.seed(RANDOM_STATE)\nnp.random.seed(RANDOM_STATE)\nos.environ['PYTHONHASHSEED'] = str(RANDOM_STATE)\n\nNUM_THREADS = 2\nNUM_THREADS_FILE = 4\nNUM_THREADS_PRED = 2\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def read_data(file_path):\nprint('Loading datasets...')\nfile_path = '../input/cat-in-the-dat-ii/'\ntrain = pd.read_csv(file_path + 'train.csv', sep=',')\ntest = pd.read_csv(file_path + 'test.csv', sep=',')\nprint('Datasets loaded')\n# return train, test\n# train, test = read_data(PATH)\n\nprint(train.shape, test.shape)\nprint(train.head(2))\nprint(test.head(2))\n\nX = train.drop(['id', 'target'], axis=1)\ncategorical_features = [col for c, col in enumerate(X.columns)\n                        if not (np.issubdtype(X.dtypes[c], np.number))]\ny = train['target']\n\nprint(len(categorical_features), X.shape, y.shape, y.mean())\nX = X.fillna(-65500)\nfor f in categorical_features:\n    X[f] = X[f].astype('category')\n\nX1, X2, y1, y2 = train_test_split(X, y, test_size=0.2,\n                                  random_state=RANDOM_STATE, stratify=y)\nprint(X1.shape, X2.shape, y1.shape, y2.shape, y1.mean(), y2.mean(), y.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc = make_scorer(roc_auc_score, greater_is_better=True,\n                      needs_threshold=True)\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)\nclf = CatBoostClassifier(thread_count=NUM_THREADS,\n                         loss_function='Logloss',\n                         cat_features=categorical_features,\n#                          scale_pos_weight = (len(y1)-sum(y1))/sum(y1),\n                         od_type='Iter',\n                         nan_mode='Min',\n                         early_stopping_rounds=300,\n                         iterations=1000,\n#                          subsample=0.5,\n                         eval_metric='AUC',\n                         metric_period=50,\n                         task_type='GPU',\n                         verbose=False\n                         )\n\n# Defining your search space\nsearch_spaces = {  # 'iterations': Integer(10, 1000),\n    'depth': Integer(1, 5),\n    'learning_rate': Real(0.02, 0.6, 'log-uniform'),\n    #                  'random_strength': Real(1e-9, 10, 'log-uniform'),\n    'random_strength': Integer(1, 30000000),\n    #                  'rsm': Real(0.1, 1.0), #cpu only\n    'bagging_temperature': Real(0.1, 2.0),\n#     'one_hot_max_size': Integer(2, 15),\n    'border_count': Integer(10, 255),\n    'min_data_in_leaf': Integer(5, 1000), #gpu only\n    'l2_leaf_reg': Integer(5, 1500000),\n    #                  'scale_pos_weight':Real(0.01, 1.0, 'uniform')\n}\n# Setting up BayesSearchCV\nopt = BayesSearchCV(clf,\n                    search_spaces,\n                    scoring=roc_auc,\n                    cv=skf,\n                    n_iter=100,\n                    n_jobs=1,# use just 1 job with CatBoost in order to avoid segmentation fault\n                    return_train_score=False,\n                    refit=True,\n                    optimizer_kwargs={'base_estimator': 'GP'},\n                    random_state=RANDOM_STATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_params = report_perf(opt, X2, y2, 'CatBoost', # for faster work using x2\n                          callbacks=[VerboseCallback(100),\n                                     DeadlineStopper(60 * 60 * 3)])  # 3 hours","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(best_params)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"print('Start fit.', datetime.datetime.now() )\n\n# params from: https://www.kaggle.com/lucamassaron/catboost-in-action-with-dnn\n\n# best_params = {'bagging_temperature': 0.8,\n#                'depth': 5,\n#                'iterations': 50000,\n#                'l2_leaf_reg': 30,\n#                'learning_rate': 0.05,\n#                'random_strength': 0.8}\n\nmodel = CatBoostClassifier(**best_params,\n                           loss_function='Logloss',\n                           eval_metric='AUC',\n                           nan_mode='Min',\n                           thread_count=4, task_type='GPU',\n                           verbose=False)\n\nmodel.fit(X1, y1, eval_set=(X2, y2), cat_features=categorical_features,\n          verbose_eval=300,\n          early_stopping_rounds=500,\n          use_best_model=True,\n          plot=True)\n\npred = model.predict_proba(X2)[:, 1]\nscore = roc_auc_score(y2, pred)\nprint(score)\nprint('End fit.', datetime.datetime.now())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = test.drop('id', axis=1)\nX_test = X_test.fillna(-65500)\nfor f in categorical_features:\n    X_test[f] = X_test[f].astype('category')\n\npd.DataFrame(\n    {'id': test['id'], 'target': model.predict_proba(X_test)[:, 1]}).to_csv(\n    'submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}