{"cells":[{"metadata":{},"cell_type":"markdown","source":"Simple CatBoost. (NO SKF, no feature engineering). \n\nParams are borrowed from https://www.kaggle.com/lucamassaron/catboost-in-action-with-dnn\nand number of iterations was increased. It gives currently best result,  \nbetter than other considered, in particular those from: \nhttps://www.kaggle.com/atharvap329/catboost-baseline (at least for a feature set considered here - feautres \"as it is\" - no transforms ).\n\n\nSimilar kernel for LightGBM:\nhttps://www.kaggle.com/alexandervc/lightgbm\nCurrently CatBoost can a little improve that result at least as the same internal \"test\" sample.\nBut actually catboost shows a little worse performnce on public leaderboard\n\nNote: \ncatboost will be speedup-ed by GPU around 20 times. \nyou should use GPU to get result in about 2 minutes.\n\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# from datetime # \nimport datetime\nimport pandas as pd\nimport numpy as np\n\n%matplotlib inline\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n#from sklearn.metrics import precision_score, recall_score, confusion_matrix, roc_curve, precision_recall_curve\nfrom sklearn.metrics import accuracy_score, roc_auc_score, f1_score, auc\n# from sklearn.model_selection import StratifiedKFold\n# import lightgbm as lgb\nfrom sklearn.model_selection import train_test_split \n\n#def read_data(file_path):\nprint('Loading datasets...')\nfile_path = '../input/cat-in-the-dat-ii/'\ntrain = pd.read_csv(file_path + 'train.csv', sep=',')\ntest = pd.read_csv(file_path + 'test.csv', sep=',')\nprint('Datasets loaded')\n# return train, test\n# train, test = read_data(PATH)\n\nprint(train.shape, test.shape)\nprint(train.head(2))\nprint(test.head(2))\n\nX = train.drop(['id','target'], axis = 1)\ncategorical_features = [col for c, col in enumerate(X.columns) \\\n                        if not ( np.issubdtype(X.dtypes[c], np.number )  )  ]\ny = train['target']\n\nprint( len(categorical_features), X.shape, y.shape, y.mean()  )\nX = X.fillna(-9999)\nfor f in categorical_features:\n    X[f] = X[f].astype('category')\n\nX1,X2, y1,y2 = train_test_split(X,y, test_size = 0.2, random_state = 0, stratify = y )\nprint(X1.shape, X2.shape, y1.shape, y2.shape, y1.mean(), y2.mean(), y.mean() )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# params from: https://www.kaggle.com/lucamassaron/catboost-in-action-with-dnn\n\nimport datetime\nfrom catboost import CatBoostClassifier\n\nprint('Start fit.', datetime.datetime.now() )\n\nbest_params = {'bagging_temperature': 0.8,\n               'depth': 5,\n               'iterations': 50000,\n               'l2_leaf_reg': 30,\n               'learning_rate': 0.05,\n               'random_strength': 0.8}\n\nmodel = CatBoostClassifier( **best_params,\n                          loss_function='Logloss',\n                          eval_metric = 'AUC',\n                          nan_mode='Min',\n                          thread_count=4,  task_type = 'GPU',\n                          verbose = False)\n\nmodel.fit(X1 , y1 , eval_set = (X2 , y2), cat_features = categorical_features,\n            verbose_eval=300, \n             early_stopping_rounds=500,\n             use_best_model=True,\n             plot=True)         \n         \npred = model.predict_proba(X2)[:,1]\nscore = roc_auc_score(y2 , pred)\nprint(score)  \nprint('End fit.', datetime.datetime.now() )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Results of launch saved:\n# 0:\tlearn: 0.6827005\ttest: 0.6822092\tbest: 0.6822092 (0)\ttotal: 84.8ms\tremaining: 1h 10m 40s\n# 300:\tlearn: 0.7806345\ttest: 0.7820942\tbest: 0.7820942 (300)\ttotal: 19.6s\tremaining: 54m 2s\n# 600:\tlearn: 0.7833911\ttest: 0.7840392\tbest: 0.7840392 (600)\ttotal: 38.7s\tremaining: 53m 3s\n# 900:\tlearn: 0.7848153\ttest: 0.7844915\tbest: 0.7844916 (898)\ttotal: 59.1s\tremaining: 53m 41s\n# 1200:\tlearn: 0.7859464\ttest: 0.7846847\tbest: 0.7846847 (1200)\ttotal: 1m 18s\tremaining: 53m 11s\n# 1500:\tlearn: 0.7870806\ttest: 0.7848300\tbest: 0.7848300 (1500)\ttotal: 1m 37s\tremaining: 52m 26s\n# 1800:\tlearn: 0.7880922\ttest: 0.7848453\tbest: 0.7848582 (1754)\ttotal: 1m 57s\tremaining: 52m 25s\n# 2100:\tlearn: 0.7891364\ttest: 0.7848671\tbest: 0.7848782 (2079)\ttotal: 2m 17s\tremaining: 52m 5s\n# 2400:\tlearn: 0.7901407\ttest: 0.7849226\tbest: 0.7849235 (2395)\ttotal: 2m 36s\tremaining: 51m 32s\n# 2700:\tlearn: 0.7911730\ttest: 0.7849048\tbest: 0.7849265 (2403)\ttotal: 2m 56s\tremaining: 51m 33s\n# bestTest = 0.7849265337\n# bestIteration = 2403\n# Shrink model to first 2404 iterations.\n# 0.784926509685652\n# End fit. 2020-01-14 20:10:23.686622\n\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = test.drop('id',axis = 1 )\nX_test = X_test.fillna(-99999)\nfor f in categorical_features:\n    X_test[f] = X_test[f].astype('category')\n    \npd.DataFrame({'id': test['id'], 'target': model.predict_proba(X_test)[:,1]}).to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}