{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import libraries and data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy as sp\nfrom scipy import stats\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Preprocessing, modelling and evaluating\nfrom sklearn import preprocessing\nfrom sklearn.metrics import confusion_matrix, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score, KFold\nfrom xgboost import XGBClassifier\nimport xgboost as xgb\n\n## Hyperopt modules\nfrom hyperopt import fmin, hp, tpe, Trials, space_eval, STATUS_OK, STATUS_RUNNING\nfrom functools import partial\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# Any results you write to the current directory are saved as output.\n\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\ntest = pd.read_csv(\"../input/cat-in-the-dat-ii/test.csv\")\nsubmission = pd.read_csv(\"../input/cat-in-the-dat-ii/sample_submission.csv\", index_col='id')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data profile"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas_profiling as pp\nsample_profile=train.sample(frac=0.01)\npp.ProfileReport(sample_profile)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Interaction testing"},{"metadata":{"trusted":true},"cell_type":"code","source":"from statsmodels.formula.api import ols\nimport statsmodels.stats.multicomp\nall_cols = train.columns.tolist()  \nall_cols.remove('id')\nall_cols.remove('target')\n\n# Fits the model with the interaction term\n# This will also automatically include the main effects for each factor\n#for col1 in all_cols:\n#    for col2 in all_cols:\n#        formula=f\"target~C(\"+col1+f\")*C(\"+all_cols[0]+f\")\"\n#        model = ols(formula, train).fit()\n#        print(col1 + f\"*\" + col2 + f\" p = {model.f_pvalue: .4f}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Target encode everything to begin with:"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Source: https://www.kaggle.com/caesarlupum/2020-20-lines-target-encoding\n#train.sort_index(inplace=True)\ntrain_y = train['target']\ntest_id = test['id']\ntt=train.drop(['target', 'id'], axis=1)\nte=test.drop('id', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from sklearn.metrics import roc_auc_score\ncat_feat_to_encode = tt.columns.tolist()  \n\nimport category_encoders as ce\noof = pd.DataFrame([])\nfrom sklearn.model_selection import StratifiedKFold\n\nfor tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True).split(tt, train_y):\n    ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=0.2)\n    ce_target_encoder.fit(tt.iloc[tr_idx, :], train_y.iloc[tr_idx])\n    oof = oof.append(ce_target_encoder.transform(tt.iloc[oof_idx, :]), ignore_index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=0.2) \nce_target_encoder.fit(tt, train_y) \ntt = oof.sort_index()\nte = ce_target_encoder.transform(te)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tt = tt.add_suffix('_tt')\nte = te.add_suffix('_tt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge train and test"},{"metadata":{"trusted":true},"cell_type":"code","source":"trest = train.drop('target', axis=1).append(test)\ntrest.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trest['missing_count'] = trest.apply(lambda x: x.count()/trest.shape[1], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#merge test and train for later OHE\ntrest1 = trest.drop('id', axis=1)\n#test1 = test.drop('id', axis=1)\ntrest1.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ordinals into numerics"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Only ord_0 is numerical values;\n#We need to transform ord_1, ord_2 and ord_3 to set it in the correctly order to feed the machine learning model\n\nord_cols = ['ord_0', 'ord_1', 'ord_2', 'ord_3']\nfrom pandas.api.types import CategoricalDtype \n\n# seting the orders of our ordinal features\nord_1 = CategoricalDtype(categories=['Novice', 'Contributor','Expert','Master', 'Grandmaster'], ordered=True)\nord_2 = CategoricalDtype(categories=['Freezing', 'Cold', 'Warm', 'Hot','Boiling Hot', 'Lava Hot'], ordered=True)\nord_3 = CategoricalDtype(categories=['a', 'b', 'c', 'd', 'e', 'f', 'g','h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'], ordered=True)\nord_4 = CategoricalDtype(categories=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I','J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R','S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'], ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transforming ordinal Features\n#news=list([ord_1,ord_2,ord_3,ord_4])\n#for i in range(3):\n#    train1[ord_cols[i]]=train1[ord_cols[i]].astype(news[i])\n#    train1[ord_cols[i]]=train1[ord_cols[i]].cat.codes\n\ntrest1.ord_1 = trest1.ord_1.astype(ord_1)\ntrest1.ord_2 = trest1.ord_2.astype(ord_2)\ntrest1.ord_3 = trest1.ord_3.astype(ord_3)\ntrest1.ord_4 = trest1.ord_4.astype(ord_4)\n\n# test dataset\n#test1.ord_1 = test1.ord_1.astype(ord_1)\n#test1.ord_2 = test1.ord_2.astype(ord_2)\n#test1.ord_3 = test1.ord_3.astype(ord_3)\n#test1.ord_4 = test1.ord_4.astype(ord_4)\n\n# Getting the codes of ordinal categoy's - train\ntrest1.ord_1 = trest1.ord_1.cat.codes\ntrest1.ord_2 = trest1.ord_2.cat.codes\ntrest1.ord_3 = trest1.ord_3.cat.codes\ntrest1.ord_4 = trest1.ord_4.cat.codes\n\n# Geting the codes of ordinal categoy's - test\n#test1.ord_1 = test1.ord_1.cat.codes\n#test1.ord_2 = test1.ord_2.cat.codes\n#test1.ord_3 = test1.ord_3.cat.codes\n#test1.ord_4 = test1.ord_4.cat.codes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Handmade imputation from a great R notebook: https://www.kaggle.com/ccccat/let-s-overfit-some"},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/ccccat/let-s-overfit-some\ntrest1['ord_0']=trest1['ord_0'].replace(-1,2.01)\ntrest1['ord_1']=trest1['ord_1'].replace(-1, 1.86)\ntrest1['ord_2']=trest1['ord_2'].replace(-1,2.37)\n\ntrest1['ord_5']=trest1['ord_5'].fillna('Zx')\ntrest1=trest1.fillna(-1)\ntrest1.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Frequency encode ord_5 after imputing NAs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/lucamassaron/categorical-feature-encoding-with-tensorflow\n# Enconding frequencies for some columns\n\n#def frequency_encoding(column, df, df_test=None):\n#    frequencies = df[column].value_counts().reset_index()\n#    df_values = df[[column]].merge(frequencies, how='left',left_on=column, right_on='index').iloc[:,-1].values\n#    if df_test is not None:\n#        df_test_values = df_test[[column]].merge(frequencies, how='left', left_on=column, right_on='index').fillna(1).iloc[:,-1].values\n#    else:\n#        df_test_values = None\n#    return df_values, df_test_values\n\n#freq_encoded = list()\n\n#for column in ['ord_5']:\n#    train_values, test_values = frequency_encoding(column, train1, test1)\n#    train1[column+'_counts'] = train_values/ len(train1)\n#    test1[column+'_counts'] = test_values/ len(test1)\n#    freq_encoded.append(column+'_counts')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make ord5 numerical"},{"metadata":{"trusted":true},"cell_type":"code","source":"s=trest1['ord_5'].map(str).unique().tolist()\ns.sort()\nn=range(0, len(s))\nsn=pd.concat([pd.DataFrame(s),pd.DataFrame(n)],axis=1)\nsn.columns = ['ord_5', 'ord_5.1']\n\ntrest1['ord_5']=trest1['ord_5'].map(str)\ntrest1=pd.merge(trest1,sn,on='ord_5', how='left')\ntrest1=trest1.drop(['ord_5'], axis=1)\n\n#test1['ord_5']=test1['ord_5'].map(str)\n#test1=pd.merge(test1,sn,on='ord_5', how='left')\n#test1=test1.drop(['ord_5'], axis=1)\n\ntrest1.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make dummy variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"dft = pd.get_dummies(trest1, columns=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4','nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','day', 'month','ord_0','ord_1','ord_2'])\n#dfte = pd.get_dummies(test1, columns=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4','nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4','day', 'month'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hash encoding nominal high cardinality features - DONT DO IT much regret"},{"metadata":{"trusted":true},"cell_type":"code","source":"high_card_feats = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"from sklearn.feature_extraction import FeatureHasher\nfh = FeatureHasher(n_features=12, input_type='string')\n#dft1[high_card_feats] = dft1[high_card_feats].apply(lambda x: x.astype(str))\n#dfte1[high_card_feats] = dfte1[high_card_feats].apply(lambda x: x.astype(str))\n#tt1=pd.DataFrame([])\n#te1=pd.DataFrame([])\n\n#for col in high_card_feats:\n#    hashed_features = fh.fit_transform(dft1[col])\n#    hashed_features = hashed_features.toarray()\n#    hashed_features = pd.DataFrame(hashed_features)\n#    hashed_features = hashed_features.add_prefix(col)\n#    tt1=pd.concat([tt1,hashed_features],axis=1)\n    \n#    hashed_features_test = fh.fit_transform(dfte1[col])\n#    hashed_features_test = hashed_features_test.toarray()\n#    hashed_features_test = pd.DataFrame(hashed_features_test)\n#    hashed_features_test = hashed_features_test.add_prefix(col)\n#    te1=pd.concat([te1,hashed_features_test],axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge dft3 and target encoded variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"#target encodings\n\ntte = pd.concat([tt,te],axis=0)\ntte=tte.reset_index(drop=True)\ndft4=pd.concat([dft,tte], axis=1)\ndft4.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Interactions NAH"},{"metadata":{"trusted":true},"cell_type":"code","source":"#dft4['ord_02']=dft4['ord_0'].map(str)+dft4['ord_2'].map(str)\n#dfte4['ord_02']=dfte4['ord_0'].map(str)+dfte4['ord_2'].map(str)\n#dft4['ord_01']=dft4['ord_0'].map(str)+dft4['ord_1'].map(str)\n#dfte4['ord_01']=dfte4['ord_0'].map(str)+dfte4['ord_1'].map(str)\n\n#numericals\n#dft4['ord_02']=dft4['ord_0_tt']*dft4['ord_2_tt']\n#dfte4['ord_02']=dfte4['ord_0']*dfte4['ord_2_tt']\n#dft4['ord_01']=dft4['ord_0_tt']*dft4['ord_1_tt']\n#dfte4['ord_01']=dfte4['ord_0_tt']*dfte4['ord_1_tt']\n\n#dft4['nomord_05']=dft4['ord_0_tt']*dft4['nom_5_tt']\n#dfte4['nomord_05']=dfte4['ord_0_tt']*dfte4['nom_5_tt']\n#dft4['nomord_09']=dft4['ord_0_tt']*dft4['nom_9_tt']\n#dfte4['nomord_09']=dfte4['ord_0_tt']*dfte4['nom_9_tt']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#drop large noms for now\ndft4=dft4.drop(high_card_feats, axis = 1)\n#dfte4=dfte4.drop(high_card_feats, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dft4 = pd.get_dummies(dft4, columns=['ord_02'],prefix=['ord_02'], drop_first=True, dummy_na=True) #drop muuda false\n#dfte4 = pd.get_dummies(dfte4, columns=['ord_02'],prefix=['ord_02'], drop_first=True, dummy_na=True)\n#dft4 = pd.get_dummies(dft4, columns=['ord_01'],prefix=['ord_01'], drop_first=True, dummy_na=True) #drop muuda false\n#dfte4 = pd.get_dummies(dfte4, columns=['ord_01'],prefix=['ord_01'], drop_first=True, dummy_na=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dft4.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X=pd.DataFrame(dft4.iloc[0:600000])\nXe=pd.DataFrame(dft4.iloc[600000:1000000])\ny=train['target']\ny = y.astype(bool)\n#X_train,X_test,y_train,y_test=train_test_split(X,y,random_state=42,test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's see which variables to drop:"},{"metadata":{},"cell_type":"markdown","source":"Old one:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#X1=dft4.drop(['id','target',\n           #  'ord_0','ord_1','ord_2',\n#             'ord_3','ord_4',\n#             'nom_0_nan','nom_1_nan','nom_2_nan','nom_3_nan','nom_4_nan','day','month',\n           # 'ord_02_nan','ord_02_-1.02','ord_02_1.04','ord_02_2.02',\n           # 'ord_0_tt','ord_02_2.0-1','ord_02_-1.03','ord_02_3.03',\n            # 'ord_01_nan','ord_01_3.02','ord_01_3.0-1','ord_01_2.03','ord_01_-1.01','ord_01_1.02',\n#              'ord_0_nan','ord_1_nan', 'ord_2_nan','ord_0_2.0'\n            # 'ord_02_3.0-1','ord_02_2.03','ord_01_2.02','ord_01_2.0-1','ord_01_1.03','ord_01_1.0-1','ord_01_-1.00'\n#           ],axis=1)\n#Xe1=dfte4.drop(['id',\n            #   'ord_0','ord_1','ord_2',\n#               'ord_3','ord_4',\n#               'nom_0_nan','nom_1_nan','nom_2_nan','nom_3_nan','nom_4_nan','day','month',\n          #  'ord_02_nan','ord_02_-1.02','ord_02_1.04','ord_02_2.02',\n          #  'ord_0_tt','ord_02_2.0-1','ord_02_-1.03','ord_02_3.03',\n           #  'ord_01_nan','ord_01_3.02','ord_01_3.0-1','ord_01_2.03','ord_01_-1.01','ord_01_1.02',\n#              'ord_0_nan','ord_1_nan', 'ord_2_nan','ord_0_2.0'\n           #  'ord_02_3.0-1','ord_02_2.03','ord_01_2.02','ord_01_2.0-1','ord_01_1.03','ord_01_1.0-1','ord_01_-1.00'\n#             ],axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After much OH:"},{"metadata":{"trusted":true},"cell_type":"code","source":"X1=X.drop(['ord_3','month_3.0','ord_4','nom_1_Star','ord_5_tt'#,'ord_2_0.0'#,'missing_count',\n     #        'ord_01_2.01.86','ord_01_nan','ord_01_2.04.0','ord_01_2.01.0','ord_01_-1.01.0','ord_02_2.04.0','ord_02_nan','ord_02_-1.05.0',\n      #       'ord_02_1.00.0','ord_01_-1.03.0','ord_5_tt','ord_01_-1.04.0','ord_01_2.00.0','ord_01_-1.02.0'\n              ],axis=1)\nXe1=Xe.drop(['ord_3','month_3.0','ord_4','nom_1_Star'#,'ord_5_tt'#,'ord_2_0.0'#,'missing_count',\n        #     'ord_01_2.01.86','ord_01_nan','ord_01_2.04.0','ord_01_2.01.0','ord_01_-1.01.0','ord_02_2.04.0','ord_02_nan','ord_02_-1.05.0',\n       #      'ord_02_1.00.0','ord_01_-1.03.0','ord_5_tt','ord_01_-1.04.0','ord_01_2.00.0','ord_01_-1.02.0'\n              ],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import statsmodels.api as sm\nX2 = sm.add_constant(X1)\nest = sm.OLS(y, X2)\nest2 = est.fit()\nprint(est2.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Calculate multicollinearity between variable (be patient):"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Code for VIF Calculation\n#https://statinfer.com/204-1-9-issue-of-multicollinearity-in-python/\n#Writing a function to calculate the VIF values\nimport statsmodels.formula.api as sm1\n\ndef vif_cal(input_data):\n    x_vars=input_data\n    xvar_names=x_vars.columns\n    for i in range(0,xvar_names.shape[0]):\n        y=x_vars[xvar_names[i]] \n        x=x_vars[xvar_names.drop(xvar_names[i])]\n        rsq=sm1.ols(formula=\"y~x\", data=x_vars).fit().rsquared  \n        vif=round(1/(1-rsq),2)\n        print (xvar_names[i], \" VIF = \" , vif)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Calculating VIF values using that function\n#Multicollinearity not an issue for dummy encoded factors\n\n#vif_cal(input_data=X)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Logistic regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_selection import f_regression\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    X_train,X_test,y_train,y_test=train_test_split(X1,y,random_state=42,test_size=0.2)\n    lr=LogisticRegression(C=1, class_weight=\"balanced\")\n    lr.fit(X_train,y_train)\n    #y_pre=lr.predict(X_test)\n    #print('Accuracy : ',accuracy_score(y_test,y_pre))\n    preds_val = lr.predict_proba(X_test)[:,1]\n    score = roc_auc_score(y_test ,preds_val)\n    print(\"score: %f\" % (score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"0.789246"},{"metadata":{"trusted":true},"cell_type":"code","source":"#pd.DataFrame({'id': test_id, 'target': lr.predict_proba(Xe1)[:,-1]}).to_csv('submission.csv', index=False) #logreg\n#pd.DataFrame({'id': test_id, 'target': glm.predict_proba(Xe1)[:,-1]}).to_csv('submission.csv', index=False) #GLM","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## XGboost "},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb\nfrom sklearn.metrics import roc_auc_score\nX_train,X_test,y_train,y_test=train_test_split(X1,y,random_state=42,test_size=0.2)\nmodel = xgb.XGBClassifier(objective ='binary:logistic',\n                      learning_rate = 0.3,\n                      max_depth = 3,\n                      n_estimators = 100,\n                      scale_pos_weight = 2,\n                      random_state = 2020,\n                      subsample = 0.8,\n                      colsample_bytree=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#model.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=True,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#preds_val = model.predict_proba(X_test)[:,1]\n#score = roc_auc_score(y_test ,preds_val)\n#print(\"score: %f\" % (score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"0.7865 XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n              colsample_bynode=1, colsample_bytree=0.7, gamma=0,\n              learning_rate=0.3, max_delta_step=0, max_depth=3,\n              min_child_weight=1, missing=None, n_estimators=400, n_jobs=1,\n              nthread=None, objective='binary:logistic', random_state=2020,\n              reg_alpha=0, reg_lambda=1, scale_pos_weight=2, seed=None,\n              silent=None, subsample=0.8, verbosity=1)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#pd.DataFrame({'id': test_id, 'target': model.predict_proba(Xe1)[:,-1]}).to_csv('submission.csv', index=False) #xgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Catboost with 3f validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install --upgrade scikit-learn\nimport category_encoders as ce\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics, preprocessing\nfrom sklearn.metrics import auc\n#from sklearn.metrics import plot_roc_curve\nimport datetime\nfrom time import time\nfrom catboost import CatBoostClassifier\nfrom sklearn.experimental import enable_hist_gradient_boosting  # noqa\nfrom sklearn.ensemble import HistGradientBoostingClassifier\nfrom sklearn.linear_model import LogisticRegression\n#from sklearn.ensemble import StackingClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_classifier():\n    clf = CatBoostClassifier(\n                               loss_function='CrossEntropy',\n                               eval_metric=\"AUC\",\n                               task_type=\"CPU\",\n                               learning_rate=0.05,\n                               n_estimators =500,   \n                               early_stopping_rounds=10,\n                               random_seed=2019,\n                               silent=True\n                              )\n        \n    return clf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#X_train,X_test,y_train,y_test=train_test_split(X1,y,random_state=42,test_size=0.2)\ny1=train['target']\nscoring = \"roc_auc\"\n\nfolds = StratifiedKFold(n_splits=3, shuffle=True, random_state=1)\nfold_preds = np.zeros([Xe1.shape[0],3])\noof_preds = np.zeros([X1.shape[0],3])\nresults = {}\n\n# Fit Folds\nfor i, (trn_idx, val_idx) in enumerate(folds.split(X1,y1)):\n    print(f\"Fold {i} stacking....\")\n    clf = make_classifier()\n    clf.fit(X1.loc[trn_idx,:], y1.loc[trn_idx])\n    tmp_pred = clf.predict_proba(X1.loc[val_idx,:])[:,1]\n    \n    oof_preds[val_idx,0] = tmp_pred \n    fold_preds[:,0] += clf.predict_proba(Xe1)[:,1] / folds.n_splits\n        \n    estimator_performance = {}\n    estimator_performance['stack_score'] = metrics.roc_auc_score(y1.loc[val_idx], tmp_pred)\n    \nprint(estimator_performance)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"0.78628"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['target'] =fold_preds[:,0] #preds\nsubmission.to_csv('submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Blend logreg and CB"},{"metadata":{"trusted":true},"cell_type":"code","source":"blend=0.5*lr.predict_proba(Xe1)[:,-1]+0.5*fold_preds[:,0]\nsubmission['target']=blend\nsubmission.to_csv('submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.3"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":true,"toc_position":{"height":"calc(100% - 180px)","left":"10px","top":"150px","width":"165px"},"toc_section_display":true,"toc_window_display":true}},"nbformat":4,"nbformat_minor":4}