{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We want to be sure to have the latest CatBoost version\n\n!pip install catboost -U","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Importing core libraries\nimport numpy as np\nimport pandas as pd\nfrom time import time\nimport pprint\nimport joblib\n\n# Classifiers\nfrom catboost import CatBoostClassifier, Pool\n\n# Model selection\nfrom sklearn.model_selection import StratifiedKFold\n\n# Metrics\nfrom sklearn.metrics import roc_auc_score, average_precision_score\nfrom sklearn.metrics import make_scorer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading the data\nX = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\")\nXt = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Separating target and ids\ny = X.target.values\nid_train = X.id\nid_test = Xt.id\n\nX.drop(['id', 'target'], axis=1, inplace=True)\nXt.drop(['id'], axis=1, inplace=True)\n\n# Classifying variables into binary, high and low cardinality nominal, ordinal and dates\nbinary_vars = [c for c in X.columns if 'bin_' in c]\n\nnominal_vars = [c for c in X.columns if 'nom_' in c]\nhigh_cardinality = [c for c in nominal_vars if len(X[c].unique()) > 16]\nlow_cardinality = [c for c in nominal_vars if len(X[c].unique()) <= 16]\n\nordinal_vars = [c for c in X.columns if 'ord_' in c]\n\ntime_vars = ['day', 'month']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some feature engineering\nX['ord_5_1'] = X['ord_5'].apply(lambda x: x[0] if type(x) == str else np.nan)\nX['ord_5_2'] = X['ord_5'].apply(lambda x: x[1] if type(x) == str else np.nan)\nXt['ord_5_1'] = Xt['ord_5'].apply(lambda x: x[0] if type(x) == str else np.nan)\nXt['ord_5_2'] = Xt['ord_5'].apply(lambda x: x[1] if type(x) == str else np.nan)\n\nordinal_vars += ['ord_5_1', 'ord_5_2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting ordinal labels into ordered values\nordinals = {\n    'ord_1' : {\n        'Novice' : 0,\n        'Contributor' : 1,\n        'Expert' : 2,\n        'Master' : 3,\n        'Grandmaster' : 4\n    },\n    'ord_2' : {\n        'Freezing' : 0,\n        'Cold' : 1,\n        'Warm' : 2,\n        'Hot' : 3,\n        'Boiling Hot' : 4,\n        'Lava Hot' : 5\n    }\n}\n\ndef return_order(X, Xt, var_name):\n    mode = X[var_name].mode()[0]\n    el = sorted(set(X[var_name].fillna(mode).unique())|set(Xt[var_name].fillna(mode).unique()))\n    return {v:e for e, v in enumerate(el)}\n\nfor mapped_var in ordinal_vars:\n    if mapped_var not in ordinals:\n        mapped_values = return_order(X, Xt, mapped_var)\n        X[mapped_var + '_num'] = X[mapped_var].replace(mapped_values)\n        Xt[mapped_var + '_num'] = Xt[mapped_var].replace(mapped_values)\n    else:\n        X[mapped_var + '_num'] = X[mapped_var].replace(ordinals[mapped_var])\n        Xt[mapped_var + '_num'] = Xt[mapped_var].replace(ordinals[mapped_var])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transforming all the labels of all variables\nfrom sklearn.preprocessing import LabelEncoder\n\nlabel_encoders = [LabelEncoder() for _ in range(X.shape[1])]\n\nfor col, column in enumerate(X.columns):\n    unique_values = pd.Series(X[column].append(Xt[column]).unique())\n    unique_values = unique_values[unique_values.notnull()]\n    label_encoders[col].fit(unique_values)\n    X.loc[X[column].notnull(), column] = label_encoders[col].transform(X.loc[X[column].notnull(), column])\n    Xt.loc[Xt[column].notnull(), column] = label_encoders[col].transform(Xt.loc[Xt[column].notnull(), column])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dealing with missing values\nX = X.fillna(-1)\nXt = Xt.fillna(-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Enconding frequencies instead of labels (so we have some numeric variables)\n\ndef frequency_encoding(column, df, df_test=None):\n    frequencies = df[column].value_counts().reset_index()\n    df_values = df[[column]].merge(frequencies, how='left', \n                                   left_on=column, right_on='index').iloc[:,-1].values\n    if df_test is not None:\n        df_test_values = df_test[[column]].merge(frequencies, how='left', \n                                                 left_on=column, right_on='index').fillna(1).iloc[:,-1].values\n    else:\n        df_test_values = None\n    return df_values, df_test_values\n\nfor column in X.columns:\n    train_values, test_values = frequency_encoding(column, X, Xt)\n    X[column+'_counts'] = train_values\n    Xt[column+'_counts'] = test_values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Target encoding of selected variables\nimport category_encoders as cat_encs\n\ncat_feat_to_encode = binary_vars + ordinal_vars + nominal_vars + time_vars\nsmoothing = 0.3\n\nenc_x = np.zeros(X[cat_feat_to_encode].shape)\n\nfor tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True).split(X, y):\n    encoder = cat_encs.TargetEncoder(cols=cat_feat_to_encode, smoothing=smoothing)\n    \n    encoder.fit(X[cat_feat_to_encode].iloc[tr_idx], y[tr_idx])\n    enc_x[oof_idx, :] = encoder.transform(X[cat_feat_to_encode].iloc[oof_idx], y[oof_idx])\n    \nencoder.fit(X[cat_feat_to_encode], y)\nenc_xt = encoder.transform(Xt[cat_feat_to_encode]).values\n\nfor idx, new_var in enumerate(cat_feat_to_encode):\n    new_var = new_var + '_enc'\n    X[new_var] = enc_x[:,idx]\n    Xt[new_var] = enc_xt[:, idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Stacking\n\"\"\"\nX_dnn = pd.read_csv(\"/kaggle/input/categorical-feature-encoding-with-tensorflow/oof.csv\")\nXt_dnn = pd.read_csv(\"/kaggle/input/categorical-feature-encoding-with-tensorflow/dnn_cv_submission.csv\")\n\nX['dnn_preds'] = X_dnn.dnn_oof\nXt['dnn_preds'] = Xt_dnn.target\n\ndel((X_dnn, Xt_dnn))\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting all to dtype float32\nX = X.astype(np.float32)\nXt = Xt.astype(np.float32)\n\n# Defining categorical variables\ncat_features = nominal_vars + ordinal_vars\n\n# Setting categorical variables to int64\nX[cat_features] = X[cat_features].astype(np.int64)\nXt[cat_features] = Xt[cat_features].astype(np.int64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n_ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initializing a CatBoostClassifier with best parameters\nbest_params = {'bagging_temperature': 0.8,\n               'depth': 5,\n               'iterations': 1000,\n               'l2_leaf_reg': 30,\n               'learning_rate': 0.05,\n               'random_strength': 0.8}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setting a n-fold stratified cross-validation (note: shuffle=True)\nSEED = 42\nFOLDS = 20\n\nskf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CV interations\nroc_auc = list()\naverage_precision = list()\noof = np.zeros(len(X))\ncv_test_preds = np.zeros(len(Xt))\nbest_iteration = list()\n\nfor train_idx, test_idx in skf.split(X, y):\n    X_train, y_train = X.iloc[train_idx, :], y[train_idx]\n    X_test, y_test = X.iloc[test_idx, :], y[test_idx]\n    \n    train = Pool(data=X_train, \n             label=y_train,            \n             feature_names=list(X_train.columns),\n             cat_features=cat_features)\n\n    val = Pool(data=X_test, \n               label=y_test,\n               feature_names=list(X_test.columns),\n               cat_features=cat_features)\n\n    catb = CatBoostClassifier(**best_params,\n                          loss_function='Logloss',\n                          eval_metric = 'AUC',\n                          nan_mode='Min',\n                          thread_count=4,\n                          verbose = False)\n    \n    catb.fit(train,\n             verbose_eval=100, \n             early_stopping_rounds=50,\n             eval_set=val,\n             use_best_model=True,\n             #task_type = \"GPU\",\n             plot=False)\n    \n    best_iteration.append(catb.best_iteration_)\n    preds = catb.predict_proba(X_test)\n    oof[test_idx] = preds[:,1]\n    \n    # CV test prediction\n    Xt_pool = Pool(data=Xt[list(X_train.columns)],\n               feature_names=list(X_train.columns),\n               cat_features=cat_features)\n    \n    cv_test_preds += catb.predict_proba(Xt_pool)[:,1] / FOLDS\n    \n    roc_auc.append(roc_auc_score(y_true=y_test, y_score=preds[:,1]))\n    average_precision.append(average_precision_score(y_true=y_test, y_score=preds[:,1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Storing results to disk\noof = pd.DataFrame({'id':id_train, 'catboost_oof': oof})\noof.to_csv(\"oof.csv\", index=False)\n\ncv_submission = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/sample_submission.csv\")\ncv_submission.target = cv_test_preds\ncv_submission.to_csv(\"./catboost_cv_submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Average cv roc auc score %0.3f ± %0.3f\" % (np.mean(roc_auc), np.std(roc_auc)))\nprint(\"Average cv roc average precision %0.3f ± %0.3f\" % (np.mean(average_precision), np.std(average_precision)))\n\nprint(\"Roc auc score OOF %0.3f\" % roc_auc_score(y_true=y, y_score=oof.catboost_oof))\nprint(\"Average precision OOF %0.3f\" % average_precision_score(y_true=y, y_score=oof.catboost_oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Blending\nXt_dnn = pd.read_csv(\"/kaggle/input/categorical-feature-encoding-with-tensorflow/dnn_cv_submission.csv\")\nXt_tab = pd.read_csv(\"/kaggle/input/tensorflow-tabnet/submission.csv\")\ncv_submission.target = cv_submission.target * 0.5 + Xt_dnn.target * 0.35 + Xt_tab.target * 0.15\ncv_submission.to_csv(\"./catboost_dnn_cv_submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}