{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# SKF LightGBM - Target Encoding"},{"metadata":{},"cell_type":"markdown","source":"* LIBRARIES\n* DATA\n* TARGET ENCODING\n* TIMER + CONFUSION MATRIX\n* SKF LGBM + METRICS \n* PRED CSV"},{"metadata":{},"cell_type":"markdown","source":"## 1. LIBRARIES"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\nimport category_encoders as ce\nfrom datetime import timedelta \nfrom datetime import datetime\nfrom scipy import interp\nimport pandas as pd\nimport numpy as np\nimport itertools\nimport warnings\n\n%matplotlib inline\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import rcParams\n\nfrom sklearn.metrics import precision_score, recall_score, confusion_matrix, roc_curve, precision_recall_curve\nfrom sklearn.metrics import accuracy_score, roc_auc_score, f1_score, auc\nfrom sklearn.model_selection import StratifiedKFold\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"## 2. DATA"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_data(file_path):\n    print('Loading datasets...')\n    train = pd.read_csv(file_path + 'train.csv', sep=',')\n    test = pd.read_csv(file_path + 'test.csv', sep=',')\n    print('Datasets loaded')\n    return train, test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '../input/cat-in-the-dat-ii/'\ntrain, test = read_data(PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Zoom datasets \ndef zoom_dataset(data):\n    Count_missing_val = data.isnull().sum()\n    Percent_missing = (data.isnull().sum()/data.isnull().count()*100)\n    Percent_no_missing = 100 - Percent_missing\n    Count_unique = data.nunique()\n    Percent_unique_val = Count_unique / len(data)*100\n    Type = data.dtypes\n    data=[[i, Counter(data[i][data[i].notna()]).most_common(3)] for i in list(data)]\n    top = pd.DataFrame(data, columns=['Name', 'Top_3']).set_index(['Name'])\n    tt = pd.concat([Percent_no_missing, Count_missing_val, Percent_missing, Count_unique, Percent_unique_val, Type], axis=1, \n                   keys=['Percent_no_missing','Count_missing_val', 'Percent_missing', ' Count_unique', 'Percent_unique_val', 'Type'])\n    tt = pd.concat([tt, top[['Top_3']]], axis=1, sort=False).reset_index()\n    return((tt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#overview train - test\nfor i in [train, test]:\n    display(zoom_dataset(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train\nf,ax=plt.subplots(8,2,figsize=(12,20))\nf.delaxes(ax[7,1])\n\nfor i,feature in enumerate(['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4', 'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4']):\n    colors = ['darkturquoise', 'darkorange']\n    sns.countplot(x=feature,data=train,hue='target',ax=ax[i//2,i%2], palette = colors, alpha=0.7, edgecolor=('black'), linewidth=2)\n    ax[i//2,i%2].grid(b=True, which='major', color='grey', linewidth=0.4)\n    ax[i//2,i%2].set_title('Count of {} vs target - train'.format(feature), fontsize=18)\n    ax[i//2,i%2].legend(loc='best')\n    ax[i//2,i%2].set_ylabel('count', fontsize=12)\n    ax[i//2,i%2].set_xlabel('modality', fontsize=12)\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test\nf,ax=plt.subplots(8,2,figsize=(12,20))\nf.delaxes(ax[7,1])\n\nfor i,feature in enumerate(['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4', 'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4']):\n    colors = ['darkturquoise']\n    sns.countplot(x=feature,data=test,ax=ax[i//2,i%2], palette = colors, alpha=0.8, edgecolor=('black'), linewidth=2)\n    ax[i//2,i%2].grid(b=True, which='major', color='grey', linewidth=0.2)\n    ax[i//2,i%2].set_title('Count of {} - test'.format(feature), fontsize=18)\n    ax[i//2,i%2].set_ylabel('count', fontsize=12)\n    ax[i//2,i%2].set_xlabel('modality', fontsize=12)\n\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. TARGET ENCODING"},{"metadata":{"trusted":true},"cell_type":"code","source":"# CREDITS : https://www.kaggle.com/caesarlupum/2020-20-lines-target-encoding\ndef encoding(train, test, smooth):\n    print('Target encoding...')\n    train.sort_index(inplace=True)\n    target = train['target']\n    test_id = test['id']\n    train.drop(['target', 'id'], axis=1, inplace=True)\n    test.drop('id', axis=1, inplace=True)\n    cat_feat_to_encode = train.columns.tolist()\n    smoothing=smooth\n    oof = pd.DataFrame([])\n    for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True).split(train, target):\n        ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)\n        ce_target_encoder.fit(train.iloc[tr_idx, :], target.iloc[tr_idx])\n        oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)\n    ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)\n    ce_target_encoder.fit(train, target)\n    train = oof.sort_index()\n    test = ce_target_encoder.transform(test)\n    features = list(train)\n    print('Target encoding done!')\n    return train, test, test_id, features, target","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4. TIMER + CONFUSION MATRIX"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Timer\ndef timer(start_time=None):\n    if not start_time:\n        start_time = datetime.now()\n        return start_time\n    elif start_time:\n        thour, temp_sec = divmod((datetime.now() - start_time).total_seconds(), 3600)\n        tmin, tsec = divmod(temp_sec, 60)\n        print('Time taken : %i hours %i minutes and %s seconds.' % (thour, tmin, round(tsec, 2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Confusion matrix\ndef plot_confusion_matrix(cm, classes,\n                          normalize = False,\n                          title = 'Confusion matrix',\n                          cmap = plt.cm.Blues) :\n    plt.imshow(cm, interpolation = 'nearest', cmap = cmap)\n    plt.title(title, fontsize=12)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation = 0)\n    plt.yticks(tick_marks, classes)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])) :\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment = 'center',\n                 color = 'white' if cm[i, j] > thresh else 'black')\n \n    plt.tight_layout()\n    plt.ylabel('True label', fontsize=12)\n    plt.xlabel('Predicted label', fontsize=12)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 5. SKF LGBM + METRICS"},{"metadata":{"trusted":true},"cell_type":"code","source":"# model\ndef run_model(splits, features, target, train):\n    # paramaters / hyperparameters\n    model = lgb.LGBMClassifier(**{\n                    'learning_rate': 0.05,\n                    'feature_fraction': 0.1,\n                    'min_data_in_leaf' : 12,\n                    'max_depth': 3,\n                    'reg_alpha': 1,\n                    'reg_lambda': 1,\n                    'objective': 'binary',\n                    'metric': 'auc',\n                    'n_jobs': -1,\n                    'n_estimators' : 5000,\n                    'feature_fraction_seed': 42,\n                    'bagging_seed': 42,\n                    'boosting_type': 'gbdt',\n                    'verbose': 1,\n                    'is_unbalance': True,\n                    'boost_from_average': False})\n    \n    # Graph size\n    plt.rcParams['figure.figsize']=(6,4)\n    \n    start_time = timer(None)\n\n    print('LGBM modeling...')\n    \n    # Metrics / fold\n    cms= []\n    tprs = []\n    aucs = []\n    y_real = []\n    y_proba = []\n    recalls = []\n    roc_aucs = []\n    mean_tpr = []\n    mean_fpr = []\n    f1_scores = []\n    accuracies = []\n    precisions = []\n\n    oof = np.zeros(len(train))\n    mean_fpr = np.linspace(0,1,100)\n    feature_importance_df = pd.DataFrame()\n    i = 1\n    \n    # Statified K Fold + model\n    folds = StratifiedKFold(n_splits=splits, shuffle=True, random_state=42)\n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)):\n        print('Fold:', fold_ )\n        model = model.fit(train.iloc[trn_idx][features], target.iloc[trn_idx],\n                          eval_set = [(train.iloc[trn_idx][features], target.iloc[trn_idx]), \n                                      (train.iloc[val_idx][features], target.iloc[val_idx])],\n                          verbose = 1000,\n                          eval_metric = 'auc',\n                          early_stopping_rounds = 1000)\n        \n        # oof\n        oof[val_idx] =  model.predict_proba(train.iloc[val_idx][features])[:,1]\n        \n        # Roc curve / fold\n        f = plt.figure(1)\n        fpr, tpr, t = roc_curve(target[val_idx], oof[val_idx])\n        tprs.append(interp(mean_fpr, fpr, tpr))\n        roc_auc = auc(fpr, tpr)\n        aucs.append(roc_auc)\n        plt.plot(fpr, tpr, lw=2, alpha=0.5, label='ROC fold %d (AUC = %0.4f)' % (i,roc_auc))\n\n        # Precion recall / fold\n        g = plt.figure(2)\n        precision, recall, _ = precision_recall_curve(target[val_idx], oof[val_idx])\n        y_real.append(target[val_idx])\n        y_proba.append(oof[val_idx])\n        plt.plot(recall, precision, lw=2, alpha=0.3, label='P|R fold %d' % (i))  \n\n        i= i+1\n\n        # append metric by fold\n        roc_aucs.append(roc_auc_score(target[val_idx], oof[val_idx]))\n        accuracies.append(accuracy_score(target[val_idx], oof[val_idx].round()))\n        recalls.append(recall_score(target[val_idx], oof[val_idx].round()))\n        precisions.append(precision_score(target[val_idx], oof[val_idx].round()))\n        f1_scores.append(f1_score(target[val_idx], oof[val_idx].round()))\n\n        # Confusion matrix \n        cms.append(confusion_matrix(target[val_idx], oof[val_idx].round()))\n\n    # Means\n    print(\n        '\\nCV roc score        : {0:.4f}, std: {1:.4f}.'.format(np.mean(roc_aucs), np.std(roc_aucs)),\n        '\\nCV accuracy score   : {0:.4f}, std: {1:.4f}.'.format(np.mean(accuracies), np.std(accuracies)),\n        '\\nCV recall score     : {0:.4f}, std: {1:.4f}.'.format(np.mean(recalls), np.std(recalls)),\n        '\\nCV precision score  : {0:.4f}, std: {1:.4f}.'.format(np.mean(precisions), np.std(precisions)),\n        '\\nCV f1 score         : {0:.4f}, std: {1:.4f}.'.format(np.mean(f1_scores), np.std(f1_scores)))\n    \n    print('Modeling done!\\n')\n    \n    # Timer end\n    timer(start_time)\n    \n    return model, cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr, train\n\ndef graph_metrics(cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr):\n\n    # Roc curve\n    f = plt.figure(1)\n    plt.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'grey')\n    mean_tpr = np.mean(tprs, axis=0)\n    mean_auc = auc(mean_fpr, mean_tpr)\n    plt.plot(mean_fpr, mean_tpr, color='blue',\n             label=r'Mean ROC (AUC = %0.4f)' % ((mean_auc)),lw=2, alpha=1)\n    plt.grid(b=True, which='major', color='grey', linewidth=0.4)\n    plt.xlabel('False Positive Rate', fontsize=12)\n    plt.ylabel('True Positive Rate', fontsize=12)\n    plt.title('LGBM - ROC by folds', fontsize=18)\n    plt.legend(loc=\"lower right\")\n\n    # Recall precision curve\n    g = plt.figure(2)\n    plt.plot([0,1],[1,0],linestyle = '--',lw = 2,color = 'grey')\n    y_real = np.concatenate(y_real)\n    y_proba = np.concatenate(y_proba)\n    precision, recall, _ = precision_recall_curve(y_real, y_proba)\n    plt.plot(recall, precision, color='blue',\n             label=r'Mean P|R')\n    plt.grid(b=True, which='major', color='grey', linewidth=0.4)\n    plt.xlabel('Recall')\n    plt.ylabel('Precision')\n    plt.title('LGBM P|R curve by folds', fontsize=18)\n    plt.legend(loc=\"lower left\")\n\n    # Confusion matrix\n    plt.rcParams[\"axes.grid\"] = False\n    cm = np.average(cms, axis=0)\n    class_names = [0,1]\n    plt.figure()\n    plot_confusion_matrix(cm, \n                          classes=class_names, \n                          title='LGBM Confusion matrix [averaged/folds]')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Encoding\ntrain, test, test_id, features, target = encoding(train, test, 0.3)\n# Modeling\nmodel, cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr, df_ml = run_model(5, features, target, train)\n# Metrics + dataviz\ngraph_metrics(cms, tprs, aucs, y_real, y_proba, mean_fpr, mean_tpr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6. PRED CSV"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame({'id': test_id, 'target': model.predict_proba(test)[:,1]}).to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}