{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nfrom sklearn.preprocessing import OrdinalEncoder,OneHotEncoder\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\nfrom sklearn.preprocessing import StandardScaler\nimport string\nimport category_encoders as ce\nimport time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Benchmark Lightgbm model  \nThis kernel means to provide a reference and starting point for viewers who are intersted in using lightgbm to build model in this research.\n* Load data \n* New features\n* Encoding  \n* Train  \n* Feature importance \n\nPlease upvote if you find it helpful"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"#load data\nrawtrain=pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\nrawtest=pd.read_csv('../input/cat-in-the-dat-ii/test.csv')\nsub=pd.read_csv('../input/cat-in-the-dat-ii/sample_submission.csv')\ntarget=rawtrain['target']\ntrain=rawtrain.drop(['id','target'],axis=1)\ntest=rawtest.drop('id',axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#======encode ordinal\ncate_ord=['ord_1','ord_2']\nfor c in cate_ord:\n    print(rawtrain[c].unique())\nlevelmap={c:i for i,c in enumerate(['Novice','Contributor', 'Expert', 'Master','Grandmaster'])}\ntrain['ord_1']=train['ord_1'].replace(levelmap)\ntest['ord_1']=test['ord_1'].replace(levelmap)\ntempratmap={c:i for i,c in enumerate(['Freezing','Cold', 'Warm','Hot' , 'Boiling Hot' ,'Lava Hot' ])}\ntrain['ord_2']=train['ord_2'].replace(tempratmap)\ntest['ord_2']=test['ord_2'].replace(tempratmap)\nlowermap={c:i for i,c in enumerate(string.ascii_lowercase)}\ntrain['ord_3']=train['ord_3'].replace(lowermap)\ntest['ord_3']=test['ord_3'].replace(lowermap)\nupperletter=rawtrain['ord_4'].unique().tolist()\nupperletter.remove(np.nan)\nupperletter.sort()\nuppermap={c:i for i,c in enumerate(string.ascii_uppercase)}\ntrain['ord_4']=train['ord_4'].replace(uppermap)\ntest['ord_4']=test['ord_4'].replace(uppermap)\n#/ord_5\nalletter=string.ascii_letters\nallmap={c:i for i,c in enumerate(alletter)}\ndef getP(x,p):\n    if pd.isnull(x):\n        return x\n    else:\n        if p==0:\n            return x[0]\n        else:\n            return x[1]\n        \ntrain['ord_5_0']=rawtrain['ord_5'].apply(lambda x: getP(x,0)).replace(allmap)\ntrain['ord_5_1']=rawtrain['ord_5'].apply(lambda x: getP(x,1)).replace(allmap)\ntest['ord_5_0']=rawtest['ord_5'].apply(lambda x: getP(x,0)).replace(allmap)\ntest['ord_5_1']=rawtest['ord_5'].apply(lambda x: getP(x,1)).replace(allmap)\ntrain=train.drop('ord_5',axis=1)\ntest=test.drop('ord_5',axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Inspired by this post:https://www.kaggle.com/teejmahal20/clustering-categorical-data-k-modes-cat-ii  \n\nResult clusters are used as a new features"},{"metadata":{"trusted":false},"cell_type":"code","source":"#======encode binary and nominal+label to num for k mode clustering\nnormcol59=['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\ntrain_cluster=train.drop(normcol59,axis=1)\ntest_cluster=test.drop(normcol59,axis=1)\nfor c in train_cluster.columns:\n    test_cluster[c].fillna(train_cluster[c].mode()[0], inplace = True)\n    train_cluster[c].fillna(train_cluster[c].mode()[0], inplace = True)\n\nbincol_labeled=['bin_3', 'bin_4']\nbinOE=OrdinalEncoder()\ntrain_cluster[bincol_labeled]=binOE.fit_transform(train_cluster[bincol_labeled])\ntest_cluster[bincol_labeled]=binOE.transform(test_cluster[bincol_labeled])\n\nnormcol_labeled=['nom_0','nom_1','nom_2', 'nom_3', 'nom_4']\nbinOE=OrdinalEncoder()\ntrain_cluster[normcol_labeled]=binOE.fit_transform(train_cluster[normcol_labeled])\ntest_cluster[normcol_labeled]=binOE.transform(test_cluster[normcol_labeled])\n#==================k mode clustering========\nfrom kmodes.kmodes import KModes\nkm = KModes(n_clusters=2, init = \"Cao\", n_init = 1, verbose=1,random_state=1990)\ntrain['cluster'] = km.fit_predict(train_cluster)\ntest['cluster'] = km.predict(test_cluster)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#======target encode binary and norminal\nbincol=['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\nnormcol=['nom_0','nom_1','nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\n#modified from category_encoders\ndef TargetEncode(trainc,testc,targetc, smooth):\n    print('Target encoding...')\n    smoothing=smooth\n    oof = np.zeros(len(trainc))\n    for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=2020, shuffle=True).split(trainc, targetc):\n        train_x=trainc.iloc[tr_idx].reset_index(drop=True)\n        valid_x=trainc.iloc[oof_idx].reset_index(drop=True)\n        target_train=targetc.iloc[tr_idx].reset_index(drop=True)\n        prior = target_train.mean()\n        tmp = target_train.groupby(train_x).agg(['sum', 'count'])\n        tmp['mean'] = tmp['sum'] / tmp['count']\n        smoothing = 1 / (1 + np.exp(-(tmp[\"count\"] - 1) / smoothing))\n        cust_smoothing = prior * (1 - smoothing) + tmp['mean'] * smoothing \n        tmp['smoothing'] = cust_smoothing\n        tmp = tmp['smoothing'].to_dict()\n        oof[oof_idx]=valid_x.map(tmp).values\n    prior = targetc.mean()\n    tmp = targetc.groupby(trainc).agg(['sum', 'count'])\n    tmp['mean'] = tmp['sum'] / tmp['count']\n    smoothing = 1 / (1 + np.exp(-(tmp[\"count\"] - 1) / smoothing))\n    cust_smoothing = prior * (1 - smoothing) + tmp['mean'] * smoothing \n    tmp['smoothing'] = cust_smoothing\n    tmp = tmp['smoothing'].to_dict()\n    testc=testc.map(tmp)\n    return oof, testc\nfor n in normcol+bincol:\n    train[n],test[n]=TargetEncode(train[n],test[n],target,0.3)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build lgb model"},{"metadata":{"trusted":false},"cell_type":"code","source":"floatlist=['ord_1','ord_2','ord_3','ord_4','ord_5_0','ord_5_1']\ntrain[floatlist]=train[floatlist].astype(float)\ntest[floatlist]=test[floatlist].astype(float)\nusedfeatures=test.columns.tolist()\ncat_cols=['cluster']\nfolds = StratifiedKFold(n_splits=10, shuffle=True, random_state=1990)\n\nparams = {#1\n        'learning_rate': 0.05,\n        'feature_fraction': 0.15,\n        'min_data_in_leaf' : 80,\n        'max_depth': 6,\n        'objective': 'binary',\n        'num_leaves':25,\n        'metric': 'auc',\n        'n_jobs': -1,\n        'feature_fraction_seed': 42,\n        'bagging_seed': 42,\n        'boosting_type': 'gbdt',\n        'verbose': 1,\n        'is_unbalance': True,\n        'boost_from_average': False}\n\nt1=time.clock()\ntraintion = np.zeros(len(train))\nvalidation = np.zeros(len(train))\npredictions = np.zeros(len(test))\nfeature_importance_df = pd.DataFrame()\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train,target)):\n    print(\"fold n°{}\".format(fold_))\n    train_x=train.iloc[trn_idx][usedfeatures].reset_index(drop=True)\n    valid_x=train.iloc[val_idx][usedfeatures].reset_index(drop=True)\n    target_train=target.iloc[trn_idx].reset_index(drop=True)\n    target_valid=target.iloc[val_idx].reset_index(drop=True)\n    trn_data = lgb.Dataset(train_x,\n                           label=target_train,\n                           categorical_feature=cat_cols\n                          )\n    val_data = lgb.Dataset(valid_x,\n                           label=target_valid,\n                           categorical_feature=cat_cols\n                          )\n\n    num_round = 1000000\n    clf = lgb.train(params,\n                    trn_data,\n                    num_round,\n                    valid_sets = [trn_data, val_data],\n                    verbose_eval=250,\n                    early_stopping_rounds = 250)\n    traintion[trn_idx] += clf.predict(train_x, num_iteration=clf.best_iteration)/(folds.n_splits-1)\n    validation[val_idx] = clf.predict(valid_x, num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = usedfeatures\n    fold_importance_df[\"importance\"] = clf.feature_importance(importance_type='gain')\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(test[usedfeatures], num_iteration=clf.best_iteration) / folds.n_splits\nt2=time.clock()-t1\nprint(\"Time used: {:<8.5f}\".format(t2))\nprint(\"Train AUC score: {:<8.5f}\".format(roc_auc_score(target,traintion)))\nprint(\"Valid AUC score: {:<8.5f}\".format(roc_auc_score(target,validation)))\n\nsub['target']=predictions\nsub.to_csv('submission.csv',index=False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize feature importance"},{"metadata":{"trusted":false},"cell_type":"code","source":"f_noimp_avg = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False))\nplt.figure(figsize=(12,20))\nsns.barplot(x=\"importance\",\n            y=\"feature\",data=f_noimp_avg.reset_index())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":4}