{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Welcome to my kernel!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"This is a small tutorial on various \"Feature Selection & Label Encoding Techniques\". Hope you find it useful.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Lets start with importing necessary libraries","execution_count":null},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn import preprocessing\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn import metrics\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import KFold\nimport statsmodels.api as sm\nfrom sklearn.metrics import precision_recall_curve\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.impute import SimpleImputer\nimport tensorflow as tf\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.datasets import make_classification\nfrom tensorflow.keras.models import Sequential \nfrom tensorflow.keras.layers import Dense \nfrom tensorflow.keras.callbacks import Callback, EarlyStopping\nimport re\nfrom sklearn.impute import KNNImputer\nfrom tensorflow.keras import layers\nfrom sklearn.feature_selection import VarianceThreshold\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn import metrics\nimport category_encoders as ce\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"# load the data\ntrain_set = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\")\ntest_set = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/test.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check the training set\ntrain_set.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Given dataset contains Numeric as well as Categorical Variables.\n\nFollowing kind of categorical variabels are present in the dataset:\n1. Binary Variables - values like Male & Female\n2. Ordinal Variables (low and high cardinality) - ordered values like \"Good\" \"V Good\" \"Excellent\"\n3. Nominal Variables (low and high cardinality) - do not have any intrinsic order, values like blood groups \"A\" \"B\" etc\n","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets segregate the features into categorical and numeric columns\ncat_vars = []\nnum_vars = []\nfor col in train_set:\n    if train_set[col].dtypes == 'O':\n        cat_vars.append(col)\n    else:\n        num_vars.append(col)\n\n# removing id and target from the list\nnum_vars.remove(\"id\")\nnum_vars.remove(\"target\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets further segregate categorical variables in ordinal, nominal,binary and date variables\nbin_vars = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4']\nord_vars = ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5'] \nnom_vars = ['nom_0','nom_1','nom_2','nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\ndat_vars = ['day','month']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets save training and test set \"id\"s for future use\ntrain_set_id = train_set.id\ntest_set_id = test_set.id\n\n# save training set target \ntrain_set_target = train_set.target\n\n# lets drop targe and ids\ntrain_set.drop([\"id\",\"target\"],1,inplace= True)\ntest_set.drop(\"id\",1,inplace= True)\n\n#create master dataset from training and test datasets\nmaster=pd.concat((train_set,test_set)).reset_index(drop=True)\nmaster.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Let's display the variables having null values\nnull_cols = []\nfor col in train_set.columns:\n    if train_set[col].isnull().sum() > 0 :\n        print('%s %s %d %s' %(col,\"have\",train_set[col].isnull().sum(),\"null values\"))    \n        null_cols.append(col)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Null Value Treatment","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"We will check the data only in the training set but will apply changes on the master dataset, why we are doing this?\nbecause we dont want to look at the test data before feeding it into the model","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets take a look at the ordinal variables\nfor col in bin_vars:\n    print(col, \":\\n\",train_set[col].value_counts(),\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# missing values in bin_0/1/2 can be replaced by 0.0\nmaster[['bin_0','bin_1','bin_2']] = master[['bin_0','bin_1','bin_2']].replace(np.nan, 0.0)\n\n# missing values in bin_3 can be replaced with \"F\"\nmaster['bin_3'] = master['bin_3'].replace(np.nan, \"F\")\n\n# missing values in bin_4 can be replaced with \"N\"\nmaster['bin_4'] = master['bin_4'].replace(np.nan, \"N\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets take a look at the ordinal variables\nfor col in ord_vars:\n    print(col, \":\",train_set[col].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# since values are evenly distributed for ordinal values, we will replace the missing values by \"missing\"\nmaster[ord_vars] = master[ord_vars].replace(np.nan,\"missing\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets take a look at the ordinal variables\nfor col in nom_vars:\n    print(col, \":\",train_set[col].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# \"Red\" is most frequent value for nom_0 variable, lets replace missing values with \"Red\"\nmaster['nom_0'] = master['nom_0'].replace(np.nan,\"Red\")\n\n# \"Theremin\" is most frequent value for nom_0 variable, lets replace missing values with \"Theremin\"\nmaster['nom_4'] = master['nom_4'].replace(np.nan,\"Theremin\")\n\n# its difficult to make a call for other variables, so, let replace with \"missing\"\nmaster[['nom_1','nom_2','nom_3','nom_5','nom_6','nom_7','nom_8','nom_9']] = master[['nom_1','nom_2','nom_3','nom_5','nom_6','nom_7','nom_8','nom_9']].replace(np.nan,\"missing\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets take a look at the ordinal variables\nfor col in dat_vars:\n    print(col, \":\",train_set[col].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets replace all the null values by -1\nmaster[dat_vars] = master[dat_vars].replace(np.nan,0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets confirm if there are any missing values leftr\nmaster.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"No Null values left, good!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Handling Categorical Variables\nAs stated above, we have different kind of categorical variabels present in the dataset. We would apply different kind of encoding techniques on them:\n\n1. Binary Variables                     - will be replaced by 0 and 1\n2. Ordinal Variables (low cardinality)  - will apply Label Encoding\n3. Ordinal Variables (high cardinality) - will apply Label Encoding\n4. Nominal Variables (low cardinality)  - will apply One Hot Encodihng\n5. Nominal Variables (high cardinality) - will apply Hashing Encoding","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets encode binary variables\n\nmaster['bin_3'] = master['bin_3'].map({'F': 1, 'T': 0})\n\nmaster['bin_4'] = master['bin_4'].map({'Y': 1, 'N': 0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets apply label encoding to ordinal variables \nlbl = preprocessing.LabelEncoder()\nfor col in ord_vars:\n    master[col] = lbl.fit_transform(master[col].astype(str).values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#for col in high_card_nom_vars:\n#    top10 = master[col].value_counts().sort_values(ascending =  False).head(10).index\n#    print(top10)\n#    for label in top10:\n#        master[label] = np.where(master[col]==label,1,0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets divide nominal variables into low and high cardinality variables \nlow_card_nom_vars = []\nhigh_card_nom_vars = []\n\nfor col in nom_vars:\n    if train_set[col].nunique()>10:\n        high_card_nom_vars.append(col)\n    else:\n        low_card_nom_vars.append(col)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets take a quick look at the lists created\nlow_card_nom_vars, high_card_nom_vars","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets apply one hot encoding for low cardinality nominal variables\ndummies = pd.get_dummies(master[low_card_nom_vars], drop_first=True)\n\n# concat dummy variables with X\nmaster = pd.concat([master, dummies], axis=1)\n\n# drop categorical variables for which we already created the dummy variables\nmaster.drop(low_card_nom_vars,1,inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets apply hashing encoding on high cardinality nominal variables\nce_hash = ce.HashingEncoder(cols = high_card_nom_vars)\nmaster = ce_hash.fit_transform(master)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"master.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"master.shape\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Selection","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Lets perform feature selection with filter techniques","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets first create train_set and test_set back from the master dataset\n\ntrain_set = master[:train_set.shape[0]]\n\ntest_set = master[train_set.shape[0]:]\n\n# lets confirm the shape of train and test datasets\ntrain_set.shape, test_set.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Removal of Constant features using VarianceThreshold","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"VarianceThreshold is a simple baseline approach to feature selection. It removes all features whose variance doesn’t meet some threshold. By default, it removes all zero-variance features, i.e. features that have the same value in all samples.","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"const_fltr = VarianceThreshold(threshold = 0)\nconst_fltr.fit(train_set)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"constant_columns = [column for column in train_set.columns\n                    if column not in train_set.columns[const_fltr.get_support()]]\n\nprint(len(constant_columns))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"None of the variables had zero variance","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Removal of Quasi-constant features","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Quasi-constant features, as the name suggests, are the features that are almost constant. In other words, these features have the same values for a very large subset of the outputs. Such features are not very useful for making predictions. There is no rule as to what should be the threshold for the variance of quasi-constant features. However, as a rule of thumb, remove those quasi-constant features that have more than 99% similar values for the output observations.","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets check the variables with \nquasi_const_fltr = VarianceThreshold(threshold = 0.01)\nquasi_const_fltr.fit(train_set)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"qconstant_columns = [column for column in train_set.columns\n                    if column not in train_set.columns[quasi_const_fltr.get_support()]]\n\nprint(len(qconstant_columns))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"None of the variables had 0.1 variance","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Removing Duplicate Features","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Duplicate features are the features that have similar values. Duplicate features do not add any value to algorithm training, rather they add overhead and unnecessary delay to the training time. Therefore, it is always recommended to remove the duplicate features from the dataset before training.\n\nUnlike constant and quasi-constant features, we have no built-in Python method that can remove duplicate features. However, we have a method that can help us identify duplicate rows in a pandas dataframe. We will use this method to first take a transpose of our dataset as shown below","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"train_set_T = train_set.T\ntrain_set_T.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Removing duplicate columns using the given method iscomputationally costly since we have to take the transpose \n# of the data matrix before we can remove duplicate features\n\n#print(train_set_T.duplicated().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I have commented out the above code to save execution time, I got none duplicates","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Lets check co-linearity between the variables","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"corrmat = train_set.corr()\nsns.heatmap(corrmat)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def get_corrdata(data,threshold):\n    corr_col = set()\n    corrmat = data.corr()\n    for i in range(len(corrmat.columns)):\n        for j in range(i):\n            if abs(corrmat.iloc[i,j]) > threshold:\n                colname = corrmat.columns[i]\n                corr_col.add(colname)\n    return corr_col       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# selecting feauters with high colinearity between them\ncorr_feat = get_corrdata(train_set,0.80)\ncorr_feat\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"No Correlation found!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Lets select features based on roc-auc score of the individual columns","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"I ran below code to find out roc_auc score of indivdual variables, so that variables with roc_auc score equals to 0.5 can be removed. but I could not find any such column.\n\nI am commenting out the code because it takes lots of time in execution","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"#roc_auc = []\n#for col in train_set.columns:\n#    rfc = RandomForestClassifier(class_weight = \"balanced\")\n#    rfc.fit(train_set[col].to_frame(),train_set_target)\n#    y_pred = rfc.predict(train_set[col].to_frame())\n#    roc_auc.append(roc_auc_score(y_pred,train_set_target))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#roc_values = pd.Series(roc_auc)\n#roc_values.index= train_set.columns\n#roc_values","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets take a copy of the updated train and test data sets. Why?\n# because I plan to build Decision Tree & Random Forest models, which do not need scaled data and I am going to scale the\n# data in the next step to be used by other models\ntrain_set_copy = train_set.copy()\ntest_set_copy = test_set.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature Scaling","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# scaling the numeric features\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\ntrain_set[num_vars] = scaler.fit_transform(train_set[num_vars]) # apply \ntest_set[num_vars] = scaler.fit_transform(test_set[num_vars])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Splitting the data into train and test\nX_train, X_test, y_train, y_test = train_test_split(train_set,train_set_target, train_size=0.7,test_size=0.3,random_state=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets create a dataframe to save model's performance\nmodel_df = pd.DataFrame(columns = [\"model_name\",\"training roc_auc\",\"test_roc_auc\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Confusion matrix function\ndef cm(pred,true):\n    confusion = metrics.confusion_matrix(pred, true)\n    print(confusion)\n    score = round(metrics.roc_auc_score(pred,true),2)\n    print(\"roc_auc score:\",score)\n    return(score)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# saving the list of current features\ncol = X_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets build a GLM and check the p-values of the features\n# class_weight=\"balanced\" will take care of the class imbalance in the dataset\nX_train_sm = sm.add_constant(X_train)\nlogm1 = sm.GLM(y_train,X_train_sm, class_weight=\"balanced\",family = sm.families.Binomial())\nlogm1.fit().summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"bin_3 has high p-value, lets drop this column and rebuild the model","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"col = col.drop('bin_3', 1)\n\nX_train_sm = sm.add_constant(X_train[col])\n# class_weight=\"balanced\" will take care of the class imbalance in the dataset\nlogm14 = sm.GLM(y_train,X_train_sm, class_weight=\"balanced\",family = sm.families.Binomial())\nres = logm14.fit()\nres.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"col = col.drop('nom_1_missing', 1)\n\nX_train_sm = sm.add_constant(X_train[col])\n# class_weight=\"balanced\" will take care of the class imbalance in the dataset\nlogm14 = sm.GLM(y_train,X_train_sm, class_weight=\"balanced\",family = sm.families.Binomial())\nres = logm14.fit()\nres.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"col = col.drop('nom_3_India', 1)\n\nX_train_sm = sm.add_constant(X_train[col])\n# class_weight=\"balanced\" will take care of the class imbalance in the dataset\nlogm14 = sm.GLM(y_train,X_train_sm, class_weight=\"balanced\",family = sm.families.Binomial())\nres = logm14.fit()\nres.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# p-values looks good now, lets check VIF\n\nvif = pd.DataFrame()\nvif['Features'] = X_train[col].columns\nvif['VIF'] = [variance_inflation_factor(X_train[col].values, i) for i in range(X_train[col].shape[1])]\nvif['VIF'] = round(vif['VIF'], 2)\nvif = vif.sort_values(by = \"VIF\", ascending = False)\nvif","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"All the p-values and VIF looks good now","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# make predictions\ny_train_pred = res.predict(X_train_sm)\n\nX_test_sm = sm.add_constant(X_test[col])\ny_test_pred = res.predict(X_test_sm)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Logistic Regression - Default","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"X_train = X_train[col]\n\nX_test = X_test[col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"logModel = LogisticRegression(class_weight = \"balanced\",solver = \"saga\")\nres = logModel.fit(X_train,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# make predictions\ny_train_pred_prob = res.predict_proba(X_train)\ny_train_pred = res.predict(X_train)\n\ny_test_pred_prob = res.predict_proba(X_test)\ny_test_pred = res.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"training scores:\")\ntrain_score = cm(y_train_pred,y_train)\n\nprint(\"\\ntest scores:\\n\")\ntest_score = cm(y_test_pred,y_test)\n\nmodel_df.loc[0] = [\"LogisticReg-Default\",train_score,test_score]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fine Tune Logistic Regression","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# set up cross validation scheme\n#folds = StratifiedKFold(n_splits = 5, shuffle = True, random_state = 4)\n\n# specify range of hyperparameters\n#params = {\"C\":np.logspace(-3,3,7), \"penalty\":[\"l1\",\"l2\"]}# l1 lasso l2 ridge\n\n## using Logistic regression for class imbalance, class_weight = balanced will take care of class imbalance in the dataset\n#model = LogisticRegression(class_weight='balanced', solver = \"saga\")\n#model_cv = GridSearchCV(estimator = model, param_grid = params, \n#                        scoring= 'roc_auc', \n#                        cv = folds, \n#                        return_train_score=True, verbose = 1)            \n#model_cv.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## reviewing the results\n#cv_results = pd.DataFrame(model_cv.cv_results_)\n#cv_results","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## segerigating results for L1 and L2 regression and plotting them differently\n#cv_results_penalty_l1 = cv_results.loc[cv_results['param_penalty']=='l1']\n#cv_results_penalty_l2 = cv_results.loc[cv_results['param_penalty']=='l2']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## plotting results for Logistic regression with L1 panelty\n#plt.figure(figsize=(8, 6))\n#plt.plot(cv_results_penalty_l1['param_C'], cv_results_penalty_l1['mean_test_score'])\n#plt.plot(cv_results_penalty_l1['param_C'], cv_results_penalty_l1['mean_train_score'])\n#plt.xlabel('C')\n#plt.ylabel('roc_auc')\n#plt.legend(['test roc_auc', 'train roc_auc'], loc='upper right')\n#plt.xscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## plotting results for Logistic regression with L2 panelty\n#plt.figure(figsize=(8, 6))\n#plt.plot(cv_results_penalty_l2['param_C'], cv_results_penalty_l2['mean_test_score'])\n#plt.plot(cv_results_penalty_l2['param_C'], cv_results_penalty_l2['mean_train_score'])\n#plt.xlabel('C')\n#plt.ylabel('roc_auc')\n#plt.legend(['test roc_auc', 'train roc_auc'], loc='upper right')\n#plt.xscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## checking best score\n#best_score = model_cv.best_score_\n#best_param = model_cv.best_params_\n\n#print(\" The highest test roc_auc is {0} at {1}\".format(best_score, best_param))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I have commented out the above code to reduce execution time, I am building the model with the best parameters below","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Building the model with the best parameters","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"## preparing final model based on best score\nmodel=LogisticRegression(C=0.1,penalty=\"l2\",class_weight=\"balanced\",solver=\"saga\")\nmodel.fit(X_train,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# make predictions\ny_train_pred_prob = model.predict_proba(X_train)\ny_train_pred = model.predict(X_train)\n\ny_test_pred_prob = model.predict_proba(X_test)\ny_test_pred = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"training scores:\")\ntrain_score = cm(y_train_pred,y_train)\n\nprint(\"\\ntest scores:\\n\")\ntest_score = cm(y_test_pred,y_test)\n\nmodel_df.loc[1] = [\"LogisticReg-Regularize\",train_score,test_score]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Artificial Neural Network","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Artificial Neural Network does not provide roc_auc as a default score after an epoch, lets write a small function to achieve the same","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"def auc(y_true, y_pred):\n    def fallback_auc(y_true, y_pred):\n        try:\n            return metrics.roc_auc_score(y_true, y_pred)\n        except:\n            return 0.5\n    return tf.py_function(fallback_auc, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(300, activation='relu',input_dim = X_train.shape[1]))\nlayers.Dropout(0.3)\nmodel.add(Dense(1, activation = 'sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=[auc])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# reserving 20% of the training data for validation purpose in each epoch\nmodel.fit(X_train, y_train,validation_split =0.2,batch_size = 128, epochs = 20)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_train_pred = model.predict_classes(X_train)\ny_test_pred = model.predict_classes(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_score = cm(y_train_pred,y_train)\ntest_score = cm(y_test_pred,y_test)\nmodel_df.loc[2] = [\"ANN\",train_score,test_score]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_test_pred = model.predict_classes(test_set[col])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lets prepare for the prediction submission\nsub = pd.DataFrame()\nsub['Id'] = test_set_id\nsub['target'] = y_test_pred\nsub.to_csv('submission_ann.csv',index=False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Decision Tree","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Decision trees do not require scaled data, hence will be using originally cleaned data set","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# Splitting the data into train and test\nX_train, X_test, y_train, y_test = train_test_split(train_set_copy,train_set_target, train_size=0.7,test_size=0.3,random_state=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Decision tree with default parameters\nmodel = DecisionTreeClassifier(class_weight = \"balanced\")\nmodel.fit(X_train,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_train_pred = model.predict(X_train)\ny_test_pred = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_score = cm(y_train_pred,y_train)\ntest_score = cm(y_test_pred,y_test)\nmodel_df.loc[3] = [\"Decision Tree\",train_score,test_score]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fine Tuning Decision Tree","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# function to fine tune different hyperparameters\ndef dtree_hyper_param_tuning(X_train,y_train,param,values,score):\n    print(param)\n    # specify number of folds for k-fold CV\n    n_folds = KFold(n_splits = 5, shuffle = True, random_state = 4)\n\n    # parameters to build the model on\n    parameters = {param: values}\n\n# instantiate the model\n    dtree = DecisionTreeClassifier(class_weight='balanced',criterion = \"gini\", \n                                   random_state = 101)\n\n    # fit tree on training data\n    tree = GridSearchCV(dtree, parameters, \n                        cv=n_folds, \n                       scoring=\"roc_auc\",\n                       return_train_score=True, verbose = 1)\n    tree.fit(X_train, y_train)\n\n    # scores of GridSearch CV\n    scores = tree.cv_results_\n    pd.DataFrame(scores).head()\n\n    # plotting accuracies with max_depth\n    plt.figure()\n    plt.plot(scores[score], \n             scores[\"mean_train_score\"], \n             label=\"training roc_auc\")\n    plt.plot(scores[score], \n             scores[\"mean_test_score\"], \n             label=\"test roc_auc\")\n    plt.xlabel(param)\n    plt.ylabel(\"auc_roc\")\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Below code has been commented out to save the time in execution of this kernel","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"# fine tune max depth\n#dtree_hyper_param_tuning(X_train,y_train,\"max_depth\",range(10,50,10), \"param_max_depth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# fine tune minimum samples leaf\n#dtree_hyper_param_tuning(X_train,y_train,\"min_samples_leaf\",[500,1000,2000],\"param_min_samples_leaf\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# fine tune minimum samples split\n#dtree_hyper_param_tuning(X_train,y_train,\"min_samples_split\",[500,1000,2000],\"param_min_samples_split\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# putting all the tuned parameters together to find out the best fit\n#param_grid = {\n#    'max_depth': [10,15,20],\n#    'min_samples_leaf': [500,700,900],\n#    'min_samples_split': [200,400,600],\n#    'criterion': [\"entropy\", \"gini\"]\n#}\n\n#n_folds = 5\n\n# Instantiate the grid search model\n#dtree = DecisionTreeClassifier(class_weight='balanced')\n#grid_search = GridSearchCV(estimator = dtree, param_grid = param_grid, \n#scoring=\"roc_auc\", cv = n_folds,\n#return_train_score=True, verbose = 1)\n\n# Fit the grid search to the data\n#grid_search.fit(X_train,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# cv results\n#cv_results = pd.DataFrame(grid_search.cv_results_)\n#cv_results.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# printing the optimal recall score and hyperparameters\n#print(\"best roc_aus score\", grid_search.best_score_)\n#print(grid_search.best_estimator_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Decision tree with best parameters\nmodel = DecisionTreeClassifier(max_depth = 20, min_samples_leaf = 500,min_samples_split = 200,class_weight = \"balanced\")\nmodel.fit(X_train,y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_train_pred = model.predict(X_train)\ny_test_pred = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_score = cm(y_train_pred,y_train)\ntest_score = cm(y_test_pred,y_test)\nmodel_df.loc[4] = [\"Decision Tree Tuned\",train_score,test_score]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# snapshot of all the models created\nmodel_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}