{"cells":[{"metadata":{},"cell_type":"markdown","source":"Submissions are evaluated on area under the ROC curve between the predicted probability and the observed target."},{"metadata":{},"cell_type":"markdown","source":"# 1.Import some libs"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport seaborn as sns\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport gc\n\nfrom scipy.stats import ttest_ind, ttest_rel\nfrom scipy import stats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.base import BaseEstimator\nfrom sklearn.impute import SimpleImputer as Imputer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score, roc_curve\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, RandomizedSearchCV\nfrom sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Some options"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\npd.set_option('use_inf_as_na', True)\n\nwarnings.simplefilter('ignore')\nmatplotlib.rcParams['figure.dpi'] = 300\nsns.set()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read & reduce"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        #else:\n            #df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfolder = '../input/cat-in-the-dat-ii/'\ntrain_df = reduce_mem_usage(pd.read_csv(folder + 'train.csv'))\ntest_df = reduce_mem_usage(pd.read_csv(folder + 'test.csv'))\nsub_df = reduce_mem_usage(pd.read_csv(folder + 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train')\nprint('All: ', train_df.shape)\nprint('test')\nprint('All: ', test_df.shape)\nprint('sub')\nprint('sub_df ', sub_df.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Analyse"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_df['target']\ntrain_df.drop(['target'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.hist(bins=len(y.value_counts()));","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### day + month feature"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['day_month'] = train_df['month'] * 100 + train_df['day']\ntest_df['day_month'] = test_df['month'] * 100 + train_df['day']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Preprocess for bin3 & bin4"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['bin_3'] = train_df['bin_3'].apply(lambda x: 1 if x == 'T' else 0)\ntrain_df['bin_4'] = train_df['bin_4'].apply(lambda x: 1 if x == 'Y' else 0)\ntest_df['bin_3'] = test_df['bin_3'].apply(lambda x: 1 if x == 'T' else 0)\ntest_df['bin_4'] = test_df['bin_4'].apply(lambda x: 1 if x == 'Y' else 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Drop ID's"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(['id'], axis=1, inplace=True)\ntest_df.drop(['id'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Describe DF"},{"metadata":{},"cell_type":"markdown","source":"#### Numerics"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_cols = test_df.describe().columns.tolist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Categoricals"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.describe(include=['O'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.describe(include=['O'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_cols = test_df.describe(include=['O']).columns.tolist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check some Null's"},{"metadata":{"trusted":true},"cell_type":"code","source":"def missing_values_table(df, info=True):\n        mis_val = df.isnull().sum()\n        mis_val_percent = 100 * df.isnull().sum() / len(df)\n        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n        mis_val_table_ren_columns = mis_val_table.rename(\n        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n        mis_val_table_ren_columns = mis_val_table_ren_columns[\n            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n        '% of Total Values', ascending=False).round(1)\n        if info:\n            print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n                \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n                  \" columns that have missing values.\")\n        return mis_val_table_ren_columns.T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Numeric columns: ')\nmissing_values_table(train_df[num_cols])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Numeric columns: ')\nmissing_values_table(test_df[num_cols])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Categorical columns: ')\nmissing_values_table(train_df[cat_cols])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Categorical columns: ')\nmissing_values_table(test_df[cat_cols])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Make mean target encoding for categorical feature\n\nLet us consider the above table (A simple binary classification). \n\n$$ MeanTargetEnc_i = {((GlobalMean * C) + (Mean_i * Size)) \\over (C + Size)} $$\n\nInstead of finding the mean of the targets, we can also focus on median and other statistical correlations….These are broadly called target encodings\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class MeanEncoding(BaseEstimator):\n    \"\"\"   In Mean Encoding we take the number \n    of labels into account along with the target variable \n    to encode the labels into machine comprehensible values    \"\"\"\n    \n    def __init__(self, feature, C=0.1):\n        self.C = C\n        self.feature = feature\n        \n    def fit(self, X_train, y_train):\n        \n        df = pd.DataFrame({'feature': X_train[self.feature], 'target': y_train}).dropna()\n        \n        self.global_mean = df.target.mean()\n        mean = df.groupby('feature').target.mean()\n        size = df.groupby('feature').target.size()\n        \n        self.encoding = (self.global_mean * self.C + mean * size) / (self.C + size)\n    \n    def transform(self, X_test):\n        \n        X_test[self.feature] = X_test[self.feature].map(self.encoding).fillna(self.global_mean).values\n        \n        return X_test\n    \n    def fit_transform(self, X_train, y_train):\n        \n        df = pd.DataFrame({'feature': X_train[self.feature], 'target': y_train}).dropna()\n        \n        self.global_mean = df.target.mean()\n        mean = df.groupby('feature').target.mean()\n        size = df.groupby('feature').target.size()\n        self.encoding = (self.global_mean * self.C + mean * size) / (self.C + size)\n        \n        X_train[self.feature] = X_train[self.feature].map(self.encoding).fillna(self.global_mean).values\n        \n        return X_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for f in cat_cols+['day', 'month']:\n    me = MeanEncoding(f, C=0.01*len(train_df[f].unique()))\n    me.fit(train_df, y)\n    train_df = me.transform(train_df)\n    test_df = me.transform(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fillna with sklearn imputer"},{"metadata":{"trusted":true},"cell_type":"code","source":"imputer = Imputer(strategy=\"mean\")\nimputer.fit(train_df)\ntrain_df = pd.DataFrame(imputer.transform(train_df), columns=train_df.columns)\ntest_df = pd.DataFrame(imputer.transform(test_df), columns=train_df.columns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Feature Engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['bin_sum'] = train_df['bin_0'] + train_df['bin_1'] + train_df['bin_2'] + train_df['bin_3'] + train_df['bin_4']\ntest_df['bin_sum'] = test_df['bin_0'] + test_df['bin_1'] + test_df['bin_2'] + test_df['bin_3'] + test_df['bin_4']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['nom_sum'] = train_df['nom_0'] + train_df['nom_1'] + train_df['nom_2'] + train_df['nom_3'] + train_df['nom_4'] + train_df['nom_5'] + train_df['nom_6'] + train_df['nom_7'] + train_df['nom_8'] + train_df['nom_9']\ntest_df['nom_sum'] = test_df['nom_0'] + test_df['nom_1'] + test_df['nom_2'] + test_df['nom_3'] + test_df['nom_4'] + test_df['nom_5'] + test_df['nom_6'] + test_df['nom_7'] + test_df['nom_8'] + test_df['nom_9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['nom_multi'] = train_df['nom_0'] * train_df['nom_1'] * train_df['nom_2'] * train_df['nom_3'] * train_df['nom_4'] * train_df['nom_5'] * train_df['nom_6'] * train_df['nom_7'] * train_df['nom_8'] * train_df['nom_9']\ntest_df['nom_multi'] = test_df['nom_0'] * test_df['nom_1'] * test_df['nom_2'] * test_df['nom_3'] * test_df['nom_4'] * test_df['nom_5'] * test_df['nom_6'] * test_df['nom_7'] * test_df['nom_8'] * test_df['nom_9']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['ord_sum'] = train_df['ord_0'] + train_df['ord_1'] + train_df['ord_2'] + train_df['ord_3'] + train_df['ord_4'] + train_df['ord_5']\ntest_df['ord_sum'] = test_df['ord_0'] + test_df['ord_1'] + test_df['ord_2'] + test_df['ord_3'] + test_df['ord_4'] + test_df['ord_5']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['ord_multi'] = train_df['ord_0'] * train_df['ord_1'] * train_df['ord_2'] * train_df['ord_3'] * train_df['ord_4'] * train_df['ord_5']\ntest_df['ord_multi'] = test_df['ord_0'] * test_df['ord_1'] * test_df['ord_2'] * test_df['ord_3'] * test_df['ord_4'] * test_df['ord_5']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Correlations"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_corr = train_df.corr()\n# plot the heatmap and annotation on it\nfig, ax = plt.subplots(figsize=(14,14))\nsns.heatmap(train_corr, xticklabels=train_corr.columns, yticklabels=train_corr.columns, annot=True, ax=ax);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_corr = test_df.corr()\n# plot the heatmap and annotation on it\nfig, ax = plt.subplots(figsize=(14,14))\nsns.heatmap(test_corr, xticklabels=test_corr.columns, yticklabels=test_corr.columns, annot=True, ax=ax);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Drop columns with high correlation\n* According to the Gauss-Markov theorem"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(['ord_0', 'month', 'nom_sum', ], axis=1, inplace=True)\ntest_df.drop(['ord_0', 'month', 'nom_sum', ], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Some statistics for feats\n* ROC-AUC for feats\n* Kolmogorov-Smirnov statistic\n* Mannwhitneyu statistic"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nskf = StratifiedKFold(n_splits=3, shuffle=True, random_state=777)\n\nfeatures_stats = []\n\nfor c in train_df.columns:\n    ks = stats.ks_2samp(train_df[c], test_df[c])\n    mv = stats.mannwhitneyu(train_df[c], test_df[c])\n    \n    train_score = []\n    test_score = []\n    \n    for train_index, val_index in skf.split(train_df, y):\n        x_train, x_valid = train_df.iloc[train_index, :][[c]], train_df.iloc[val_index, :][[c]]\n        y_train, y_valid = y[train_index], y[val_index]\n        \n        logreg = LogisticRegression()\n        logreg.fit(x_train, y_train)\n        train_score.append(roc_auc_score(y_train, logreg.predict_proba(x_train)[:, 1]))\n        test_score.append(roc_auc_score(y_valid, logreg.predict_proba(x_valid)[:, 1]))\n        \n    train_score_ = np.mean(train_score)\n    test_score_ = np.mean(test_score)\n    \n    features_stats.append([c, train_score_, test_score_, ks[0], ks[1], mv[0], mv[1]])\n    \nfeatures_stats = pd.DataFrame(features_stats, columns=['Name', 'Train_AUC', 'Test_AUC', 'KS_Stats', 'KS_pvalue', 'MV_Stats', 'MV_pvalue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_stats.sort_values('Test_AUC', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Scale it"},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = StandardScaler()\ntrain_df_scale = scaler.fit_transform(train_df)\ntest_df_scale = scaler.transform(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 7. Fit it"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_scores=[]\ntest_scores=[]\n\nskf_1 = StratifiedKFold(n_splits=5, shuffle=True, random_state=777)\n\nlr_grid_l1 = {\"C\":[100, 10, 1, 0.5, 0.1, 0.5, 0.25, 0.05, 0.01, 0.005, 0.001], \"penalty\":['l1'], \"solver\":['liblinear',]}\nlr_grid_l2 = {\"C\":[100, 10, 1, 0.5, 0.1, 0.5, 0.25, 0.05, 0.01, 0.005, 0.001], \"penalty\":['l2'], \"solver\":['newton-cg','lbfgs', 'saga']}\nlr_grid_el = {\"C\":[1, 0.5, 0.1, 0.5, 0.25, 0.05, 0.01, 0.005, 0.001], \n              \"l1_ratio\":[1, 0.5, 0.1, 0.5, 0.25, 0.05, 0.01, 0.005, 0.001], \"penalty\":['elasticnet'], \"solver\":['saga']}\n\nfor no, (train_index_1, val_index_1) in enumerate(skf_1.split(train_df_scale, y)):\n    x_train, x_valid = train_df_scale[train_index_1, :], train_df_scale[val_index_1, :]\n    y_train, y_valid = y[train_index_1], y[val_index_1]\n    \n    logreg_l1=LogisticRegression()\n    logreg_cv_l1=RandomizedSearchCV(logreg_l1, lr_grid_l1, cv=5, verbose=False, scoring='roc_auc', n_jobs=-1)\n    logreg_cv_l1.fit(x_train, y_train)\n    logreg_model_l1 = LogisticRegression(**logreg_cv_l1.best_params_).fit(x_train, y_train)\n    train_pred_l1 = logreg_model_l1.predict_proba(train_df_scale)[:, 1]\n    test_pred_l1 = logreg_model_l1.predict_proba(test_df_scale)[:, 1]\n    train_scores.append(train_pred_l1)\n    test_scores.append(test_pred_l1)\n    \n    print('Fold Log L1: ', no, 'CV AUC: ', logreg_cv_l1.best_score_, \n          'Best params: ', logreg_cv_l1.best_params_)\n    \n    logreg_l2=LogisticRegression()\n    logreg_cv_l2=RandomizedSearchCV(logreg_l2, lr_grid_l2, cv=3, verbose=False, scoring='roc_auc', n_jobs=-1)\n    logreg_cv_l2.fit(x_train, y_train)\n    logreg_model_l2 = LogisticRegression(**logreg_cv_l2.best_params_).fit(x_train, y_train)\n    train_pred_l2 = logreg_model_l2.predict_proba(train_df_scale)[:, 1]\n    test_pred_l2 = logreg_model_l2.predict_proba(test_df_scale)[:, 1]\n    train_scores.append(train_pred_l2)\n    test_scores.append(test_pred_l2)\n    print('Fold Log L2: ', no, 'CV AUC: ', logreg_cv_l2.best_score_, \n          'Best params: ', logreg_cv_l2.best_params_)\n        \n    logreg_el=LogisticRegression()\n    logreg_cv_el=RandomizedSearchCV(logreg_el, lr_grid_el, cv=3, verbose=False, scoring='roc_auc', n_jobs=-1)\n    logreg_cv_el.fit(x_train, y_train)\n    logreg_model_el = LogisticRegression(**logreg_cv_el.best_params_).fit(x_train, y_train)\n    train_pred_el = logreg_model_el.predict_proba(train_df_scale)[:, 1]\n    test_pred_el = logreg_model_el.predict_proba(test_df_scale)[:, 1]\n    train_scores.append(train_pred_el)\n    test_scores.append(test_pred_el)\n    print('Fold Log EL: ', no, 'CV AUC: ', logreg_cv_el.best_score_, \n          'Best params: ', logreg_cv_el.best_params_)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 8. Predict it"},{"metadata":{"trusted":true},"cell_type":"code","source":"logreg=LogisticRegression(C=0.01)\nlogreg.fit(np.array(train_scores).T, y)\nsub_df['target'] = logreg.predict_proba(np.array(test_scores).T)[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":4}