{"cells":[{"metadata":{},"cell_type":"markdown","source":"> **\nThis version tries to implement different strategies for dealing with missing values mostly in ordinal data. Most tests show nominal variables perform better untouched, dealing with them with OneHotEncoder. The final solution is Stack of the all method used in this kernel**"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --no-warn-conflicts -U -q scikit-learn==0.22.1 autoimpute impyute fancyimpute","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"from fancyimpute import NuclearNormMinimization, SoftImpute, BiScaler, IterativeSVD, NuclearNormMinimization\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer, PowerTransformer\nfrom sklearn.linear_model import LogisticRegression, RidgeClassifier\nfrom autoimpute.imputations import SingleImputer, MultipleImputer\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import SimpleImputer, IterativeImputer\nfrom sklearn.model_selection import StratifiedKFold\nimport time, os, warnings, random, string, re, gc\nfrom sklearn.feature_selection import RFE, RFECV\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.metrics import roc_auc_score\nfrom IPython.display import display\nfrom scipy.stats import rankdata\nfrom autoimpute.visuals import *\nimport matplotlib.pyplot as plt\nimport category_encoders as ce\nimport plotly_express as px\nimport impyute as impy \nimport seaborn as sns\nimport pandas as pd \nimport scipy as sp\nimport numpy as np\n\nsns.set_style('whitegrid')\nwarnings.filterwarnings('ignore')\nwarnings.simplefilter('ignore')\npd.set_option('display.max_columns', 1000)\npd.set_option('display.max_rows', 500)\nSEED = 2020\nSPLITS = 25","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def set_seed(seed=SEED):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\nset_seed()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base =  pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')\nbaseTe =  pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')\nbaseTe['target'] = -1\nscore = dict()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(base.isna().sum(axis=1).describe(), columns=['Value']).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(base.isna().sum(axis=0)/len(base), columns=['missing percent']).sort_values('missing percent', ascending=False).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_md_locations(base)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_nullility_dendogram(base)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def null_analysis(df):\n  '''\n  desc: get nulls for each column in counts & percentages\n  arg: dataframe\n  return: dataframe\n  '''\n  null_cnt = df.isnull().sum() # calculate null counts\n  null_cnt = null_cnt[null_cnt!=0] # remove non-null cols\n  null_percent = null_cnt / len(df) * 100 # calculate null percentages\n  null_table = pd.concat([pd.DataFrame(null_cnt), pd.DataFrame(null_percent)], axis=1)\n  null_table.columns = ['counts', 'percentage']\n  null_table.sort_values('counts', ascending=False, inplace=True)\n  return null_table","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"null_table = null_analysis(base)\npx.bar(null_table.reset_index(), x='index', y='percentage', text='counts', height=500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score = dict()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base.nom_1.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def SiavashMapping(df):\n    \n    bin_3_mapping = {'T':1 , 'F':0}\n    bin_4_mapping = {'Y':1 , 'N':0}\n    nom_0_mapping = {'Red' : 0, 'Blue' : 1, 'Green' : 2}\n    nom_1_mapping = {'Trapezoid' : 0, 'Star' : 1, 'Circle': 2, 'Triangle' : 3, 'Polygon' : 4, 'Square': 5}\n    nom_2_mapping = {'Hamster' : 0 , 'Axolotl' : 1, 'Lion' : 2, 'Dog' : 3, 'Cat' : 4, 'Snake' : 5}\n    nom_3_mapping = {'Russia' : 0, 'Canada' : 1, 'Finland' : 2, 'Costa Rica' : 3, 'China' : 4, 'India' : 5}\n    nom_4_mapping = {'Bassoon' : 0, 'Theremin' : 1, 'Oboe' : 2, 'Piano' : 3}\n    nom_5_mapping = dict(zip((df.nom_5.dropna().unique()), range(len((df.nom_5.dropna().unique())))))\n    nom_6_mapping = dict(zip((df.nom_6.dropna().unique()), range(len((df.nom_6.dropna().unique())))))\n    nom_7_mapping = dict(zip((df.nom_7.dropna().unique()), range(len((df.nom_7.dropna().unique())))))\n    nom_8_mapping = dict(zip((df.nom_8.dropna().unique()), range(len((df.nom_8.dropna().unique())))))\n    nom_9_mapping = dict(zip((df.nom_9.dropna().unique()), range(len((df.nom_9.dropna().unique())))))\n    ord_1_mapping = {'Novice' : 0, 'Contributor' : 1, 'Expert' : 2, 'Master': 3, 'Grandmaster': 4}\n    ord_2_mapping = { 'Freezing': 0, 'Cold': 1, 'Warm' : 2, 'Hot': 3, 'Boiling Hot' : 4, 'Lava Hot' : 5}\n    ord_3_mapping = {'a':0, 'b':1, 'c':2 ,'d':3 ,'e':4, 'f':5, 'g':6, 'h':7, 'i':8, 'j':9, 'k':10, 'l':11, 'm':12, 'n':13, 'o':14}\n    ord_4_mapping = {'A':0, 'B':1, 'C':2, 'D':3, 'E':4, 'F':5, 'G':6, 'H':7, 'I':8, 'J':9, 'K':10,'L':11,'M':12,\n                     'N':13,'O':14,'P':15,'Q':16,'R':17,'S':18,'T':19,'U':20,'V':21,'W':22,'X':23,'Y':24,'Z':25}\n    sorted_ord_5 = sorted(df.ord_5.dropna().unique())\n    ord_5_mapping = dict(zip(sorted_ord_5, range(len(sorted_ord_5))))\n    df['bin_3'] = df.loc[df.bin_3.notnull(), 'bin_3'].map(bin_3_mapping)\n    df['bin_4'] = df.loc[df.bin_4.notnull(), 'bin_4'].map(bin_4_mapping)\n    df['nom_0'] = df.loc[df.nom_0.notnull(), 'nom_0'].map(nom_0_mapping)\n    df['nom_1'] = df.loc[df.nom_1.notnull(), 'nom_1'].map(nom_1_mapping)\n    df['nom_2'] = df.loc[df.nom_2.notnull(), 'nom_2'].map(nom_2_mapping)\n    df['nom_3'] = df.loc[df.nom_3.notnull(), 'nom_3'].map(nom_3_mapping)\n    df['nom_4'] = df.loc[df.nom_4.notnull(), 'nom_4'].map(nom_4_mapping)\n    df['nom_5'] = df.loc[df.nom_5.notnull(), 'nom_5'].map(nom_5_mapping)\n    df['nom_6'] = df.loc[df.nom_6.notnull(), 'nom_6'].map(nom_6_mapping)\n    df['nom_7'] = df.loc[df.nom_7.notnull(), 'nom_7'].map(nom_7_mapping)\n    df['nom_8'] = df.loc[df.nom_8.notnull(), 'nom_8'].map(nom_8_mapping)\n    df['nom_9'] = df.loc[df.nom_9.notnull(), 'nom_9'].map(nom_9_mapping)\n    df['ord_1'] = df.loc[df.ord_1.notnull(), 'ord_1'].map(ord_1_mapping)\n    df['ord_2'] = df.loc[df.ord_2.notnull(), 'ord_2'].map(ord_2_mapping)\n    df['ord_3'] = df.loc[df.ord_3.notnull(), 'ord_3'].map(ord_3_mapping)\n    df['ord_4'] = df.loc[df.ord_4.notnull(), 'ord_4'].map(ord_4_mapping)\n    df['ord_5'] = df.loc[df.ord_5.notnull(), 'ord_5'].map(ord_5_mapping)\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def AntMapping(df, ordinal):\n    ord_maps = {\n        'ord_0': {val: i for i, val in enumerate([1, 2, 3])},\n        'ord_1': {\n            val: i\n            for i, val in enumerate(\n                ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster']\n            )\n        },\n        'ord_2': {\n            val: i\n            for i, val in enumerate(\n                ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']\n            )\n        },\n        **{col: {val: i for i, val in enumerate(sorted(df[col].dropna().unique()))} for col in ['ord_3', 'ord_4', 'ord_5', 'day', 'month']},\n    }\n    ord_cols = pd.concat([df[col].map(ord_map).fillna(max(ord_map.values())//2).astype('float32') for col, ord_map in ord_maps.items()], axis=1)\n    ord_cols /= ord_cols.max()\n    ord_sqr = 4*(ord_cols - 0.5)**2\n    ord_cols_sqr = [feat+'_sqr' for feat in ordinal]\n    df[ordinal] = ord_cols\n    df[ord_cols_sqr] = ord_sqr\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def CountEncoding(df, cols, df_test=None):\n    for col in cols:\n        frequencies = df[col].value_counts().reset_index()\n        df_values = df[[col]].merge(frequencies, how='left', left_on=col, right_on='index').iloc[:,-1].values\n        df[col+'_counts'] = df_values\n        if df_test is not None:\n            df_test_values = df_test[[col]].merge(frequencies, how='left', left_on=col, right_on='index').fillna(1).iloc[:,-1].values\n            df_test[col+'_counts'] = df_test_values\n    count_cols = [col+'_counts' for col in cols]\n    if df_test is not None:\n        return df, df_test, count_cols\n    else:\n        return df, count_cols\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def YurgensenMapping(df):\n    \n    def TLC(s):\n        s = str(s)\n        return (((ord(s[0]))-64)*52+((ord(s[1]))-64)-6)\n    \n    df['ord_0_ord_2'] = df['ord_0'].astype('str')+df['ord_2'].astype('str')\n    df['ord_0'] = df['ord_0'].fillna(2.01)\n    df.loc[df.ord_2=='Freezing',    'ord_2'] = 0\n    df.loc[df.ord_2=='Cold',        'ord_2'] = 1\n    df.loc[df.ord_2=='Warm',        'ord_2'] = 2\n    df.loc[df.ord_2=='Hot',         'ord_2'] = 3\n    df.loc[df.ord_2=='Boiling Hot', 'ord_2'] = 4\n    df.loc[df.ord_2=='Lava Hot',    'ord_2'] = 5\n    df['ord_2'] = df['ord_2'].fillna(2.37)\n    df.loc[df.ord_1=='Novice',      'ord_1'] = 0\n    df.loc[df.ord_1=='Contributor', 'ord_1'] = 1\n    df.loc[df.ord_1=='Expert',      'ord_1'] = 2\n    df.loc[df.ord_1=='Master',      'ord_1'] = 3\n    df.loc[df.ord_1=='Grandmaster', 'ord_1'] = 4\n    df['ord_1'] = df['ord_1'].fillna(1.86)\n    df['ord_5'] = df.loc[df.ord_5.notnull(), 'ord_5'].apply(lambda x: TLC(x))\n    df['ord_5'] = df['ord_5'].fillna('Zx').apply(lambda x: TLC(x))\n    df['ord_5'] = df['ord_5'].rank()\n    df['ord_3'] = df.loc[df.ord_3.notnull(), 'ord_3'].apply(lambda x: ord(str(x))-96)\n    df['ord_3'] = df['ord_3'].fillna(8.44)\n    df['ord_4'] = df.loc[df.ord_4.notnull(), 'ord_4'].apply(lambda x: ord(str(x))-64)\n    df['ord_4'] = df['ord_4'].fillna(14.31)\n    \n    return df\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def RidgeClf(train, test, ordinal, ohe, scaler, seed, splits, drop_idx=None, dimreducer=None):\n       \n    y_train = train['target'].values.copy()\n    train_length = train.shape[0]\n    test['target'] = -1\n    data = pd.concat([train, test], axis=0).reset_index(drop=True)\n    X_ohe = pd.get_dummies(data[ohe],columns=ohe,drop_first=True,dummy_na=True,sparse=True, dtype='int8').sparse.to_coo()\n    if dimreducer is not None:\n        X_ohe = sp.sparse.csr_matrix(dimreducer.fit_transform(X_ohe))\n        gc.collect()\n    if ordinal is not None:\n        if scaler is not None:\n            X_ord = scaler.fit_transform(data[ordinal])\n        else: \n            X_ord = data[ordianl].values\n        data_ = sp.sparse.hstack([X_ohe, X_ord]).tocsr()\n    else:\n        data_ = sp.sparse.hstack([X_ohe]).tocsr()\n    \n    train = data_[:train_length]\n    test = data_[train_length:]\n    model = RidgeClassifier(alpha=152.5)\n    skf = StratifiedKFold(n_splits=splits, shuffle=True, random_state=seed)\n    oof = np.zeros((train.shape[0],))\n    y_pred = np.zeros((test.shape[0],))\n\n    for tr_ind, val_ind in skf.split(train, y_train):\n        if drop_idx is not None:\n            idx = list(set(tr_ind)-set(drop_idx))\n            X_tr, X_val = train[idx],  train[val_ind]\n            y_tr, y_val = y_train[idx], y_train[val_ind]\n        else:\n            X_tr, X_val = train[tr_ind],  train[val_ind]\n            y_tr, y_val = y_train[tr_ind], y_train[val_ind]\n        train_set = {'X':X_tr, 'y':y_tr}\n        val_set = {'X':X_val, 'y':y_val}\n        model.fit(train_set['X'], train_set['y'])\n        oof[val_ind] = model.decision_function(val_set['X'])\n        y_pred += model.decision_function(test) / splits\n    oof_auc_score = roc_auc_score(y_train, oof)\n    oof = rankdata(oof)/len(oof)\n    y_pred = rankdata(y_pred)/len(y_pred)\n    return oof, y_pred, oof_auc_score\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def LogRegClf(train, test, ordinal, ohe, scaler, seed, splits, drop_idx=None, dimreducer=None):\n    params = { \n        'fit_intercept' : True,\n        'random_state': SEED,   \n        'penalty' : 'l2',\n        'verbose' : 0,   \n        'solver' : 'lbfgs',     \n        'max_iter' : 1000,\n        'n_jobs' : 4,\n        'C' : 0.05,\n            }\n    y_train = train['target'].values.copy()\n    train_length = train.shape[0]\n    test['target'] = -1\n    data = pd.concat([train, test], axis=0).reset_index(drop=True)\n    X_ohe = pd.get_dummies(data[ohe],columns=ohe,drop_first=True,dummy_na=True,sparse=True, dtype='int8').sparse.to_coo()\n    if dimreducer is not None:\n        X_ohe = sp.sparse.csr_matrix(dimreducer.fit_transform(X_ohe))\n        gc.collect()\n    if ordinal is not None:\n        if scaler is not None:\n            X_ord = scaler.fit_transform(data[ordinal])\n        else:\n            X_ord = data[ordinal].values\n        data_ = sp.sparse.hstack([X_ohe, X_ord]).tocsr()\n    else:\n        data_ = sp.sparse.hstack([X_ohe]).tocsr()\n    \n    train = data_[:train_length]\n    test = data_[train_length:]\n    model = LogisticRegression(**params)\n    skf = StratifiedKFold(n_splits=splits, shuffle=True, random_state=seed)\n    oof = np.zeros((train.shape[0],))\n    y_pred = np.zeros((test.shape[0],))\n\n    for tr_ind, val_ind in skf.split(train, y_train):\n        if drop_idx is not None:\n            idx = list(set(tr_ind)-set(drop_idx))\n            X_tr, X_val = train[idx],  train[val_ind]\n            y_tr, y_val = y_train[idx], y_train[val_ind]\n        else:\n            X_tr, X_val = train[tr_ind],  train[val_ind]\n            y_tr, y_val = y_train[tr_ind], y_train[val_ind]\n        train_set = {'X':X_tr, 'y':y_tr}\n        val_set = {'X':X_val, 'y':y_val}\n        model.fit(train_set['X'], train_set['y'])\n        oof[val_ind] = model.predict_proba(val_set['X'])[:, 1]\n        y_pred += model.predict_proba(test)[:, 1] / splits\n    oof_auc_score = roc_auc_score(y_train, oof)\n    oof = rankdata(oof)/len(oof)\n    y_pred = rankdata(y_pred)/len(y_pred)\n    return oof, y_pred, oof_auc_score\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Imputing with a constant value means creating new class for missing values in each column of data. Sklearn's SimpleImputer and IterativeImputer have four strategies (initial strategies): \n> * Constant\n> * Most-frequent\n> * Mean\n> * Median"},{"metadata":{},"cell_type":"markdown","source":"First we impute all columns and use StandardScaler for ordinal columns and OHE for for others."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest = baseTe.copy()\nfeatures = [feat for feat in train.columns if feat not in ['target','id']]\nohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal = [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp = SimpleImputer(strategy='constant')\ntrain[features] = imp.fit_transform(train[features])\ntest[features]  = imp.transform(test[features])\ntrain[features] = train[features].astype(np.int16)\ntest[features]  = test[features].astype(np.int16)\nscaler = RobustScaler(quantile_range=(10.0, 90.0))\noof1, pred1, score['Constant'] = RidgeClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Constant Imputation : {score[\"Constant\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This time we impute ordinal columns and use StandardScaler for ordinal columns and OHE for for others and let the OHE handle missing values in ohe columns."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SimpleImputer(strategy='constant')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof2, pred2, score['Constant-OrdinalOnly'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Constant Imputation of ordinal columns: {score[\"Constant-OrdinalOnly\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"OHE on all features except 'ord_3', 'ord_4', 'ord_5'"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SimpleImputer(strategy='constant')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof3, pred3, score['Constant-Ord4Only'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Constant Imputation of ord_4 column: {score[\"Constant-Ord4Only\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"OneHotEncoding all columns"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =  features\nordinal = None\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof4, pred4, score['CompleteOHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for No imputation just OHE: {score[\"CompleteOHE\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Complete OHE on all columns after constant imputation on all columns"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =  features\nordinal = None\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SimpleImputer(strategy='constant')\ntrain[features] = imp.fit_transform(train[features])\ntest[features]  = imp.transform(test[features])\ntrain[features] = train[features].astype(np.int16)\ntest[features]  = test[features].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof5, pred5, score['Constant-CompleteOHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Constant Imputation with complete OHE: {score[\"Constant-CompleteOHE\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The mode imputer calculates the mode of the observed dataset and uses it to impute missing observations. After mode imputation we apply OHE to all columns."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =  features\nordinal = None\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SingleImputer(strategy='mode')\ntrain[features] = imp.fit_transform(train[features])\ntest[features]  = imp.transform(test[features])\ntrain[features] = train[features].astype(np.int16)\ntest[features]  = test[features].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof6, pred6, score['Mode-CompleteOHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Mode Imputation with complete OHE: {score[\"Mode-CompleteOHE\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Median imputing with OHE of non-ordinal features"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SingleImputer(strategy='mean')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof7, pred7, score['Mean-OHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Mean Imputation with OHE on non-ordinal columns: {score[\"Mean-OHE\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The Ordinal Feature mappings from next method are borrowed from [let-s-overfit-some from Sergey Yurgensen](https://www.kaggle.com/ccccat/let-s-overfit-some)"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\ntrain = YurgensenMapping(train)\ntest  = YurgensenMapping(test)\nfeatures = [feat for feat in train.columns if feat not in ['target','id']]\nohe =      [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =  [feat for feat in features if feat not in ohe]\nscaler  =  RobustScaler(quantile_range=(10.0, 90.0))\noof8, pred8, score['Yurgensen'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Yurgensen ordinal-Mapping : {score[\"Yurgensen\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Imputing ordinal features with max(x)/2 and OHE on all non-ordinal columns. Most of this part is borrowed from [oh-my-plain-logreg from Ant](https://www.kaggle.com/superant/oh-my-plain-logreg)"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures = [feat for feat in train.columns if feat not in ['target','id']]\nohe =      [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =  [feat for feat in features if feat not in ohe] + ['day', 'month']\ntrain  = AntMapping(train, ordinal)\ntest   = AntMapping(test , ordinal)\nscaler = None\noof9, pred9, score['Ant'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Max Imputation of ordinals with OHE on non-ordinal columns: {score[\"Ant\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Imputing ordinal features with max(x)/2 and OHE on all non-ordinal columns with Frequency Encoding of ordinal columns. "},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures = [feat for feat in train.columns if feat not in ['target','id']]\nohe =      [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =  [feat for feat in features if feat not in ohe] + ['day', 'month']\ntrain  = AntMapping(train, ordinal)\ntest   = AntMapping(test , ordinal)\ntrain, test, count_cols = CountEncoding(train, ordinal, test)\nordinal += count_cols\nscaler  = RobustScaler(quantile_range=(10.0, 90.0))\noof10, pred10, score['Ant-CE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Max Imputation of ordinals with OHE on non-ordinal columns with CountEncoding of Ordinals: {score[\"Ant-CE\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Mean imputation of ordinal columns with OHE of non-ordinals"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SimpleImputer(strategy='median')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof11, pred11, score['median-OrdPartial'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for median Imputation of ord_0, ord_1 and ord_2 columns: {score[\"median-OrdPartial\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The Norm Strategy constructs a normal distribution using the sample mean and variance of the observed data. The imputer then randomly samples from this distribution to impute missing data."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SingleImputer(strategy='norm')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof12, pred12, score['Norm'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Norm Imputation of ordinals: {score[\"Norm\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"LOCF carries the last observation forward to impute missing data. A Similar stragy is NOCB which carries the next observation backward to impute missing data. "},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SingleImputer(strategy='locf')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof13, pred13, score['LOCF'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for LOCF Imputation of ordinals: {score[\"LOCF\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This default univariate imputation determines how to impute based on the column type of each column in a dataframe. It uses mean for numerc data and mode for categorical data."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SingleImputer(strategy='default univariate')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof14, pred14, score['DefUnivariate'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Default Univariate Imputation: {score[\"DefUnivariate\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Interpolate imputes missing values uses a valid pd.Series interpolation strategy."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe = features\nordinal = None\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SingleImputer(strategy='interpolate')\ntrain[features] = imp.fit_transform(train[features])\ntest[features]  = imp.transform(test[features])\ntrain[features] = train[features].astype(np.int16)\ntest[features]  = test[features].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof15, pred15, score['Interpolate-OHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'Average AUC score for Interpolate Imputation with OHE om all columns: {score[\"Interpolate-OHE\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" Simple implementation of [Exact Matrix Completion via Convex Optimization](http://statweb.stanford.edu/~candes/papers/MatrixCompletion.pdf)"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe = features\nordinal = None\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SoftImpute(max_iters=100, verbose=False)\ntrain[features] = imp.fit_transform(train[features])\ntest[features]  = imp.fit_transform(test[features])\ntrain[features] = train[features].astype(np.int16)\ntest[features]  = test[features].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof16, pred16, score['SoftImpute-OHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for SoftImpute Imputation : {score[\"SoftImpute-OHE\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The LeastSquares Strategy produces predictions using the least squares methodology. The prediction from the line of best fit given a set of predictors become the imputations."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SingleImputer(strategy='least squares')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof17, pred17, score['LQ-PartialOHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for LQ Imputation with Partial OHE : {score[\"LQ-PartialOHE\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The Stochastic Strategy predicts using the least squares methodology. The imputer then samples from the regression’s error distribution and adds the random draw to the prediction. This draw adds the stochastic element to the imputations."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             SingleImputer(strategy='stochastic')\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof18, pred18, score['stochastic'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for stochastic Imputation : {score[\"stochastic\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"IterativeImputer a Multivariate imputer that estimates each feature from all the others with an initila strategy. An estimator (the default estimator is BayesianRidge) used at each step of the round-robin imputation. If sample_posterior is True, the estimator must support return_std in its predict method.\n"},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             IterativeImputer(max_iter=500, initial_strategy='most_frequent', random_state=SEED)\ntrain[features] =  imp.fit_transform(train[features])\ntest[features]  =  imp.transform(test[features])\ntrain[features] =  train[features].astype(np.int16)\ntest[features]  =  test[features].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof19, pred19, score['Iterative'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Iterative Imputation : {score[\"Iterative\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Another important strategy is listwise dropping (drop all the rows with missing values) which is not a viable option in this comp so instead we use a threshold of 4 missing values in each row for dropping in combination with iterative imputation."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\ndrop_idx =  base[(base.isna().sum(axis=1)>3)].index.values\ndrop_idx =  [i for i in drop_idx]\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             IterativeImputer(max_iter=500, initial_strategy='most_frequent', random_state=SEED)\ntrain[features] =  imp.fit_transform(train[features])\ntest[features]  =  imp.transform(test[features])\ntrain[features] =  train[features].astype(np.int16)\ntest[features]  =  test[features].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof20, pred20, score['IterativeWithDrop'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=drop_idx)\nprint(f'AUC score for Iterative Imputation With threshold Drop : {score[\"IterativeWithDrop\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Another technique is adding indicators for imputed data which is implemented in sklearn and have the type of MissingIndicator. If the option is selected in sklearn's imputers the indicator array is added to the original data. "},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\ndrop_idx =  base[(base.isna().sum(axis=1)>3)].index.values\ndrop_idx =  [i for i in drop_idx]\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp = IterativeImputer(max_iter=500, initial_strategy='most_frequent', random_state=SEED, add_indicator=True)\nindicator_cols = [feat+'_ind' for feat in ordinal]\nfor col in indicator_cols:\n    train[col] = 0\n    test[col]  = 0\n    train[col] = train[col].astype(np.uint8)\n    test[col]  = test[col].astype(np.uint8)\n\ntrain[ordinal+indicator_cols] =  imp.fit_transform(train[ordinal])\ntest[ordinal+indicator_cols]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =           RobustScaler(quantile_range=(10.0, 90.0))\nohe   += indicator_cols\noof21, pred21, score['IterativeWithIndicator'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=drop_idx)\nprint(f'AUC score for Iterative Imputation With Indicator : {score[\"IterativeWithIndicator\"]}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Dimentionality-Reduction of OHE cols in Iterative Mode Imputation with MissingIndicator"},{"metadata":{},"cell_type":"markdown","source":"We can also add the sum of missing values to data. This is IterativeImputer with Sum of missing values foe each row."},{"metadata":{"trusted":false},"cell_type":"code","source":"train = base.copy()\ntest  = baseTe.copy()\ntrain['nulls'] = train.isna().sum(axis=1)\ntest['nulls']  = test.isna().sum(axis=1)\nfeatures =  [feat for feat in train.columns if feat not in ['target','id']]\nohe =       [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5', 'nulls']]\nordinal =   [feat for feat in features if feat not in ohe]\ntrain[features] = SiavashMapping(train[features])\ntest[features]  = SiavashMapping(test[features])\nimp =             IterativeImputer(max_iter=500, initial_strategy='most_frequent', random_state=SEED)\ntrain[ordinal] =  imp.fit_transform(train[ordinal])\ntest[ordinal]  =  imp.transform(test[ordinal])\ntrain[ordinal] =  train[ordinal].astype(np.int16)\ntest[ordinal]  =  test[ordinal].astype(np.int16)\nscaler =          RobustScaler(quantile_range=(10.0, 90.0))\noof22, pred22, score['IterativeWithSum'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)\nprint(f'AUC score for Iterative Imputation with sum of missing values: {score[\"IterativeWithSum\"]}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"scores = pd.DataFrame(score, index=['OOF-AUC']).T.sort_values(by='OOF-AUC', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"ax = scores.plot(kind='barh', title ='ROC-AUC Score For Different Imputation Techniques', figsize=(15, 10), legend=True, fontsize=12, alpha = 0.85, cmap = 'gist_gray')\nax.set_xlim((0.7,0.8))\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"More options to consider:\n> * Random Forest from missingpy\n> * Bayesian Binary and Multinomial Logistic from autoimp\n> * PMM, LRD from autoimp\n> * NuclearNormMinimization from fancyimpute\n> * IterativeSVD from fancyimpute\n"},{"metadata":{},"cell_type":"markdown","source":"![](http://)Here we use stacking of top models with RidgeClassifier and RFECV and create the final submission file!"},{"metadata":{"trusted":false},"cell_type":"code","source":"def StackModels():\n    \n    idx = baseTe.id.values\n    train_oofs = [oof1, oof2, oof3, oof4, oof5, oof6, oof7, oof8, oof9, oof10, oof11, oof12, oof13, oof14, oof15, oof16, oof17, oof18, oof19]\n    test_oofs  = [pred1, pred2, pred3, pred4, pred5, pred6, pred7, pred8, pred9, pred10, pred11, pred12, pred13, pred14, pred15, pred16, pred17, pred18, pred19]\n    X_train = pd.concat([pd.DataFrame(file) for file in train_oofs], axis=1)\n    X_test = pd.concat([pd.DataFrame(file) for file in test_oofs], axis=1)\n    X_train.columns = ['y_' + str(i) for i in range(len(train_oofs))]\n    X_test.columns = ['y_' + str(i) for i in range(len(train_oofs))]\n    X_train = pd.concat([X_train, base[['target']]], axis=1)\n    X_test = pd.concat([X_test, baseTe[['target']]], axis=1)\n    for i, (oof, pred) in enumerate(zip(train_oofs, test_oofs)):\n        train_oofs[i] = rankdata(oof)/len(oof)\n        test_oofs[i] = rankdata(pred)/len(pred)\n    for f in X_train.columns:\n        X_train[f] = X_train[f].astype('float32')\n        X_test[f] = X_test[f].astype('float32')  \n    features = np.array([f for f in X_train.columns if f not in ['target']])\n    target = ['target']\n    oof_pred_final = np.zeros((len(base), ))\n    y_pred_final = np.zeros((len(baseTe),))\n    skf = StratifiedKFold(n_splits=SPLITS, shuffle=True, random_state=SEED)\n    model = RidgeClassifier()\n    selector = RFECV(model, step=1, cv = skf, scoring='roc_auc', verbose=0, n_jobs=4)\n    selector.fit(X_train[features], X_train[target])\n    selected_features = [i for i, y in enumerate(selector.ranking_) if y == 1]\n    selected_features = features[selected_features]\n    for fold, (tr_ind, val_ind) in enumerate(skf.split(X_train, X_train[target])):\n        x_tr, x_val = X_train[selected_features].iloc[tr_ind], X_train[selected_features].iloc[val_ind]\n        y_tr, y_val = X_train[target].iloc[tr_ind], X_train[target].iloc[val_ind]\n        train_set = {'X':x_tr, 'y':y_tr}\n        val_set = {'X':x_val, 'y':y_val}\n        model = RidgeClassifier()\n        model.fit(train_set['X'],train_set['y'])\n        fold_pred = model.decision_function(val_set['X'])\n        oof_pred_final[val_ind] = fold_pred\n        y_pred_final += model.decision_function(X_test[selected_features]) / (SPLITS)\n    oof_auc_score = roc_auc_score(base[target], oof_pred_final)\n    print(f'OOF Stack ROC-AUC Score is: {oof_auc_score:.7f}')\n    y_pred_final = rankdata(y_pred_final)/len(y_pred_final)\n    np.save('oof_pred_final.npy',oof_pred_final)\n    np.save('y_pred_final.npy', y_pred_final)\n    print('*'* 36)\n    print('        OOF files saved!')\n    print('*'* 36)\n    submission = pd.DataFrame.from_dict({\n        'id': idx,\n        'target': y_pred_final\n        })\n    submission.to_csv('submission.csv', index=False)\n    print('*'* 36)\n    print('     Submission file saved!')\n    print('*'* 36)\n    \n    return\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"StackModels()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}