{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Features interaction test"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport warnings\nwarnings.simplefilter('ignore')\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv', index_col='id')\ntest = pd.read_csv('../input/cat-in-the-dat-ii/test.csv', index_col='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(3).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def summary(df):\n    summary = pd.DataFrame(df.dtypes, columns=['dtypes'])\n    summary = summary.reset_index()\n    summary['Name'] = summary['index']\n    summary = summary[['Name', 'dtypes']]\n    summary['Missing'] = df.isnull().sum().values    \n    summary['Uniques'] = df.nunique().values\n    summary['First Value'] = df.loc[0].values\n    summary['Second Value'] = df.loc[1].values\n    summary['Third Value'] = df.loc[2].values\n    return summary\n\n\nsummary(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Handle missing values"},{"metadata":{},"cell_type":"markdown","source":"Add number of missing values in row as a feature"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['missing_count'] = train.isnull().sum(axis=1)\ntest['missing_count'] = test.isnull().sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Replace missing values with constants"},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_number = -99999\nmissing_string = 'MISSING_STRING'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_features = [\n    'bin_0', 'bin_1', 'bin_2',\n    'ord_0',\n    'day', 'month'\n]\n\nstring_features = [\n    'bin_3', 'bin_4',\n    'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5',\n    'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def impute(train, test, columns, value):\n    for column in columns:\n        train[column] = train[column].fillna(value)\n        test[column] = test[column].fillna(value)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"impute(train, test, numerical_features, missing_number)\nimpute(train, test, string_features, missing_string)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature engineering"},{"metadata":{},"cell_type":"markdown","source":"Split 'ord_5' preserving missing values"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['ord_5_1'] = train['ord_5'].str[0]\ntrain['ord_5_2'] = train['ord_5'].str[1]\n\ntrain.loc[train['ord_5'] == missing_string, 'ord_5_1'] = missing_string\ntrain.loc[train['ord_5'] == missing_string, 'ord_5_2'] = missing_string\n\ntrain = train.drop('ord_5', axis=1)\n\n\ntest['ord_5_1'] = test['ord_5'].str[0]\ntest['ord_5_2'] = test['ord_5'].str[1]\n\ntest.loc[test['ord_5'] == missing_string, 'ord_5_1'] = missing_string\ntest.loc[test['ord_5'] == missing_string, 'ord_5_2'] = missing_string\n\ntest = test.drop('ord_5', axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Features"},{"metadata":{"trusted":true},"cell_type":"code","source":"simple_features = [\n    'missing_count'\n]\n\nohe_features = [\n    'bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4',\n    'nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4',\n    'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5_1', 'ord_5_2',\n    'day', 'month'\n]\n\ntarget_features = [\n    'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9'\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Extract target variable"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train['target'].copy()\nx_train = train.drop('target', axis=1)\ndel train\n\nx_test = test.copy()\ndel test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Standard scaler"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n\nscaler = StandardScaler()\nsimple_x_train = scaler.fit_transform(x_train[simple_features])\nsimple_x_test = scaler.transform(x_test[simple_features])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## OHE"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\n\n\nohe = OneHotEncoder(dtype='uint16', handle_unknown='ignore')\nohe_x_train = ohe.fit_transform(x_train[ohe_features])\nohe_x_test = ohe.transform(x_test[ohe_features])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Target encoder"},{"metadata":{"trusted":true},"cell_type":"code","source":"from category_encoders import TargetEncoder\nfrom sklearn.model_selection import StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform(transformer, x_train, y_train, cv):\n    oof = pd.DataFrame(index=x_train.index, columns=x_train.columns)\n    for train_idx, valid_idx in cv.split(x_train, y_train):\n        x_train_train = x_train.loc[train_idx]\n        y_train_train = y_train.loc[train_idx]\n        x_train_valid = x_train.loc[valid_idx]\n        transformer.fit(x_train_train, y_train_train)\n        oof_part = transformer.transform(x_train_valid)\n        oof.loc[valid_idx] = oof_part\n    return oof","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = TargetEncoder(drop_invariant=True, smoothing=0.2)\n\ncv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\ntarget_x_train = transform(target, x_train[target_features], y_train, cv).astype('float')\n\ntarget.fit(x_train[target_features], y_train)\ntarget_x_test = target.transform(x_test[target_features]).astype('float')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge all"},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy\n\n\nfinal_x_train = scipy.sparse.hstack([simple_x_train, ohe_x_train, target_x_train]).tocsr()\nfinal_x_test = scipy.sparse.hstack([simple_x_test, ohe_x_test, target_x_test]).tocsr()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature interaction test"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test(x_train, y_train):\n    logit = LogisticRegression(C=0.54321, solver='lbfgs', max_iter=5000)\n    scores = []\n    cv = KFold(n_splits=5, random_state=42)\n    for train_idx, valid_idx in cv.split(x_train, y_train):\n        x_train_train = x_train[train_idx]\n        y_train_train = y_train[train_idx]\n        x_train_valid = x_train[valid_idx]\n        y_train_valid = y_train[valid_idx]\n        logit.fit(x_train_train, y_train_train)\n        y_pred = logit.predict_proba(x_train_valid)[:, 1]\n        score = roc_auc_score(y_train_valid, y_pred)\n        print('Fold score:', score)\n        scores.append(score)\n    score = np.mean(scores)\n    print('Average score:', score)\n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"baseline_score = test(final_x_train, y_train)\nbaseline_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Modify these parameters\n\nfeatures_to_test = ohe_features + target_features\n\noffset = 0\n\nrun_time_limit = 120 * 60 # sec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools\n\n\nfeature_combinations = list(itertools.combinations(features_to_test, 2))\nprint('Total number of combinations:', len(feature_combinations))\n\nfeature_combinations = feature_combinations[offset:]\nprint('Number of combinations to test:', len(feature_combinations))\n\n# for features in feature_combinations:\n#     print(features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\n\n\nstart_time = time.time()\n\nnew_features = {}\n\nfor features in feature_combinations:\n    new_feature = features[0] + '__' + features[1]\n    print('Test feature:', new_feature, '/ interaction of', features[0], 'and', features[1])\n    \n    temp = pd.DataFrame(index=x_train.index)\n    temp[new_feature] = x_train[features[0]].astype(str) + '_' + x_train[features[1]].astype(str)\n    ohe = OneHotEncoder(dtype='uint16', handle_unknown='ignore')\n    encoded_temp = ohe.fit_transform(temp)\n    \n    score = test(scipy.sparse.hstack([final_x_train, encoded_temp]).tocsr(), y_train)\n    print('Score =', score)\n    \n    new_features[new_feature] = score\n    \n    # limit execution time\n    run_time = time.time() - start_time\n    if run_time > run_time_limit:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_features_df = pd.DataFrame.from_dict(new_features, orient='index', columns=['score']).sub(baseline_score)\nnew_features_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.figure(figsize=(12, 12))\np = sns.barplot(x=new_features_df.index, y='score', data=new_features_df, color='gray')\n\n# Rotate labels\nfor x in p.get_xticklabels():\n    x.set_rotation(90)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"next_run_offset = offset + len(new_features_df)\nnext_run_offset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_features_df[new_features_df['score'] > 0].index.tolist()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}