{"cells":[{"metadata":{},"cell_type":"markdown","source":"Objective: See how different encoding methods work with Logistic Regression model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport category_encoders as ce\n\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score, balanced_accuracy_score, roc_auc_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.preprocessing import MinMaxScaler, MaxAbsScaler, KBinsDiscretizer, FunctionTransformer\nfrom sklearn.model_selection import cross_val_score\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport gc; gc.enable()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.describe()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filling missing values\nfor col in df:\n    if df[col].isna().sum() > 0:\n        df[col] = df[col].fillna(df[col].mode()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#One Hot Encoding","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = df['target']\nX_train = df.drop(['target', 'id'], axis=1)\nX_test = test_df.drop(['id'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"combined_data = pd.concat([X_train, X_test], axis=0, sort=False)\ncombined_data = pd.get_dummies(combined_data, columns=combined_data.columns, drop_first=True, sparse=True)\nX_train = combined_data.iloc[: len(df)]\nX_test = combined_data.iloc[len(df): ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Shape of training dataset: {X_train.shape}')\nprint(f'Shape of test dataset: {X_test.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#compress data to make everything run faster\nX_train = X_train.sparse.to_coo().tocsr()\nX_test = X_test.sparse.to_coo().tocsr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(X_train, y, test_size=0.30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = LogisticRegression(verbose = 100, max_iter = 600, C=0.5, solver='lbfgs')\n\nlr.fit(x_train, y_train)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_preds = lr.predict_proba(x_train)[:,1]\ntest_preds = lr.predict_proba(x_test)[:,1]\n\nprint(\"AUC:\"); print(\"=\"*len(\"AUC:\"))\nprint(\"TRAIN:\", roc_auc_score(y_train, train_preds))\nprint(\"TEST:\", roc_auc_score(y_test, test_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Target Encoding","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#fill missing values\nfor col in df:\n    if df[col].isna().sum() > 0:\n        df[col] = df[col].fillna(df[col].mode()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"used_cols = [c for c in df.columns.tolist() if c not in ['target', 'id']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create a target encoder\nce_target_encoder = ce.TargetEncoder(cols = used_cols, smoothing=.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = df['target']\nte_df = df.drop(['target', 'id'], axis=1)\nte_test_df = test_df.drop('id', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ce_target_encoder.fit(te_df, target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"te_df = ce_target_encoder.transform(te_df)\nte_test_df = ce_target_encoder.transform(te_test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = target\nX_train = te_df\nX_test = te_test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_id = test_df['id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(X_train, y, test_size=0.30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = LogisticRegression(verbose = 100, max_iter = 600, C=0.5, solver='lbfgs')\n\nlr.fit(x_train, y_train)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_preds = lr.predict_proba(x_train)[:,1]\ntest_preds = lr.predict_proba(x_test)[:,1]\n\nprint(\"AUC:\"); print(\"=\"*len(\"AUC:\"))\nprint(\"TRAIN:\", roc_auc_score(y_train, train_preds))\nprint(\"TEST:\", roc_auc_score(y_test, test_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"../input/cat-in-the-dat-ii/sample_submission.csv\")\nsubmission[\"id\"] = test_id\nsubmission[\"target\"] = lr.predict_proba(X_test)[:, 1]\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* One Hot encoding everything took way longer than Target encoding\n* In this instance Target Encoding performed better\n* Logistic Regression is a good easy to use model\n"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.9"}},"nbformat":4,"nbformat_minor":1}