{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn import preprocessing\nfrom sklearn import model_selection\nimport os\n\nsns.set_theme()\nsns.set_style(\"ticks\")\nsns.despine()\n\npd.set_option(\"display.max_columns\", None)\npd.set_option(\"display.max_rows\", None)\n\n%matplotlib inline\n\nplt.rc('figure', figsize=(15, 10))\nplt.rc('figure', dpi=300)\nplt.rc('savefig', dpi=300)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-30T04:18:46.673574Z","iopub.execute_input":"2022-11-30T04:18:46.674767Z","iopub.status.idle":"2022-11-30T04:18:48.052206Z","shell.execute_reply.started":"2022-11-30T04:18:46.674627Z","shell.execute_reply":"2022-11-30T04:18:48.050414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    DATA_DIR = \"../input/ai511-homeloan-2022\"\n    \n    @classmethod\n    def get_filepath(cls, filename):\n        return os.path.join(cls.DATA_DIR, filename)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:18:48.056858Z","iopub.execute_input":"2022-11-30T04:18:48.057447Z","iopub.status.idle":"2022-11-30T04:18:48.064186Z","shell.execute_reply.started":"2022-11-30T04:18:48.057387Z","shell.execute_reply":"2022-11-30T04:18:48.062725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Dataset","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(Config.get_filepath(\"train_data.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:18:53.154282Z","iopub.execute_input":"2022-11-30T04:18:53.154705Z","iopub.status.idle":"2022-11-30T04:18:56.263891Z","shell.execute_reply.started":"2022-11-30T04:18:53.154676Z","shell.execute_reply":"2022-11-30T04:18:56.262892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 184,506 training examples and 121 features, excluding the target.","metadata":{}},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:18:56.265530Z","iopub.execute_input":"2022-11-30T04:18:56.265873Z","iopub.status.idle":"2022-11-30T04:18:56.277449Z","shell.execute_reply.started":"2022-11-30T04:18:56.265841Z","shell.execute_reply":"2022-11-30T04:18:56.275538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:18:56.279907Z","iopub.execute_input":"2022-11-30T04:18:56.280297Z","iopub.status.idle":"2022-11-30T04:18:56.373148Z","shell.execute_reply.started":"2022-11-30T04:18:56.280271Z","shell.execute_reply":"2022-11-30T04:18:56.371711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Change all column names to lowercase so that it is easier to refer to them\ndf.columns = df.columns.str.lower()\ndf.columns","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:18:56.377171Z","iopub.execute_input":"2022-11-30T04:18:56.377494Z","iopub.status.idle":"2022-11-30T04:18:56.389707Z","shell.execute_reply.started":"2022-11-30T04:18:56.377467Z","shell.execute_reply":"2022-11-30T04:18:56.388522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target Distribution\n\nThe target is binary and extremely imbalanced. ~92% of borrowers did not default on their loan while ~8% of them did. Clearly, any model we train will have to be configured to handle this imbalance.","metadata":{}},{"cell_type":"code","source":"df[\"target\"].value_counts(normalize=True) * 100","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:18:57.360732Z","iopub.execute_input":"2022-11-30T04:18:57.361140Z","iopub.status.idle":"2022-11-30T04:18:57.379013Z","shell.execute_reply.started":"2022-11-30T04:18:57.361112Z","shell.execute_reply":"2022-11-30T04:18:57.377367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extracting Categorical Features","metadata":{}},{"cell_type":"markdown","source":"Some obvious categorical features can be extracted using `select_dtypes()` and excluding all numeric columns. There are 16 such features. These features need to be encoded, either using one-hot encoding or label encoding, to be used in a model.","metadata":{}},{"cell_type":"code","source":"non_numeric = df.select_dtypes(exclude=\"number\")","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:18:58.675291Z","iopub.execute_input":"2022-11-30T04:18:58.675750Z","iopub.status.idle":"2022-11-30T04:18:58.713563Z","shell.execute_reply.started":"2022-11-30T04:18:58.675712Z","shell.execute_reply":"2022-11-30T04:18:58.711807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_numeric.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:18:59.185107Z","iopub.execute_input":"2022-11-30T04:18:59.185576Z","iopub.status.idle":"2022-11-30T04:18:59.205632Z","shell.execute_reply.started":"2022-11-30T04:18:59.185542Z","shell.execute_reply":"2022-11-30T04:18:59.204026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below, we store these feature names in a list so that we can use them later.","metadata":{}},{"cell_type":"code","source":"cat_cols = non_numeric.columns.to_list()\ncat_cols","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:01.091482Z","iopub.execute_input":"2022-11-30T04:19:01.091912Z","iopub.status.idle":"2022-11-30T04:19:01.099512Z","shell.execute_reply.started":"2022-11-30T04:19:01.091884Z","shell.execute_reply":"2022-11-30T04:19:01.098597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Additionally, we also know that any feature thats starts with `flag_` is categorical since it is a binary feature. We will add those to the list of categorical features. Note that doing this will add `flag_own_car` and `flag_own_realty` twice, which we will remove.","metadata":{}},{"cell_type":"code","source":"cat_cols.extend(df.columns[df.columns.str.startswith(\"flag_\")].to_list())\ncat_cols.remove(\"flag_own_car\")\ncat_cols.remove(\"flag_own_realty\")\ncat_cols","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:04.015733Z","iopub.execute_input":"2022-11-30T04:19:04.016230Z","iopub.status.idle":"2022-11-30T04:19:04.026589Z","shell.execute_reply.started":"2022-11-30T04:19:04.016187Z","shell.execute_reply":"2022-11-30T04:19:04.025206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We find additional categorical columns using `value_counts()`.","metadata":{}},{"cell_type":"code","source":"l = [\n    \"cnt_children\",\n    \"cnt_fam_members\",\n    \"region_rating_client\",\n    \"region_rating_client_w_city\",\n    \"reg_region_not_live_region\",\n    \"reg_region_not_work_region\",\n    \"live_region_not_work_region\",\n    \"reg_city_not_live_city\",\n    \"reg_city_not_work_city\",\n    \"live_city_not_work_city\",\n    \"amt_req_credit_bureau_hour\",\n    \"amt_req_credit_bureau_day\",\n    \"amt_req_credit_bureau_week\",\n    \"amt_req_credit_bureau_mon\",\n    \"amt_req_credit_bureau_qrt\",\n    \"amt_req_credit_bureau_year\",\n]\n\nit_l = iter(l)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:07.513485Z","iopub.execute_input":"2022-11-30T04:19:07.514806Z","iopub.status.idle":"2022-11-30T04:19:07.520684Z","shell.execute_reply.started":"2022-11-30T04:19:07.514735Z","shell.execute_reply":"2022-11-30T04:19:07.519430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run this cell repeatedly to see the value_counts() output of each of the columns\ndf[next(it_l)].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:12.771904Z","iopub.execute_input":"2022-11-30T04:19:12.772376Z","iopub.status.idle":"2022-11-30T04:19:12.782455Z","shell.execute_reply.started":"2022-11-30T04:19:12.772335Z","shell.execute_reply":"2022-11-30T04:19:12.781709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols.extend(l)\ncat_cols","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:14.439883Z","iopub.execute_input":"2022-11-30T04:19:14.440550Z","iopub.status.idle":"2022-11-30T04:19:14.450032Z","shell.execute_reply.started":"2022-11-30T04:19:14.440505Z","shell.execute_reply":"2022-11-30T04:19:14.448690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the end, we end up with 62 categorical features.","metadata":{}},{"cell_type":"code","source":"len(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:16.923825Z","iopub.execute_input":"2022-11-30T04:19:16.925337Z","iopub.status.idle":"2022-11-30T04:19:16.933028Z","shell.execute_reply.started":"2022-11-30T04:19:16.925288Z","shell.execute_reply":"2022-11-30T04:19:16.931542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extracting Numerical Features\n\nAll features other than the target, the ID column and the categorical features are numerical.","metadata":{}},{"cell_type":"code","source":"numerical_cols = df.columns.drop(cat_cols + [\"target\", \"sk_id_curr\"]).to_list()\nnumerical_cols","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:18.815521Z","iopub.execute_input":"2022-11-30T04:19:18.817027Z","iopub.status.idle":"2022-11-30T04:19:18.828371Z","shell.execute_reply.started":"2022-11-30T04:19:18.816975Z","shell.execute_reply":"2022-11-30T04:19:18.827087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the end, we end up with 58 numerical features.","metadata":{}},{"cell_type":"code","source":"len(numerical_cols)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:21.149330Z","iopub.execute_input":"2022-11-30T04:19:21.149784Z","iopub.status.idle":"2022-11-30T04:19:21.156298Z","shell.execute_reply.started":"2022-11-30T04:19:21.149753Z","shell.execute_reply":"2022-11-30T04:19:21.155190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Null Values","metadata":{}},{"cell_type":"markdown","source":"According to [this discussion](https://www.kaggle.com/competitions/home-credit-default-risk/discussion/57248), there are some columns which have `XNA` or `XAP` to denote null values. Below, we find such columns and replace those values with `np.nan`.","metadata":{}},{"cell_type":"code","source":"x_null_cols = [col for col in df.columns if ((df[col] == \"XNA\") | (df[col] == \"XAP\")).any()]\n\nfor col in x_null_cols:\n    df.loc[(df[col] == \"XNA\") | (df[col] == \"XAP\"), col] = np.nan","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:24.183447Z","iopub.execute_input":"2022-11-30T04:19:24.184050Z","iopub.status.idle":"2022-11-30T04:19:24.640741Z","shell.execute_reply.started":"2022-11-30T04:19:24.183992Z","shell.execute_reply":"2022-11-30T04:19:24.639917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below, we take a look at which categorical features have null values in them. Luckily, we see that only a few of the categorical features have null values.","metadata":{}},{"cell_type":"code","source":"df[cat_cols].isna().any()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:28.027429Z","iopub.execute_input":"2022-11-30T04:19:28.028246Z","iopub.status.idle":"2022-11-30T04:19:28.199229Z","shell.execute_reply.started":"2022-11-30T04:19:28.028203Z","shell.execute_reply":"2022-11-30T04:19:28.198301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_null_cols = [col for col in cat_cols if df[col].isna().any()]\ncat_null_cols","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:31.170593Z","iopub.execute_input":"2022-11-30T04:19:31.171395Z","iopub.status.idle":"2022-11-30T04:19:31.290877Z","shell.execute_reply.started":"2022-11-30T04:19:31.171354Z","shell.execute_reply":"2022-11-30T04:19:31.289932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below, we visualize the proportion of null values in each categorical features. Some interesting things can be observed from the pie chart below:\n\n- All `amt_req` features have the same number of null values. This suggests that there is some relationship between these null values that needs to be explored.\n- All `obs_` and `def_` features have the same number of null values. This leads to a similar conclusion as above.\n- The features related to the current housing information of the borrower have a very proportion of null values.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 15))\ndata = [(sum(df[col].isna()) / len(df)) * 100 for col in cat_null_cols]\nax = sns.barplot(x=data, y=cat_null_cols, orient=\"h\")\n_ = ax.bar_label(ax.containers[0])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:32.871841Z","iopub.execute_input":"2022-11-30T04:19:32.872293Z","iopub.status.idle":"2022-11-30T04:19:34.742098Z","shell.execute_reply.started":"2022-11-30T04:19:32.872259Z","shell.execute_reply":"2022-11-30T04:19:34.740491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We do the same thing for the numerical features. Note that according to [this discussion](https://www.kaggle.com/competitions/home-credit-default-risk/discussion/57247), the value `365243` is a standin for null values in `days_employed`. Thus, we will replace it will `np.nan`.","metadata":{}},{"cell_type":"code","source":"df.loc[df[\"days_employed\"] == 365243, \"days_employed\"] = np.nan\nnumerical_null_cols = [col for col in numerical_cols if df[col].isna().any()]\nnumerical_null_cols","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:40.446263Z","iopub.execute_input":"2022-11-30T04:19:40.446763Z","iopub.status.idle":"2022-11-30T04:19:40.492474Z","shell.execute_reply.started":"2022-11-30T04:19:40.446721Z","shell.execute_reply":"2022-11-30T04:19:40.491281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Similar to above, the most interesting observation from the plot below is that the null values are almost exclusively present in features related to current housing information of a borrower. All features which have `_avg`, `_mode` and `_medi` as suffixes are related to the current housing information of the borrorwer. There might be a relationship between this presence of null values and the target. We will explore this later.\n\nIt is also evident that the amount of null values is very high for almost all the features. A few features like `amt_annuity`, `amt_goods_price` and `day_last_phone_change` have very small proportion of null values (less than 0.1%) in them. It is safe to say that dropping those rows where these columns are null won't affect the size of the dataset or the distribution of the target significantly.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20, 15))\ndata = [(sum(df[col].isna()) / len(df)) * 100 for col in numerical_null_cols]\nax = sns.barplot(x=data, y=numerical_null_cols, orient=\"h\")\n_ = ax.bar_label(ax.containers[0])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:43.121432Z","iopub.execute_input":"2022-11-30T04:19:43.122079Z","iopub.status.idle":"2022-11-30T04:19:46.784873Z","shell.execute_reply.started":"2022-11-30T04:19:43.122042Z","shell.execute_reply":"2022-11-30T04:19:46.783804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below, we remove null-containing rows with respect to those features which have less than 0.5% null values since removing this fraction of rows will not have that great an affect on the dataset","metadata":{}},{"cell_type":"code","source":"threshold = 5e-3\nto_be_dropped = [col for col in cat_null_cols if sum(df[col].isna()) / len(df) < threshold]\nto_be_dropped.extend(col for col in numerical_null_cols if sum(df[col].isna()) / len(df) < threshold)\n\nfor col in to_be_dropped:\n    null_rows = df[col].isna()\n    prop = sum(null_rows) / len(df)\n    if prop < threshold:\n        df = df.dropna(subset=[col])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:54.231129Z","iopub.execute_input":"2022-11-30T04:19:54.231580Z","iopub.status.idle":"2022-11-30T04:19:57.347788Z","shell.execute_reply.started":"2022-11-30T04:19:54.231541Z","shell.execute_reply":"2022-11-30T04:19:57.346335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After the removal, the distribution of null values is as shown below.","metadata":{}},{"cell_type":"code","source":"cat_null_cols = [col for col in cat_null_cols if sum(df[col].isna()) / len(df) != 0]\nnumerical_null_cols = [col for col in numerical_null_cols if sum(df[col].isna()) / len(df) != 0]","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:57.349817Z","iopub.execute_input":"2022-11-30T04:19:57.350554Z","iopub.status.idle":"2022-11-30T04:19:58.667159Z","shell.execute_reply.started":"2022-11-30T04:19:57.350520Z","shell.execute_reply":"2022-11-30T04:19:58.666018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 15))\ndata = [(sum(df[col].isna()) / len(df)) * 100 for col in cat_null_cols]\nax = sns.barplot(x=data, y=cat_null_cols, orient=\"h\")\n_ = ax.bar_label(ax.containers[0])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:19:58.668899Z","iopub.execute_input":"2022-11-30T04:19:58.669155Z","iopub.status.idle":"2022-11-30T04:20:00.400528Z","shell.execute_reply.started":"2022-11-30T04:19:58.669130Z","shell.execute_reply":"2022-11-30T04:20:00.399486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 15))\ndata = [(sum(df[col].isna()) / len(df)) * 100 for col in numerical_null_cols]\nax = sns.barplot(x=data, y=numerical_null_cols, orient=\"h\")\n_ = ax.bar_label(ax.containers[0])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:20:00.482619Z","iopub.execute_input":"2022-11-30T04:20:00.483116Z","iopub.status.idle":"2022-11-30T04:20:03.640810Z","shell.execute_reply.started":"2022-11-30T04:20:00.483076Z","shell.execute_reply":"2022-11-30T04:20:03.639234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Due to the removal, we have only lost ~`1%` of the training examples and the distribution of the target is mostly unchanged.","metadata":{}},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:20:12.723781Z","iopub.execute_input":"2022-11-30T04:20:12.724971Z","iopub.status.idle":"2022-11-30T04:20:12.731866Z","shell.execute_reply.started":"2022-11-30T04:20:12.724924Z","shell.execute_reply":"2022-11-30T04:20:12.730178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"target\"].value_counts(normalize=True) * 100","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:20:14.340077Z","iopub.execute_input":"2022-11-30T04:20:14.340543Z","iopub.status.idle":"2022-11-30T04:20:14.352247Z","shell.execute_reply.started":"2022-11-30T04:20:14.340502Z","shell.execute_reply":"2022-11-30T04:20:14.350892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Per-feature Insights in Some Numerical Variables","metadata":{}},{"cell_type":"markdown","source":"Before we look at some numerical features, we engineer 3 new features.\n\n- `loan_rate` - This is the ratio between `amt_credit` and `amt_annuity`. Thus, it is a direct measure of the duration of the loan.\n- `loan_income_ratio` - This is the ratio between `amt_credit` and `amt_income_total`. Thus, it acts as an indirect measure of the amount of liability that the applicant is taking on themselves. A higher value indicates that the loan amount is much bigger than the applicant's income, suggesting that they've taken a bigger liability than someone with a smaller ratio.\n- `annuity_income_ratio` -This is the ratio betweem `amt_annuity` and `amt_income_total`. It is another measure of liability but measures it a finer timescale (per month vs over multiple months) since `amt_annuity` is the amount to be paid each month against the loan.","metadata":{}},{"cell_type":"code","source":"df[\"loan_rate\"] = df[\"amt_credit\"] / df[\"amt_annuity\"]\ndf[\"loan_income_ratio\"] = df[\"amt_credit\"] / df[\"amt_income_total\"]\ndf[\"annuity_income_ratio\"] = df[\"amt_annuity\"] / df[\"amt_income_total\"]\n\nnumerical_cols.extend(\n    [\"loan_rate\", \"loan_income_ratio\", \"annuity_income_ratio\"]\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:20:16.423578Z","iopub.execute_input":"2022-11-30T04:20:16.424812Z","iopub.status.idle":"2022-11-30T04:20:16.437830Z","shell.execute_reply.started":"2022-11-30T04:20:16.424765Z","shell.execute_reply":"2022-11-30T04:20:16.435370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = [\n    \"amt_income_total\",\n    \"amt_credit\",\n    \"amt_annuity\",\n    \"loan_rate\",\n    \"loan_income_ratio\",\n    \"annuity_income_ratio\",\n    \"amt_goods_price\",\n    \"ext_source_1\",\n    \"ext_source_2\",\n    \"ext_source_3\",\n    \"days_employed\",\n    \"days_registration\",\n    \"obs_60_cnt_social_circle\",\n    \"def_60_cnt_social_circle\",\n    \"livingarea_medi\",\n]","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:24:02.133051Z","iopub.execute_input":"2022-11-30T04:24:02.133502Z","iopub.status.idle":"2022-11-30T04:24:02.140190Z","shell.execute_reply.started":"2022-11-30T04:24:02.133465Z","shell.execute_reply":"2022-11-30T04:24:02.138917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution","metadata":{}},{"cell_type":"markdown","source":"Below, we look at the distributions of the above numerical features.\n\nWe see that most of the features are quite skewed in nature. Depending on the model we use, it might be necessary to apply some transformations on these features to make the model perform better and faster.","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(5, 3, figsize=(20, 15))\n\nfor col, ax in zip(l, axes.flat):\n    sns.kdeplot(data=df, x=col, fill=True, ax=ax)\n    \nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:24:02.815779Z","iopub.execute_input":"2022-11-30T04:24:02.816223Z","iopub.status.idle":"2022-11-30T04:24:16.791291Z","shell.execute_reply.started":"2022-11-30T04:24:02.816189Z","shell.execute_reply":"2022-11-30T04:24:16.789751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution by Target","metadata":{}},{"cell_type":"markdown","source":"We first make a violin plot of all the above features with respect to the target. Some interesting observations are:\n\n- In case of `amt_income_total`, the variability is too low to make any sort of conclusion.\n- The `ext_source_` features show the most difference in distributions with respect to the target. It is likely that they will play an important role in any model.\n- The other features have small differences in distributions but the differences are still present.","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(5, 3, figsize=(20, 15))\n\nfor col, ax in zip(l, axes.flat):\n    sns.violinplot(data=df, x=\"target\", y=col, ax=ax)\n    \nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:24:16.793580Z","iopub.execute_input":"2022-11-30T04:24:16.793949Z","iopub.status.idle":"2022-11-30T04:24:38.712588Z","shell.execute_reply.started":"2022-11-30T04:24:16.793921Z","shell.execute_reply":"2022-11-30T04:24:38.711365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The difference in the distributions with respect to the target can also be seen in the plots below.","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(5, 3, figsize=(20, 15))\n\nfor col, ax in zip(l, axes.flat):\n    sns.kdeplot(data=df, x=col, hue=\"target\", fill=True, ax=ax)\n    \nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:24:38.714050Z","iopub.execute_input":"2022-11-30T04:24:38.714427Z","iopub.status.idle":"2022-11-30T04:24:53.955999Z","shell.execute_reply.started":"2022-11-30T04:24:38.714391Z","shell.execute_reply":"2022-11-30T04:24:53.954436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Null Value Analysis of `ext_source` Features","metadata":{}},{"cell_type":"markdown","source":"We know from the overall null value analysis above that two of these features have null values in them, with `ext_source_1` having the highest amount. Below, we see if the presence of null values leads to some differences in the target. This can allow us to engineer three new features, denoting whether the corresponding `ext_source` feature is null for that row and lead to a better performing model.\n\nWe see that in case of `ext_source_1`, there is a difference between how the target behaves when there are null values present. When there are null values, it is more likely that the applicant will default on the loan while when there are no null values, they are more likely to not default on the loan. `ext_source_3` shows the same difference.\n\nThus, we will add two features, `es1_is_missing` and `es3_is_missing` which denote whether the corresponding feature is null for that row or not.","metadata":{}},{"cell_type":"code","source":"features = [\"ext_source_1\", \"ext_source_3\"]\n\nf, axes = plt.subplots(1, 2, figsize=(15, 10))\n\nfor feat, ax in zip(features, axes.flat):\n    sns.histplot(\n        x=df[feat].isna().astype(\"str\"),\n        hue=df[\"target\"],\n        multiple=\"dodge\",\n        stat=\"percent\",\n        shrink=0.8,\n        common_norm=False,\n        ax=ax\n    )\n    ax.bar_label(ax.containers[0])\n    ax.bar_label(ax.containers[1])\n    \nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-11-13T16:54:45.513518Z","iopub.execute_input":"2022-11-13T16:54:45.513967Z","iopub.status.idle":"2022-11-13T16:54:47.788237Z","shell.execute_reply.started":"2022-11-13T16:54:45.513927Z","shell.execute_reply":"2022-11-13T16:54:47.787057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[[\"es1_is_missing\", \"es3_is_missing\"]] = df[[\"ext_source_1\", \"ext_source_3\"]].isna()\ncat_cols.extend([\"es1_is_missing\", \"es3_is_missing\"])","metadata":{"execution":{"iopub.status.busy":"2022-11-13T16:54:47.789869Z","iopub.execute_input":"2022-11-13T16:54:47.790731Z","iopub.status.idle":"2022-11-13T16:54:47.878019Z","shell.execute_reply.started":"2022-11-13T16:54:47.790697Z","shell.execute_reply":"2022-11-13T16:54:47.876607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Per-Feature Insights in Some Categorical Features","metadata":{}},{"cell_type":"code","source":"l = [\n    \"name_contract_type\",\n    \"code_gender\",\n    \"name_income_type\",\n    \"name_education_type\",\n    \"name_family_status\",\n    \"name_housing_type\",\n    \"occupation_type\",\n    \"flag_own_car\",\n    \"flag_own_realty\",\n    \"cnt_children\",\n    \"cnt_fam_members\",\n    \"region_rating_client_w_city\",\n]","metadata":{"execution":{"iopub.status.busy":"2022-11-13T16:54:47.880225Z","iopub.execute_input":"2022-11-13T16:54:47.880693Z","iopub.status.idle":"2022-11-13T16:54:47.887623Z","shell.execute_reply.started":"2022-11-13T16:54:47.880650Z","shell.execute_reply":"2022-11-13T16:54:47.885718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution","metadata":{}},{"cell_type":"markdown","source":"We see that:\n\n- Most loans are cash loans.\n- Most applicants are female.\n- The most common source of income is working (salaried employee)\n- The most common education level is secondary and most of the applicants are married.\n- Most applicants do not own a car but do own some form of realty.\n- Most applicants are from a region with a rating of 2.\n\nNote that some bars are not visible for some categories. That's because those categories are present in very small number.","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(4, 3, figsize=(25, 20))\n\nfor col, ax in zip(l, axes.flat):\n    sns.countplot(data=df, y=col, orient=\"h\", ax=ax)\n    \nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-11-13T16:54:47.889434Z","iopub.execute_input":"2022-11-13T16:54:47.889854Z","iopub.status.idle":"2022-11-13T16:55:19.485127Z","shell.execute_reply.started":"2022-11-13T16:54:47.889819Z","shell.execute_reply":"2022-11-13T16:55:19.483673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution by Target","metadata":{}},{"cell_type":"markdown","source":"Note: Some percentages are coming up as `0` but are actually only close to `0`. They appear as `0` due to the `fmt` parameter in `bar_label()`.\n\nWe see that there are significant differences in the distribution of the features with respect to the target. For example:\n\n- Cash loans have a higher default rate than revolving loans.\n- An applicant who owns a car or some form of realty is less likely to default on the loan.\n- Applicants with fewer family members are less likely to default on the loan.\n- Applicants living in regions with a rating of 3 are more likely to default on their loan.\n- Married applicants are less likely to default on their loans, etc.","metadata":{}},{"cell_type":"code","source":"f, axes = plt.subplots(4, 3, figsize=(25, 20))\n\nfor col, ax in zip(l, axes.flat):\n    sns.histplot(\n        y=df[col].astype(\"str\"),\n        hue=df[\"target\"],\n        multiple=\"dodge\",\n        stat=\"percent\",\n        shrink=0.8,\n        common_norm=False,\n        ax=ax,\n    )\n    ax.bar_label(ax.containers[0], fmt=\"%.3f\", fontsize=\"x-small\")\n    ax.bar_label(ax.containers[1], fmt=\"%.3f\", fontsize=\"x-small\")\n    ax.legend([\"1\", \"0\"], loc=\"lower right\", title=\"target\")\n    \nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-11-13T16:55:19.486690Z","iopub.execute_input":"2022-11-13T16:55:19.487015Z","iopub.status.idle":"2022-11-13T16:55:31.167472Z","shell.execute_reply.started":"2022-11-13T16:55:19.486985Z","shell.execute_reply":"2022-11-13T16:55:31.165939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We'll also look one additional feature that seems important but can't be visualized conveniently in these group plots.","metadata":{}},{"cell_type":"markdown","source":"### `organization_type`\n\nWe observe significant differences in the distribution of the feature with respect to the target. For example, organizations belonging to categories `Business Entity Type 3`, `Construction`, `Self-Employed`, `Trade: type 7`, `Transport: type 3` are much more likely to default on their loan.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(25, 30))\nax = sns.histplot(y=df[\"organization_type\"], hue=df[\"target\"], stat=\"percent\", multiple=\"dodge\", shrink=0.8, common_norm=False)\n_ = ax.bar_label(ax.containers[0], fmt=\"%.3f\")\n_ = ax.bar_label(ax.containers[1], fmt=\"%.3f\")","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:25:31.233813Z","iopub.execute_input":"2022-11-30T04:25:31.235376Z","iopub.status.idle":"2022-11-30T04:25:36.453015Z","shell.execute_reply.started":"2022-11-30T04:25:31.235297Z","shell.execute_reply":"2022-11-30T04:25:36.451583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Current Housing Features\n\nA major problem in the dataset is that all the features related to the current housing of the applicant have large number of null values in them. It would not be entirely wise to do some naive imputation on them. Rather, it would be better if this missing value data can be aggregated into a feature.\n\nWe can say that if some of the features have null values, it implies that the applicant submitted an incomplete form. Thus, we can create a feature which denotes whether the applicant submitted an incomplete application or not. Intuition tells us that an applicant with an incomplete application is more likely to default on their loan.\n\nWe will use the following heuristic to create this feature: If more than 35 fields related to the application are null, the application is incomplete. Otherwise, it is complete.","metadata":{}},{"cell_type":"code","source":"application_feats = [\n    col for col in df.columns\n    if col.endswith(\"_avg\") or\n    col.endswith(\"_mode\") or\n    col.endswith(\"_medi\") or \n    col.find(\"_document_\") != -1\n]\napplication_feats.extend(\n    [\"flag_mobil\", \"flag_emp_phone\", \"flag_work_phone\", \"flag_email\", \"flag_phone\"]\n)\n\ndf[\"application_is_incomplete\"] = df[application_feats].isna().sum(axis=1) > 35\ncat_cols.append(\"application_is_incomplete\")","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:25:36.454985Z","iopub.execute_input":"2022-11-30T04:25:36.455704Z","iopub.status.idle":"2022-11-30T04:25:36.594992Z","shell.execute_reply.started":"2022-11-30T04:25:36.455673Z","shell.execute_reply":"2022-11-30T04:25:36.592802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that there is a clear difference in the distribution of the feature with respect to the target. An applicant is signifiacntly more likely to default if the application is incomplete.","metadata":{}},{"cell_type":"code","source":"ax = sns.histplot(x=df[\"application_is_incomplete\"].astype(\"str\"), hue=df[\"target\"], stat=\"percent\", multiple=\"dodge\", shrink=0.8, common_norm=False)\n_ = ax.bar_label(ax.containers[0])\n_ = ax.bar_label(ax.containers[1])","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:00.401028Z","iopub.execute_input":"2022-11-30T04:26:00.402049Z","iopub.status.idle":"2022-11-30T04:26:01.748066Z","shell.execute_reply.started":"2022-11-30T04:26:00.402006Z","shell.execute_reply":"2022-11-30T04:26:01.746118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before we proceed any further, we need to add all the features we have engineered to the test set. Moreover, we need to replace `XNA` and `XAP` from the test set as well.","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(Config.get_filepath(\"test_data.csv\"))\ntest_df.columns = test_df.columns.str.lower()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:02.482033Z","iopub.execute_input":"2022-11-30T04:26:02.482473Z","iopub.status.idle":"2022-11-30T04:26:04.468171Z","shell.execute_reply.started":"2022-11-30T04:26:02.482441Z","shell.execute_reply":"2022-11-30T04:26:04.467087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_null_cols = [col for col in test_df.columns if ((test_df[col] == \"XNA\") | (test_df[col] == \"XAP\")).any()]\n\nfor col in x_null_cols:\n    test_df.loc[(test_df[col] == \"XNA\") | (test_df[col] == \"XAP\"), col] = np.nan","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:05.399175Z","iopub.execute_input":"2022-11-30T04:26:05.400288Z","iopub.status.idle":"2022-11-30T04:26:05.711390Z","shell.execute_reply.started":"2022-11-30T04:26:05.400208Z","shell.execute_reply":"2022-11-30T04:26:05.710161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Ratios\ntest_df[\"loan_rate\"] = test_df[\"amt_credit\"] / test_df[\"amt_annuity\"]\ntest_df[\"loan_income_ratio\"] = test_df[\"amt_credit\"] / test_df[\"amt_income_total\"]\ntest_df[\"annuity_income_ratio\"] = test_df[\"amt_annuity\"] / test_df[\"amt_income_total\"]\n\n# es1_is_missing and es3_is_missing\ntest_df[[\"es1_is_missing\", \"es3_is_missing\"]] = test_df[[\"ext_source_1\", \"ext_source_3\"]].isna()\n\n# application_is_incomplete\ntest_df[\"application_is_incomplete\"] = test_df[application_feats].isna().sum(axis=1) > 35","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:07.473334Z","iopub.execute_input":"2022-11-30T04:26:07.474692Z","iopub.status.idle":"2022-11-30T04:26:07.555700Z","shell.execute_reply.started":"2022-11-30T04:26:07.474632Z","shell.execute_reply":"2022-11-30T04:26:07.554866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Folds for Cross Validation\n\nWe will use stratified K-fold cross validation as our CV strategy due to the imbalance in the dataset. We will then save the datasets thus generated.","metadata":{}},{"cell_type":"code","source":"df = df.reset_index()\n\ndf[\"kfold\"] = -1\n\nskf = model_selection.StratifiedKFold(n_splits=5, random_state=42, shuffle=True)\ngroups = skf.split(X=df.drop(\"target\", axis=1), y=df[\"target\"])\n\nfor idx, (_, val_idx) in enumerate(groups):\n    df.loc[val_idx, \"kfold\"] = idx\n\ndf = df.set_index(\"index\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:09.518849Z","iopub.execute_input":"2022-11-30T04:26:09.519671Z","iopub.status.idle":"2022-11-30T04:26:09.833568Z","shell.execute_reply.started":"2022-11-30T04:26:09.519602Z","shell.execute_reply":"2022-11-30T04:26:09.832714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"./train_with_new_features.csv\", index=False)\ntest_df.to_csv(\"./test_with_new_features.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:17.175075Z","iopub.execute_input":"2022-11-30T04:26:17.175583Z","iopub.status.idle":"2022-11-30T04:26:31.140895Z","shell.execute_reply.started":"2022-11-30T04:26:17.175539Z","shell.execute_reply":"2022-11-30T04:26:31.139234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imputing Missing Values\n\nModels such as XGBoost and LightGBM can handle null values gracefully, usually not requiring explicit imputation. But one of the goals here is to train a logistic regression model as the first baseline which we will try to beat with more complex models. Logistic regression cannot handle missing values and thus, we need to impute them for the time being.\n\nFor numerical features, we will use the median (since it is more robust when there are outliers) to replace null values in each column and for categorical features, we will use the mode.","metadata":{}},{"cell_type":"code","source":"for col in numerical_cols:\n    df.loc[df[col].isna(), col] = df[col].median()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:31.143709Z","iopub.execute_input":"2022-11-30T04:26:31.144230Z","iopub.status.idle":"2022-11-30T04:26:31.488140Z","shell.execute_reply.started":"2022-11-30T04:26:31.144179Z","shell.execute_reply":"2022-11-30T04:26:31.487383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[numerical_cols].isna().any()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:31.489732Z","iopub.execute_input":"2022-11-30T04:26:31.490308Z","iopub.status.idle":"2022-11-30T04:26:31.894904Z","shell.execute_reply.started":"2022-11-30T04:26:31.490273Z","shell.execute_reply":"2022-11-30T04:26:31.893343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat_cols:\n    df.loc[df[col].isna(), col] = df[col].mode()[0]","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:31.898064Z","iopub.execute_input":"2022-11-30T04:26:31.898772Z","iopub.status.idle":"2022-11-30T04:26:32.330449Z","shell.execute_reply.started":"2022-11-30T04:26:31.898727Z","shell.execute_reply":"2022-11-30T04:26:32.329688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[cat_cols].isna().any()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:32.331841Z","iopub.execute_input":"2022-11-30T04:26:32.332314Z","iopub.status.idle":"2022-11-30T04:26:32.491302Z","shell.execute_reply.started":"2022-11-30T04:26:32.332281Z","shell.execute_reply":"2022-11-30T04:26:32.490321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scaling Numerical Features\n\nWe will now use min-max scaling so that all numerical features are in the same range of 0-1. This will improve the training speed and also improve numerical stability.","metadata":{}},{"cell_type":"code","source":"scaler = preprocessing.MinMaxScaler()\n\ndf[numerical_cols] = scaler.fit_transform(df[numerical_cols].to_numpy())","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:32.492985Z","iopub.execute_input":"2022-11-30T04:26:32.493386Z","iopub.status.idle":"2022-11-30T04:26:32.688890Z","shell.execute_reply.started":"2022-11-30T04:26:32.493351Z","shell.execute_reply":"2022-11-30T04:26:32.687139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[numerical_cols].describe()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:32.690582Z","iopub.execute_input":"2022-11-30T04:26:32.691038Z","iopub.status.idle":"2022-11-30T04:26:33.230325Z","shell.execute_reply.started":"2022-11-30T04:26:32.690992Z","shell.execute_reply":"2022-11-30T04:26:33.228970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before proceeding further, we will repeat the steps for the test set. Note that the test set uses the median and mode of the train set, and the same scaler that was fit on the train set. This is to prevent information leakage.","metadata":{}},{"cell_type":"code","source":"for col in numerical_cols:\n    test_df.loc[test_df[col].isna(), col] = df[col].median()\n    \nfor col in cat_cols:\n    test_df.loc[test_df[col].isna(), col] = df[col].mode()[0]\n    \ntest_df[numerical_cols] = scaler.transform(test_df[numerical_cols].to_numpy())","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:33.234215Z","iopub.execute_input":"2022-11-30T04:26:33.235127Z","iopub.status.idle":"2022-11-30T04:26:33.972696Z","shell.execute_reply.started":"2022-11-30T04:26:33.234794Z","shell.execute_reply":"2022-11-30T04:26:33.971879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"./train_scaled_no_nulls.csv\", index=False)\ntest_df.to_csv(\"./test_scaled_no_nulls.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:33.973769Z","iopub.execute_input":"2022-11-30T04:26:33.974763Z","iopub.status.idle":"2022-11-30T04:26:53.170996Z","shell.execute_reply.started":"2022-11-30T04:26:33.974733Z","shell.execute_reply":"2022-11-30T04:26:53.169611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoding Categorical Variables\n\nWe will encode the categorical variables using both one-hot encoding and label encoding, and save both the datasets. This will allow us to experiment with both the methods and see which performs better. Note that in case of one-hot encoding, a feature will be one-hot encoded only if it is not binary.","metadata":{}},{"cell_type":"markdown","source":"## One-Hot Encoding","metadata":{}},{"cell_type":"code","source":"df_ohe = df.copy()\ntest_df_ohe = test_df.copy()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:53.174865Z","iopub.execute_input":"2022-11-30T04:26:53.175994Z","iopub.status.idle":"2022-11-30T04:26:53.255864Z","shell.execute_reply.started":"2022-11-30T04:26:53.175937Z","shell.execute_reply":"2022-11-30T04:26:53.254332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = preprocessing.OneHotEncoder(handle_unknown=\"ignore\", drop=\"if_binary\")\n\nencoder.fit(df_ohe[cat_cols].to_numpy())\nfeature_names = encoder.get_feature_names_out(cat_cols)\n\ntrain_enc = encoder.transform(df_ohe[cat_cols].to_numpy())\ntest_enc = encoder.transform(test_df_ohe[cat_cols].to_numpy())\n\ntrain_enc_df = pd.DataFrame(data=train_enc.toarray(), columns=feature_names)\ntest_enc_df = pd.DataFrame(data=test_enc.toarray(), columns=feature_names)\n                                         \ntrain_enc_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:26:53.257910Z","iopub.execute_input":"2022-11-30T04:26:53.258515Z","iopub.status.idle":"2022-11-30T04:27:00.531716Z","shell.execute_reply.started":"2022-11-30T04:26:53.258475Z","shell.execute_reply":"2022-11-30T04:27:00.530053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ohe = df_ohe.drop(cat_cols, axis=1)\ndf_ohe = pd.concat([df_ohe, train_enc_df], axis=1)\n\ntest_df_ohe = test_df_ohe.drop(cat_cols, axis=1)\ntest_df_ohe = pd.concat([test_df_ohe, test_enc_df], axis=1)\n\ndf_ohe.head()","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:27:00.533791Z","iopub.execute_input":"2022-11-30T04:27:00.534206Z","iopub.status.idle":"2022-11-30T04:27:02.323827Z","shell.execute_reply.started":"2022-11-30T04:27:00.534168Z","shell.execute_reply":"2022-11-30T04:27:02.322392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ohe.shape, test_df_ohe.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:27:02.325467Z","iopub.execute_input":"2022-11-30T04:27:02.326044Z","iopub.status.idle":"2022-11-30T04:27:02.333200Z","shell.execute_reply.started":"2022-11-30T04:27:02.326007Z","shell.execute_reply":"2022-11-30T04:27:02.331415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label Encoding\n\nWhile we can try to use `LabelEncoder`, one problem that we are bound to encounter is that there will be a mismatch between the categories present in the train and test sets. Thus, we will use the `OrdinalEncoder` instead, which comes with built-in support for handling this situation. Specificially, we can encode unseen categories in the test set with some value which is distinct from the values produced by the encoder.","metadata":{}},{"cell_type":"code","source":"encoder = preprocessing.OrdinalEncoder(handle_unknown=\"use_encoded_value\", unknown_value=-1)\nencoder.fit(df[cat_cols].to_numpy())\ndf[cat_cols] = encoder.transform(df[cat_cols].to_numpy())\ntest_df[cat_cols] = encoder.transform(test_df[cat_cols].to_numpy())","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:27:02.334866Z","iopub.execute_input":"2022-11-30T04:27:02.335254Z","iopub.status.idle":"2022-11-30T04:27:08.499003Z","shell.execute_reply.started":"2022-11-30T04:27:02.335214Z","shell.execute_reply":"2022-11-30T04:27:08.498197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving the Datasets","metadata":{}},{"cell_type":"code","source":"df_ohe.to_csv(\"./train_ohe_scaled_no_null.csv\", index=False)\ntest_df_ohe.to_csv(\"./test_ohe_scaled_no_null.csv\", index=False)\n\ndf.to_csv(\"./train_le_scaled_no_null.csv\", index=False)\ntest_df.to_csv(\"./test_le_scaled_no_null.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:27:08.500150Z","iopub.execute_input":"2022-11-30T04:27:08.501397Z","iopub.status.idle":"2022-11-30T04:28:27.154958Z","shell.execute_reply.started":"2022-11-30T04:27:08.501358Z","shell.execute_reply":"2022-11-30T04:28:27.153223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Categorical and Numerical Features\n\nWe will also save the lists of categorical and numerical features as CSV files for later reference.","metadata":{}},{"cell_type":"code","source":"cat_df = pd.DataFrame(data={\"column\": cat_cols})\ncat_df.to_csv(\"categorical_features.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:28:27.156573Z","iopub.execute_input":"2022-11-30T04:28:27.157587Z","iopub.status.idle":"2022-11-30T04:28:27.164376Z","shell.execute_reply.started":"2022-11-30T04:28:27.157547Z","shell.execute_reply":"2022-11-30T04:28:27.163419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_df = pd.DataFrame(data={\"column\": numerical_cols})\nnum_df.to_csv(\"numerical_features.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-30T04:28:27.165999Z","iopub.execute_input":"2022-11-30T04:28:27.166247Z","iopub.status.idle":"2022-11-30T04:28:27.177634Z","shell.execute_reply.started":"2022-11-30T04:28:27.166223Z","shell.execute_reply":"2022-11-30T04:28:27.176256Z"},"trusted":true},"execution_count":null,"outputs":[]}]}