{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.feature_selection import f_classif\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.decomposition import PCA\nfrom string import ascii_uppercase\nimport collections\nfrom sklearn.preprocessing import StandardScaler\nfrom catboost import CatBoostClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn import metrics\nfrom eli5.sklearn import PermutationImportance\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-17T17:04:07.955284Z","iopub.execute_input":"2022-12-17T17:04:07.956277Z","iopub.status.idle":"2022-12-17T17:04:17.417925Z","shell.execute_reply.started":"2022-12-17T17:04:07.956085Z","shell.execute_reply":"2022-12-17T17:04:17.416750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_point(x, y, val, ax):\n    a = pd.concat({'x': x, 'y': y, 'val': val}, axis=1)\n    for i, point in a.iterrows():\n        ax.text(point['x']+.02, point['y'], str(point['val']))\n\ndef WOE_based_IV_category(df,target,cont_var):\n    woe_df=pd.DataFrame()\n    bins_list=list()\n    event_list=list()\n    non_event_list=list()\n    for var in df[cont_var].unique():\n        even_count=np.nansum(df[df[cont_var]==var][target]>0)\n        non_even_count=np.nansum(df[df[cont_var]==var][target]<1)\n        event_list.append(even_count)\n        non_event_list.append(non_even_count)\n        bins_list.append(var)\n        \n    woe_df=pd.DataFrame({\n        \"bin\":bins_list,\n        \"No_events\":event_list,\n        \"No_nonevents\":non_event_list\n    })\n    woe_df[\"event_pct\"]=woe_df[\"No_events\"]/sum(woe_df[\"No_events\"])\n    woe_df[\"nonevent_pct\"]=woe_df[\"No_nonevents\"]/sum(woe_df[\"No_nonevents\"])\n    woe_df[\"WOE\"]=np.log(woe_df[\"event_pct\"]/woe_df[\"nonevent_pct\"])\n    woe_df[\"IV\"]=(woe_df[\"event_pct\"]-woe_df[\"nonevent_pct\"])*woe_df[\"WOE\"]\n    return woe_df\n        \n        \ndef WOE_based_IV(df,target,cont_var, limits):\n    woe_df=pd.DataFrame()\n    bins_list=list()\n    event_list=list()\n    non_event_list=list()\n    for i in range(1,len(limits)):\n        even_count=np.nansum(df[(limits[i-1]<df[cont_var])&(df[cont_var]<=limits[i])][target]>0)\n        non_even_count=np.nansum(df[(limits[i-1]<df[cont_var])&(df[cont_var]<=limits[i])][target]<1)\n        event_list.append(even_count)\n        non_event_list.append(non_even_count)\n        bins_list.append(\"lower: \"+str(limits[i-1])+\" - upper: \"+str(limits[i]))\n        \n    woe_df=pd.DataFrame({\n        \"bin\":bins_list,\n        \"No_events\":event_list,\n        \"No_nonevents\":non_event_list\n    })\n    woe_df[\"event_pct\"]=woe_df[\"No_events\"]/sum(woe_df[\"No_events\"])\n    woe_df[\"nonevent_pct\"]=woe_df[\"No_nonevents\"]/sum(woe_df[\"No_nonevents\"])\n    woe_df[\"WOE\"]=np.log(woe_df[\"event_pct\"]/woe_df[\"nonevent_pct\"])\n    woe_df[\"IV\"]=(woe_df[\"event_pct\"]-woe_df[\"nonevent_pct\"])*woe_df[\"WOE\"]\n    return woe_df\n        \ndef fit_model_using_classifier(alg,\n                               dtrain,\n                               predictors,\n                               target=\"target\",\n                               performCV=True, \n                               printFeatureImportance=True, \n                               cv_folds=3,\n                               repeat=5,\n                               scoring='roc_auc',\n                               only_top_x_feature=60\n                              ):\n    \"\"\"\n    I used the function found in this source\n    https://www.analyticsvidhya.com/blog/2016/02/complete-guide-parameter-tuning-gradient-boosting-gbm-python/\n    I modified the code slightly\n    \"\"\"\n\n    # Perform cross-validation:\n    cv_score=list()\n    if performCV:\n        for i in range(0,repeat):\n            cv_score_temp = cross_val_score(\n                            alg, \n                            dtrain[predictors], \n                            dtrain[target], \n                            cv=cv_folds, \n                            scoring=scoring)\n            cv_score=cv_score+list(cv_score_temp)\n    \n    # Fit the algorithm on the data\n    alg.fit(dtrain[predictors], dtrain[target])\n        \n    # Predict training set:\n    dtrain_predictions = alg.predict(dtrain[predictors])\n    dtrain_predprob = alg.predict_proba(dtrain[predictors])[:,1]        \n    \n    # Print model report:\n    print(\"\\nModel Report\")\n    print(\"Accuracy : \" + str(round(metrics.accuracy_score(\n        dtrain[target].values, dtrain_predictions),4)))\n    print(\"AUC Score (Train): \" + str(round(\n        metrics.roc_auc_score(dtrain[target], dtrain_predprob),4)))\n    \n    if performCV:\n        print(\"\\n Cross validation summary (\"+scoring+\")\")\n        print(\"Average: \"+str(round(np.mean(cv_score),4)))\n        print(\"Std    : \"+str(round(np.std(cv_score),4)))\n        print(\"Min    : \"+str(round(np.min(cv_score),4)))\n        print(\"Max    : \"+str(round(np.max(cv_score),4)))\n                \n    # Print Feature Importance:\n    if printFeatureImportance and \"feature_importances_\" in dir(alg):\n        plt.figure(figsize=(20,6))\n        feat_imp = pd.Series(alg.feature_importances_, predictors).sort_values(ascending=False)\n        feat_imp.head(only_top_x_feature).plot(kind='bar', title='Feature Importances',fontsize=12, color=\"#74B72E\")\n        plt.ylabel('Feature Importance Score')\n        return alg, feat_imp\n    else:\n        return alg, list()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:04:17.420579Z","iopub.execute_input":"2022-12-17T17:04:17.421754Z","iopub.status.idle":"2022-12-17T17:04:17.449545Z","shell.execute_reply.started":"2022-12-17T17:04:17.421706Z","shell.execute_reply":"2022-12-17T17:04:17.447712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 0. Load data and set the colors","metadata":{}},{"cell_type":"code","source":"PALETTE=\"Spectral\"\ncol_light=\"#AEF359\"\ncol_dark=\"#74B72E\"\nsns.color_palette(\"Spectral\", as_cmap=True)\n# model and selection params\nprot_goods_min=0.000001\nscoref=\"roc_auc\"\nrepeat_numb=3\nbest_feature_numb=100\niteration_numb=1\ntotal_trh=300\nselect_dummy=100","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:04:17.451114Z","iopub.execute_input":"2022-12-17T17:04:17.452017Z","iopub.status.idle":"2022-12-17T17:04:17.468730Z","shell.execute_reply.started":"2022-12-17T17:04:17.451982Z","shell.execute_reply":"2022-12-17T17:04:17.467680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission=pd.read_csv(\"/kaggle/input/emsi-5iir-ds/sample_submission.csv\")\ntrain=pd.read_csv(\"/kaggle/input/emsi-5iir-ds/train.csv\")\ntest=pd.read_csv(\"/kaggle/input/emsi-5iir-ds/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:04:17.472339Z","iopub.execute_input":"2022-12-17T17:04:17.473244Z","iopub.status.idle":"2022-12-17T17:04:34.218371Z","shell.execute_reply.started":"2022-12-17T17:04:17.473149Z","shell.execute_reply":"2022-12-17T17:04:34.217111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.describe())\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:04:34.219684Z","iopub.execute_input":"2022-12-17T17:04:34.220026Z","iopub.status.idle":"2022-12-17T17:04:35.453800Z","shell.execute_reply.started":"2022-12-17T17:04:34.219998Z","shell.execute_reply":"2022-12-17T17:04:35.452939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train.info())\n# no missing values in train\ndisplay(test.info())\n# no missing values in test","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:04:35.455050Z","iopub.execute_input":"2022-12-17T17:04:35.456044Z","iopub.status.idle":"2022-12-17T17:04:35.647040Z","shell.execute_reply.started":"2022-12-17T17:04:35.456011Z","shell.execute_reply":"2022-12-17T17:04:35.645959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate column names\ncol_base=list(train.columns[1:-1])\nprint(\"Number of original features: \"+str(len(col_base)))\ncol_base_numeric=list(set(col_base)-set([\"f_27\"]))\ncol_base_numeric.sort()\ncol_string=[\"f_27\"]\ncol_category=[\"f_07\",\"f_08\",\"f_09\",\"f_10\",\"f_11\",\"f_12\",\"f_13\",\"f_14\",\"f_15\",\"f_16\",\"f_17\",\"f_18\",\"f_29\",\"f_30\"] # category ~= only a few possible values\ncol_continuous=list(set(col_base)-set(col_string)-set(col_category))\ncol_continuous.sort()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:04:35.648543Z","iopub.execute_input":"2022-12-17T17:04:35.649003Z","iopub.status.idle":"2022-12-17T17:04:35.656217Z","shell.execute_reply.started":"2022-12-17T17:04:35.648971Z","shell.execute_reply":"2022-12-17T17:04:35.655061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. EDA","metadata":{}},{"cell_type":"code","source":"# Limited info on the distribution nothing really special at this point\nfor i in range(0,6):\n    plt.figure(figsize=(20,6))\n    index=0\n    for col in col_base_numeric[i*5:min(31,(5*i+5))]:\n        plt.subplot(1,5,index+1)\n        plt.hist(train[col],\n                 bins=100,\n                 density=False,\n                 color=col_dark);\n        plt.title(col, fontsize=10);\n        index+=1\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:04:35.657746Z","iopub.execute_input":"2022-12-17T17:04:35.658200Z","iopub.status.idle":"2022-12-17T17:04:45.111443Z","shell.execute_reply.started":"2022-12-17T17:04:35.658157Z","shell.execute_reply":"2022-12-17T17:04:45.110213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0,6):\n    plt.figure(figsize=(20,6))\n    index=0\n    for col in col_base_numeric[i*5:min(30,(5*i+5))]:\n        plt.subplot(1,5,index+1)\n        plt.hist(train[train[\"target\"]<1][col],\n                 bins=100,\n                 density=False,\n                 color=col_dark);\n        plt.hist(train[train[\"target\"]>0][col],\n                 bins=100,\n                 density=False,\n                 color=col_light);\n        plt.legend([\"target = 0\", \"target = 1\"], fontsize=6)\n        plt.title(col, fontsize=10);\n        index+=1\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:04:45.113379Z","iopub.execute_input":"2022-12-17T17:04:45.113867Z","iopub.status.idle":"2022-12-17T17:05:07.154711Z","shell.execute_reply.started":"2022-12-17T17:04:45.113819Z","shell.execute_reply":"2022-12-17T17:05:07.153697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18, 14))\nsns.heatmap(train[col_continuous + ['target']].corr(), center=0, annot=True, fmt='.2f',cmap=PALETTE)\nplt.title('Continuous variable correlation with target', fontsize=20);\nplt.show()\n# Note: calculating correlations with a binary target is not very accurate, but mathematically feasible...","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:07.158187Z","iopub.execute_input":"2022-12-17T17:05:07.158539Z","iopub.status.idle":"2022-12-17T17:05:09.411196Z","shell.execute_reply.started":"2022-12-17T17:05:07.158510Z","shell.execute_reply":"2022-12-17T17:05:09.409982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature 27 must be treated separately","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:09.413191Z","iopub.execute_input":"2022-12-17T17:05:09.414013Z","iopub.status.idle":"2022-12-17T17:05:09.418947Z","shell.execute_reply.started":"2022-12-17T17:05:09.413963Z","shell.execute_reply":"2022-12-17T17:05:09.417830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"characters_present=list()\nfor c in ascii_uppercase:\n    train_val=train[\"f_27\"].str.count(c)\n    if sum(train_val)>0:\n        train[c]=train_val\n        test[c]=test[\"f_27\"].str.count(c)\n        characters_present.append(c)\ntrain['dist_char']=train[\"f_27\"].apply(set).apply(len)\ntrain['most_common_numb']=train[\"f_27\"].apply(lambda s:collections.Counter(s).most_common(1)[0][1])\n\ntest['dist_char']=test[\"f_27\"].apply(set).apply(len)\ntest['most_common_numb']=test[\"f_27\"].apply(lambda s:collections.Counter(s).most_common(1)[0][1])\nfrom_character_var=characters_present+['dist_char','most_common_numb']","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:09.420755Z","iopub.execute_input":"2022-12-17T17:05:09.421150Z","iopub.status.idle":"2022-12-17T17:05:51.404024Z","shell.execute_reply.started":"2022-12-17T17:05:09.421102Z","shell.execute_reply":"2022-12-17T17:05:51.402729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woe_table=pd.DataFrame()\niv_table=pd.DataFrame()\nfor col in col_category+from_character_var:\n    temp_df=train[[col,\"target\"]].groupby(col).agg({\"target\":[sum,len]}).reset_index()\n    temp_df.columns=[\"value\",\"goods\",\"total\"]\n    temp_df[\"bads\"]=temp_df[\"total\"]-temp_df[\"goods\"]\n    temp_df[\"dist_bads\"]=temp_df[\"bads\"]/sum(temp_df[\"bads\"])\n    temp_df[\"dist_goods\"]=temp_df[\"goods\"]/sum(temp_df[\"goods\"])\n    temp_df[\"feature\"]=col\n    temp_df[\"dist_goods\"]=np.where(temp_df[\"dist_goods\"]<prot_goods_min,prot_goods_min,temp_df[\"dist_goods\"])\n    temp_df[\"dist_bads\"]=np.where(temp_df[\"dist_bads\"]<prot_goods_min,prot_goods_min,temp_df[\"dist_bads\"])\n    temp_df[\"WOE\"]=np.log(temp_df[\"dist_goods\"]/(temp_df[\"dist_bads\"]))*100\n    temp_df[\"IV_component\"]=(temp_df[\"dist_goods\"]-temp_df[\"dist_bads\"])*temp_df[\"WOE\"]\n    woe_table=woe_table.append(temp_df)\n    iv_table=iv_table.append(pd.DataFrame({\n        \"feature\":[col],\n        \"IV\": [sum(temp_df[\"IV_component\"])]\n    }))\n\nwoe_table.sort_values(by=[\"WOE\"], inplace=True)\n# we also generate a WOE lavel\nwoe_table[\"feature_with_value\"]=woe_table[\"feature\"]+\"_value_\"+woe_table[\"value\"].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:51.405647Z","iopub.execute_input":"2022-12-17T17:05:51.406344Z","iopub.status.idle":"2022-12-17T17:05:54.022289Z","shell.execute_reply.started":"2022-12-17T17:05:51.406299Z","shell.execute_reply":"2022-12-17T17:05:54.021129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can plot the highlights here\nplt.figure(figsize=(20,6))\nax = sns.barplot(x=\"feature\", y=\"IV\", data=iv_table, palette=PALETTE)\nplt.title('Comparison of categorical features by their information values', fontsize=20);\nplt.ylabel('Information Value for categorical feature', fontsize=14);\nplt.xlabel('Feature name', fontsize=14);\nplt.xticks(rotation = 90);","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:54.023661Z","iopub.execute_input":"2022-12-17T17:05:54.024005Z","iopub.status.idle":"2022-12-17T17:05:54.572728Z","shell.execute_reply.started":"2022-12-17T17:05:54.023975Z","shell.execute_reply":"2022-12-17T17:05:54.571411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can plot the highlights here\nplt.figure(figsize=(20,6))\nax = sns.barplot(x=\"feature\", y=\"IV\", data=iv_table[iv_table[\"feature\"].isin(col_category)], palette=PALETTE)\nplt.title('Comparison of categorical features by their information values (excluding f_27 related features)', fontsize=20);\nplt.ylabel('Information Value for categorical feature', fontsize=14);\nplt.xlabel('Feature name', fontsize=14);\nplt.xticks(rotation = 90);","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:54.574150Z","iopub.execute_input":"2022-12-17T17:05:54.574997Z","iopub.status.idle":"2022-12-17T17:05:54.920472Z","shell.execute_reply.started":"2022-12-17T17:05:54.574963Z","shell.execute_reply":"2022-12-17T17:05:54.919220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nax = sns.barplot(x=\"feature_with_value\", y=\"WOE\", data=woe_table[woe_table[\"total\"]>50].head(30), palette=PALETTE)\nplt.title('Comparison of categorical feature values where WOE is small (higher prob. of target=0)', fontsize=20);\nplt.ylabel('WOE', fontsize=14);\nplt.xlabel('Feature name with value', fontsize=14);\nplt.xticks(rotation = 90);","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:54.921751Z","iopub.execute_input":"2022-12-17T17:05:54.922080Z","iopub.status.idle":"2022-12-17T17:05:55.448517Z","shell.execute_reply.started":"2022-12-17T17:05:54.922051Z","shell.execute_reply":"2022-12-17T17:05:55.447150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nax = sns.barplot(x=\"feature_with_value\", y=\"WOE\", data=woe_table[woe_table[\"total\"]>50].tail(30), palette=PALETTE)\nplt.title('Comparison of categorical feature values where WOE is large (higher prob. of target=1)', fontsize=20);\nplt.ylabel('WOE', fontsize=14);\nplt.xlabel('Feature name with value', fontsize=14);\nplt.xticks(rotation = 90);","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:55.450362Z","iopub.execute_input":"2022-12-17T17:05:55.450882Z","iopub.status.idle":"2022-12-17T17:05:55.974803Z","shell.execute_reply.started":"2022-12-17T17:05:55.450828Z","shell.execute_reply":"2022-12-17T17:05:55.973409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_calc=pd.DataFrame()\nfor i in range(0,10):\n    train[\"pos_candidate\"]=train[\"f_27\"].str.slice(i,i+1)\n    woe_res=WOE_based_IV_category(df=train,target=\"target\",cont_var=\"pos_candidate\")\n    woe_res[\"position\"]=i\n    all_calc=all_calc.append(woe_res)\nall_calc[\"WOE\"]=abs(all_calc[\"WOE\"])\nall_calc.sort_values(by=[\"WOE\"],inplace=True)\nall_calc[\"total\"]=all_calc[\"No_events\"]+all_calc[\"No_nonevents\"]","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:05:55.976532Z","iopub.execute_input":"2022-12-17T17:05:55.977007Z","iopub.status.idle":"2022-12-17T17:06:20.574736Z","shell.execute_reply.started":"2022-12-17T17:05:55.976962Z","shell.execute_reply":"2022-12-17T17:06:20.573600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We can generate dummies using these variables\n# The WOE is decent in some case, so might have information value\nselected_indicators=all_calc[all_calc[\"total\"]>(total_trh*2/3)].tail(60).copy()\ndisplay(selected_indicators)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:06:20.576239Z","iopub.execute_input":"2022-12-17T17:06:20.576929Z","iopub.status.idle":"2022-12-17T17:06:20.609561Z","shell.execute_reply.started":"2022-12-17T17:06:20.576885Z","shell.execute_reply":"2022-12-17T17:06:20.608312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"position_chars=list()\nfor char, position in zip(list(selected_indicators[\"bin\"]),list(selected_indicators[\"position\"])):\n    train[char+\"_pos_\"+str(position)]=np.where(train[\"f_27\"].str.slice(position,position+1)==char,1,0)\n    test[char+\"_pos_\"+str(position)]=np.where(test[\"f_27\"].str.slice(position,position+1)==char,1,0)\n    position_chars.append(char+\"_pos_\"+str(position))","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:06:20.611102Z","iopub.execute_input":"2022-12-17T17:06:20.611433Z","iopub.status.idle":"2022-12-17T17:06:47.247513Z","shell.execute_reply.started":"2022-12-17T17:06:20.611403Z","shell.execute_reply":"2022-12-17T17:06:47.246257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Feature generation","metadata":{}},{"cell_type":"code","source":"# Now we need an algorithm to create new categorical variables\n# WE create dummy variables for the top x feature - value pairs based on WOE\nwoe_table[\"WOE_abs\"]=abs(woe_table[\"WOE\"])\nwoe_table.sort_values(by=[\"WOE_abs\"], inplace=True)\nselect_dummies=woe_table[woe_table[\"total\"]>total_trh].tail(select_dummy)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:06:47.249035Z","iopub.execute_input":"2022-12-17T17:06:47.249392Z","iopub.status.idle":"2022-12-17T17:06:47.259127Z","shell.execute_reply.started":"2022-12-17T17:06:47.249358Z","shell.execute_reply":"2022-12-17T17:06:47.257546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dummy_names=list()\nfor v,feature,dummy_name in zip(list(select_dummies[\"value\"]),\n                                list(select_dummies[\"feature\"]),\n                                list(select_dummies[\"feature_with_value\"])):\n    dummy_names.append(dummy_name)\n    train[dummy_name]=np.where(train[feature]==int(v),1,0)\n    test[dummy_name]=np.where(test[feature]==int(v),1,0)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:06:47.260881Z","iopub.execute_input":"2022-12-17T17:06:47.261253Z","iopub.status.idle":"2022-12-17T17:06:48.732115Z","shell.execute_reply.started":"2022-12-17T17:06:47.261205Z","shell.execute_reply":"2022-12-17T17:06:48.730965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Here I attemp to generate new features, I consider:\n# 1) multiplicative relationship\n# 2) a pairwise additive relationship based on some common factor (I deploy a PCA here)\n\ntrain_add=train[[\"id\",\"target\"]+col_continuous].copy()\ntest_add=test[[\"id\"]+col_continuous].copy()\ncol_continuous_corr=col_continuous.copy()\nfor k in range(0,iteration_numb):\n    print(\"iteration (mults) : \"+str(k))\n    additional_cols=list()\n    for i in range(0,len(col_continuous_corr)):\n        for j in range(min(i+1,len(col_continuous_corr)),len(col_continuous_corr)):\n            f_i, _ =f_classif(np.array(train_add[[col_continuous_corr[i]]]).reshape(-1,1),np.array(train_add[\"target\"]))\n            f_j, _ =f_classif(np.array(train_add[[col_continuous_corr[j]]]).reshape(-1,1),np.array(train_add[\"target\"]))\n            f_cand, _ =f_classif(np.array(train_add[col_continuous_corr[i]]*train_add[col_continuous_corr[j]]).reshape(-1,1),np.array(train_add[\"target\"]))\n            if f_cand>(1.03*max(f_i,f_j)):\n                train_add[col_continuous_corr[i]+\"_\"+col_continuous_corr[j]+\"_m\"]=train_add[col_continuous_corr[i]]*train_add[col_continuous_corr[j]]\n                additional_cols.append(col_continuous_corr[i]+\"_\"+col_continuous_corr[j]+\"_m\")\n                test_add[col_continuous_corr[i]+\"_\"+col_continuous_corr[j]+\"_m\"]=test_add[col_continuous_corr[i]]*test_add[col_continuous_corr[j]]\n    col_continuous_corr=list(set(col_continuous_corr).union(set(additional_cols)))\n    \n    pca = PCA(n_components=2)\n    additional_cols=list()\n    print(\"iteration (PCA) : \"+str(k))\n    if k==0:\n        for i in range(0,len(col_continuous_corr)):\n            for j in range(min(i+1,len(col_continuous_corr)),len(col_continuous_corr)):\n                pca_comps = pca.fit_transform(train_add[[col_continuous_corr[i], col_continuous_corr[j]]])\n                np.array(np.matmul(train_add[[col_continuous_corr[i], col_continuous_corr[j]]],pca.components_[0]))\n                f_i, _ =f_classif(np.array(train_add[[col_continuous_corr[i]]]).reshape(-1,1),np.array(train_add[\"target\"]))\n                f_j, _ =f_classif(np.array(train_add[[col_continuous_corr[j]]]).reshape(-1,1),np.array(train_add[\"target\"]))\n                f_cand, _ =f_classif(np.array(np.matmul(train_add[[col_continuous_corr[i], col_continuous_corr[j]]],pca.components_[0])).reshape(-1,1),np.array(train_add[\"target\"]))\n                if f_cand>(1.03*max(f_i,f_j)):\n                    train_add[col_continuous_corr[i]+\"_\"+col_continuous_corr[j]+\"_pca\"]=np.matmul(train_add[[col_continuous_corr[i], col_continuous_corr[j]]],pca.components_[0])\n                    additional_cols.append(col_continuous_corr[i]+\"_\"+col_continuous_corr[j]+\"_pca\")\n                    test_add[col_continuous_corr[i]+\"_\"+col_continuous_corr[j]+\"_pca\"]=np.matmul(test_add[[col_continuous_corr[i], col_continuous_corr[j]]],pca.components_[0])\n        col_continuous_corr=list(set(col_continuous_corr).union(set(additional_cols)))","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:06:48.733969Z","iopub.execute_input":"2022-12-17T17:06:48.734492Z","iopub.status.idle":"2022-12-17T17:09:22.722943Z","shell.execute_reply.started":"2022-12-17T17:06:48.734446Z","shell.execute_reply":"2022-12-17T17:09:22.721209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I add a constant temporarily, so we can estimate logits with one variable\ntrain_add[\"zero\"]=1","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:09:22.725707Z","iopub.execute_input":"2022-12-17T17:09:22.726389Z","iopub.status.idle":"2022-12-17T17:09:22.736506Z","shell.execute_reply.started":"2022-12-17T17:09:22.726328Z","shell.execute_reply":"2022-12-17T17:09:22.734805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We assess the predictive power of the new variables\nanova_result=pd.DataFrame()\nfor col in col_continuous_corr:\n    fval, _=f_classif(np.array(train_add[col]).reshape(-1,1),np.array(train_add[\"target\"]))\n    clf=LogisticRegression(random_state=0).fit(train_add[[\"zero\",col]],np.array(train_add[\"target\"]))\n    accuracy=clf.score(train_add[[\"zero\",col]],np.array(train_add[\"target\"]))\n\n    anova_result=anova_result.append(\n                    pd.DataFrame({\"feature\":[col],\n                                  \"anova\": [fval[0]],\n                                  \"accuracy\": [accuracy],\n                                  \"coefficient\": [clf.coef_[0][1]]\n                                 })\n                )\nanova_result.sort_values(by=[\"anova\"],ascending=False,inplace=True)\nprint(len(anova_result))\nanova_result=anova_result.drop_duplicates(subset=[\"anova\"]) # I'm not proud of this, but some metrics are symmetrical so we drop\n# We can see how many variables did we drop...\nprint(len(anova_result))","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:09:22.739589Z","iopub.execute_input":"2022-12-17T17:09:22.741314Z","iopub.status.idle":"2022-12-17T17:10:44.804406Z","shell.execute_reply.started":"2022-12-17T17:09:22.741247Z","shell.execute_reply":"2022-12-17T17:10:44.803193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,10))\nsns.scatterplot(data=anova_result, x=\"anova\", y=\"accuracy\", color=col_dark)\nplt.title('Comparison of numeric features', fontsize=20);\nplt.xlabel('Anova F-statistic for feature', fontsize=14);\nplt.ylabel('Naive Logistic regression accuracy with single feature', fontsize=14);\nplt.show()\n\n# only top features\nanova_result_top=anova_result.head(10).copy()\nplt.figure(figsize=(20,10))\nsns.scatterplot(data=anova_result_top, x=\"anova\", y=\"accuracy\", color=col_dark)\nplt.title('Comparison of numeric features (only top 10)', fontsize=20);\nplt.xlabel('Anova F-statistic for feature', fontsize=14);\nplt.ylabel('Naive Logistic regression accuracy with single feature', fontsize=14);\nlabel_point(anova_result_top[\"anova\"], anova_result_top[\"accuracy\"], anova_result_top[\"feature\"], plt.gca())  \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:10:44.805970Z","iopub.execute_input":"2022-12-17T17:10:44.807132Z","iopub.status.idle":"2022-12-17T17:10:45.320222Z","shell.execute_reply.started":"2022-12-17T17:10:44.807085Z","shell.execute_reply":"2022-12-17T17:10:45.319041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(18, 14))\nsns.heatmap(train_add[list(anova_result_top[\"feature\"])+ col_continuous + ['target']].corr(), center=0, annot=True, fmt='.2f',cmap=PALETTE)\nplt.title('Continuous vayeriable correlation with target (with top features)', fontsize=20);\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:10:45.321652Z","iopub.execute_input":"2022-12-17T17:10:45.322042Z","iopub.status.idle":"2022-12-17T17:10:50.346009Z","shell.execute_reply.started":"2022-12-17T17:10:45.322009Z","shell.execute_reply":"2022-12-17T17:10:50.345107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:10:50.352268Z","iopub.execute_input":"2022-12-17T17:10:50.353176Z","iopub.status.idle":"2022-12-17T17:10:50.635701Z","shell.execute_reply.started":"2022-12-17T17:10:50.353139Z","shell.execute_reply":"2022-12-17T17:10:50.634449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Based on AMbrusM and WTI 200 I create some combined features manually and try to apply a similar WOE methodology to what I've used earlier, \n# so I can create bounds.\n# https://www.kaggle.com/competitions/tabular-playground-series-may-2022/discussion/323892\n# https://www.kaggle.com/competitions/tabular-playground-series-may-2022/discussion/323766    \n# Qute from Ambrus A\n# the projection to f_02 and f_21\n# the projection to f_05 and f_22\n# the projection to f_00+f_01 and f_26","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:10:50.639026Z","iopub.execute_input":"2022-12-17T17:10:50.640325Z","iopub.status.idle":"2022-12-17T17:10:50.652312Z","shell.execute_reply.started":"2022-12-17T17:10:50.640283Z","shell.execute_reply":"2022-12-17T17:10:50.651317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"f_00_f_01_p\"]=train[\"f_00\"]+train[\"f_01\"]","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:10:50.653689Z","iopub.execute_input":"2022-12-17T17:10:50.655010Z","iopub.status.idle":"2022-12-17T17:10:50.668001Z","shell.execute_reply.started":"2022-12-17T17:10:50.654959Z","shell.execute_reply":"2022-12-17T17:10:50.666745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,7))\nplt.ylim(-10, 10)\nplt.xlim(-10, 10)\nsns.scatterplot(data=train, x=\"f_02\", y=\"f_21\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_02 and f_21\");\nplt.show()\nplt.figure(figsize=(10,7))\nsns.scatterplot(data=train, x=\"f_05\", y=\"f_22\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_05 and f_22\");\nplt.ylim(-10, 10)\nplt.xlim(-10, 10)\nplt.show()\nplt.figure(figsize=(10,7))\nsns.scatterplot(data=train, x=\"f_00_f_01_p\", y=\"f_26\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_00_f_01_p and f_26\");\nplt.ylim(-10, 10)\nplt.xlim(-10, 10)\nplt.show()\n# We can see certain line appearing, which indicates that based on the weighted sum of the feature values, we have different regions...\n# E.g. f_02 and f_21: if the two features sum is smaller than aroun -5 we have a different target prob...\n# We can try to determine the slope (f_02+y*f_21) or we can assume y (the slope) is simply 1... which is more or less inline with our observations\n# Right now it seems the slope is indeed 1, x+y<k value can be used here.","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:10:50.669501Z","iopub.execute_input":"2022-12-17T17:10:50.669838Z","iopub.status.idle":"2022-12-17T17:11:35.698360Z","shell.execute_reply.started":"2022-12-17T17:10:50.669807Z","shell.execute_reply":"2022-12-17T17:11:35.697045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"f_21_f_02_p\"]=1.0*train[\"f_21\"]+1.0*train[\"f_02\"]\ntrain[\"f_22_f_05_p\"]=1.0*train[\"f_22\"]+1.0*train[\"f_05\"]\ntrain[\"f_00_f_01_p_f26_p\"]=1.0*train[\"f_00_f_01_p\"]+1.0*train[\"f_26\"]\n\n# I try to determine the optimal cuts...\n# I did this manually based on the charts and some WOE calcs...\n# f_21_f_02_p : -5.3 and 5.2 when x+y\n# f_22_f_05_p : -5.4 and 5.1 when x+y\n# f_00_f_01_p_f26_p: -5.0 and 5.0 when x+y\n\n# I try to determine the first optimal cut...\noptimal_cut_l=pd.DataFrame()\nfor i in np.arange(-12,-3,0.1):\n    #print(i)\n    woe_table=WOE_based_IV(train,target=\"target\",cont_var=\"f_22_f_05_p\", limits=[-100,i,5.2,100])\n    optimal_cut_l=optimal_cut_l.append(pd.DataFrame({\n        \"value\":[i],\n        \"WOE\":[woe_table.head(1)[\"WOE\"][0]]\n    }))\n\n\noptimal_cut_u=pd.DataFrame()\nfor i in np.arange(3,12,0.1):\n    #print(i)\n    woe_table=WOE_based_IV(train,target=\"target\",cont_var=\"f_22_f_05_p\", limits=[-100,-5.3,i,100])\n    optimal_cut_u=optimal_cut_u.append(pd.DataFrame({\n        \"value\":[i],\n        \"WOE\":[woe_table.tail(1)[\"WOE\"][2]]\n    }))","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:11:35.700146Z","iopub.execute_input":"2022-12-17T17:11:35.701243Z","iopub.status.idle":"2022-12-17T17:17:33.370369Z","shell.execute_reply.started":"2022-12-17T17:11:35.701204Z","shell.execute_reply":"2022-12-17T17:17:33.369158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14,7))\nplt.plot(optimal_cut_u[\"value\"],optimal_cut_u[\"WOE\"],color=col_dark,label=\"upper cut value\")\nplt.plot(abs(optimal_cut_l[\"value\"]),optimal_cut_l[\"WOE\"],color=col_light,label=\"lower cut value\")\nplt.legend()\nplt.xlabel(\"Cut value\")\nplt.ylabel(\"WOE score\")\nplt.title(\"WOE and cut value choice (for lower cut value the abs(cut value) is presented, f_22_f_05_p)\");","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:17:33.371684Z","iopub.execute_input":"2022-12-17T17:17:33.372041Z","iopub.status.idle":"2022-12-17T17:17:33.598973Z","shell.execute_reply.started":"2022-12-17T17:17:33.372009Z","shell.execute_reply":"2022-12-17T17:17:33.597826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"woe_table=WOE_based_IV(train,target=\"target\",cont_var=\"f_00_f_01_p_f26_p\", limits=[-100,-5.0,5.0,100])\ndisplay(woe_table)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:17:33.600707Z","iopub.execute_input":"2022-12-17T17:17:33.601398Z","iopub.status.idle":"2022-12-17T17:17:35.702143Z","shell.execute_reply.started":"2022-12-17T17:17:33.601351Z","shell.execute_reply":"2022-12-17T17:17:35.701072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let us extend the feature list\ncont_features_interact=[\"f_00_f_01_p_f26_p\",\"f_22_f_05_p\",\"f_21_f_02_p\"]\ntest[\"f_00_f_01_p\"]=test[\"f_00\"]+test[\"f_01\"]\ntest[\"f_21_f_02_p\"]=test[\"f_21\"]+test[\"f_02\"]\ntest[\"f_22_f_05_p\"]=test[\"f_22\"]+test[\"f_05\"]\ntest[\"f_00_f_01_p_f26_p\"]=test[\"f_00_f_01_p\"]+test[\"f_26\"]\n\n# f_21_f_02_p : -5.3 and 5.2 when x+y\n# f_22_f_05_p : -5.4 and 5.1 when x+y\n# f_00_f_01_p_f26_p: -5.0 and 5.0 when x+y\n\ntrain[\"f_21_f_02_p_UPPER\"]=np.where(train[\"f_21_f_02_p\"]>=5.2,1,0)\ntest[\"f_21_f_02_p_UPPER\"]=np.where(test[\"f_21_f_02_p\"]>=5.2,1,0)\ntrain[\"f_21_f_02_p_LOWER\"]=np.where(train[\"f_21_f_02_p\"]<=-5.3,1,0)\ntest[\"f_21_f_02_p_LOWER\"]=np.where(test[\"f_21_f_02_p\"]<=-5.3,1,0)\n\ntrain[\"f_22_f_05_p_UPPER\"]=np.where(train[\"f_22_f_05_p\"]>=5.1,1,0)\ntest[\"f_22_f_05_p_UPPER\"]=np.where(test[\"f_22_f_05_p\"]>=5.1,1,0)\ntrain[\"f_22_f_05_p_LOWER\"]=np.where(train[\"f_22_f_05_p\"]<=-5.4,1,0)\ntest[\"f_22_f_05_p_LOWER\"]=np.where(test[\"f_22_f_05_p\"]<=-5.4,1,0)\n\ntrain[\"f_00_f_01_p_f26_p_UPPER\"]=np.where(train[\"f_00_f_01_p_f26_p\"]>=5.0,1,0)\ntest[\"f_00_f_01_p_f26_p_UPPER\"]=np.where(test[\"f_00_f_01_p_f26_p\"]>=5.0,1,0)\ntrain[\"f_00_f_01_p_f26_p_LOWER\"]=np.where(train[\"f_00_f_01_p_f26_p\"]<=-5.0,1,0)\ntest[\"f_00_f_01_p_f26_p_LOWER\"]=np.where(test[\"f_00_f_01_p_f26_p\"]<=-5.0,1,0)\n\ncont_features_interact=cont_features_interact+[\n    \"f_21_f_02_p_UPPER\", \"f_21_f_02_p_LOWER\",\n    \"f_22_f_05_p_UPPER\", \"f_22_f_05_p_LOWER\",\n    \"f_00_f_01_p_f26_p_UPPER\", \"f_00_f_01_p_f26_p_LOWER\"\n]","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:17:35.703498Z","iopub.execute_input":"2022-12-17T17:17:35.703938Z","iopub.status.idle":"2022-12-17T17:17:35.781141Z","shell.execute_reply.started":"2022-12-17T17:17:35.703905Z","shell.execute_reply":"2022-12-17T17:17:35.780044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explore othger variables\n# We were able to reconcile AmbrusM's estimates by simple WOE, we can extend the heuristic to look for similar pairs\noptimal_cut_all=pd.DataFrame()\nfor i in range(0,len(col_continuous)):\n    print(col_continuous[i])\n    for j in range(i,len(col_continuous)):\n        train[\"candidate\"]=train[col_continuous[i]]+train[col_continuous[j]]\n        woe_table=WOE_based_IV(train,target=\"target\",cont_var=\"candidate\", limits=[-100,-5.1,5.1,100])\n        optimal_cut_all=optimal_cut_all.append(pd.DataFrame({\n            \"variables\":[col_continuous[i]+\"__\"+col_continuous[j]],\n            \"WOE_lower\":[woe_table.head(1)[\"WOE\"][0]],\n            \"WOE_upper\":[woe_table.tail(1)[\"WOE\"][2]]\n        }))","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:17:35.782697Z","iopub.execute_input":"2022-12-17T17:17:35.783360Z","iopub.status.idle":"2022-12-17T17:22:02.010858Z","shell.execute_reply.started":"2022-12-17T17:17:35.783326Z","shell.execute_reply":"2022-12-17T17:22:02.009735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The pairs identified earlier are here...\noptimal_cut_all[(abs(optimal_cut_all[\"WOE_lower\"])>1)|(abs(optimal_cut_all[\"WOE_upper\"])>1)]","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:22:02.012340Z","iopub.execute_input":"2022-12-17T17:22:02.012911Z","iopub.status.idle":"2022-12-17T17:22:02.029392Z","shell.execute_reply.started":"2022-12-17T17:22:02.012865Z","shell.execute_reply":"2022-12-17T17:22:02.027866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We see patterns with f_21, but this seems to only impact f_21...\nsns.scatterplot(data=train, x=\"f_01\", y=\"f_21\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_01 and f_21\");\nplt.show()\n\nsns.scatterplot(data=train, x=\"f_00\", y=\"f_21\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_00 and f_21\");\nplt.show()\n\nsns.scatterplot(data=train, x=\"f_05\", y=\"f_21\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_05 and f_21\");\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:22:02.030637Z","iopub.execute_input":"2022-12-17T17:22:02.031267Z","iopub.status.idle":"2022-12-17T17:22:47.192873Z","shell.execute_reply.started":"2022-12-17T17:22:02.031232Z","shell.execute_reply":"2022-12-17T17:22:47.191782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# However there are patterns with other features\nsns.scatterplot(data=train, x=\"f_22\", y=\"f_24\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_22 and f_24\");\nplt.show()\n\nsns.scatterplot(data=train, x=\"f_02\", y=\"f_26\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_02 and f_26\");\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:22:47.194089Z","iopub.execute_input":"2022-12-17T17:22:47.194438Z","iopub.status.idle":"2022-12-17T17:23:16.986526Z","shell.execute_reply.started":"2022-12-17T17:22:47.194408Z","shell.execute_reply":"2022-12-17T17:23:16.985313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimal_cut_substr=pd.DataFrame()\nfor i in range(0,len(col_continuous)):\n    print(col_continuous[i])\n    for j in range(i,len(col_continuous)):\n        train[\"candidate\"]=train[col_continuous[i]]-train[col_continuous[j]]\n        woe_table=WOE_based_IV(train,target=\"target\",cont_var=\"candidate\", limits=[-100,-5.1,5.1,100])\n        optimal_cut_substr=optimal_cut_substr.append(pd.DataFrame({\n            \"variables\":[col_continuous[i]+\"__\"+col_continuous[j]],\n            \"WOE_lower\":[woe_table.head(1)[\"WOE\"][0]],\n            \"WOE_upper\":[woe_table.tail(1)[\"WOE\"][2]]\n        }))","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:23:16.988097Z","iopub.execute_input":"2022-12-17T17:23:16.988486Z","iopub.status.idle":"2022-12-17T17:27:41.177190Z","shell.execute_reply.started":"2022-12-17T17:23:16.988446Z","shell.execute_reply":"2022-12-17T17:27:41.176059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimal_cut_substr[(abs(optimal_cut_substr[\"WOE_lower\"])>0.8)|(abs(optimal_cut_substr[\"WOE_upper\"])>0.8)]","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:27:41.178877Z","iopub.execute_input":"2022-12-17T17:27:41.179432Z","iopub.status.idle":"2022-12-17T17:27:41.194162Z","shell.execute_reply.started":"2022-12-17T17:27:41.179380Z","shell.execute_reply":"2022-12-17T17:27:41.193035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.scatterplot(data=train, x=\"f_03\", y=\"f_21\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_03 and f_21\");\nplt.show()\n\nsns.scatterplot(data=train, x=\"f_19\", y=\"f_21\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_19 and f_21\");\nplt.show()\n\ntrain[\"f_03_f_19_cand\"]=train[\"f_03\"]+train[\"f_19\"]\n\nsns.scatterplot(data=train, x=\"f_03_f_19_cand\", y=\"f_21\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_03_f_19_cand and f_21\");\nplt.show()\n\nsns.scatterplot(data=train, x=\"f_03\", y=\"f_19\",s=1, hue=\"target\",palette=PALETTE);\nplt.title(\"f_03 and f_19\");\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:27:41.195988Z","iopub.execute_input":"2022-12-17T17:27:41.196413Z","iopub.status.idle":"2022-12-17T17:28:48.178464Z","shell.execute_reply.started":"2022-12-17T17:27:41.196371Z","shell.execute_reply":"2022-12-17T17:28:48.177299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Conclusion there seems to be only weaker relationships here, so no need to add new dummy variables...","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:28:48.180012Z","iopub.execute_input":"2022-12-17T17:28:48.180412Z","iopub.status.idle":"2022-12-17T17:28:48.185515Z","shell.execute_reply.started":"2022-12-17T17:28:48.180376Z","shell.execute_reply":"2022-12-17T17:28:48.184313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Assemble train and test dataset","metadata":{}},{"cell_type":"code","source":"train_final=train_add.drop(columns=[\"zero\"]).merge(train[[\"id\"]+dummy_names+position_chars+col_category+from_character_var+cont_features_interact],how=\"left\",on=\"id\")\ndel train\ndel train_add\ntest_final=test_add.merge(test[[\"id\"]+dummy_names+position_chars+col_category+from_character_var+cont_features_interact],how=\"left\",on=\"id\")\nfeature_columns=list(train_final.columns[2:])\ndel test\ndel test_add","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:28:48.187090Z","iopub.execute_input":"2022-12-17T17:28:48.187522Z","iopub.status.idle":"2022-12-17T17:28:56.602397Z","shell.execute_reply.started":"2022-12-17T17:28:48.187480Z","shell.execute_reply":"2022-12-17T17:28:56.601221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\ntrain_final[feature_columns]=scaler.fit_transform(train_final[feature_columns])\ntest_final[feature_columns]=scaler.transform(test_final[feature_columns])","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:28:56.603677Z","iopub.execute_input":"2022-12-17T17:28:56.604010Z","iopub.status.idle":"2022-12-17T17:30:25.205672Z","shell.execute_reply.started":"2022-12-17T17:28:56.603980Z","shell.execute_reply":"2022-12-17T17:30:25.204417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Initial model estimation","metadata":{}},{"cell_type":"code","source":"log = LogisticRegression(verbose=False, C=0.1, max_iter=1000)\n\nlog, feat_imp=fit_model_using_classifier(log, \n                                           dtrain=train_final, \n                                           predictors=feature_columns,\n                                           repeat=repeat_numb,\n                                           scoring=scoref)\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:30:25.207001Z","iopub.execute_input":"2022-12-17T17:30:25.207343Z","iopub.status.idle":"2022-12-17T17:38:12.694670Z","shell.execute_reply.started":"2022-12-17T17:30:25.207311Z","shell.execute_reply":"2022-12-17T17:38:12.693507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = CatBoostClassifier(verbose=False)\n\nclf, feat_imp=fit_model_using_classifier(clf, \n                                           dtrain=train_final,\n                                           predictors=feature_columns,\n                                           repeat=repeat_numb,\n                                           scoring=scoref)\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-12-17T17:38:12.696172Z","iopub.execute_input":"2022-12-17T17:38:12.696901Z","iopub.status.idle":"2022-12-17T18:21:04.902852Z","shell.execute_reply.started":"2022-12-17T17:38:12.696857Z","shell.execute_reply":"2022-12-17T18:21:04.901566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"catboost_submission=pd.DataFrame({\n    \"id\": test_final[\"id\"],\n    \"target\": clf.predict_proba(test_final[feature_columns])[:,1]\n})\ncatboost_submission.to_csv(\"catboost_submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T18:21:04.904208Z","iopub.execute_input":"2022-12-17T18:21:04.904558Z","iopub.status.idle":"2022-12-17T18:21:08.915565Z","shell.execute_reply.started":"2022-12-17T18:21:04.904528Z","shell.execute_reply":"2022-12-17T18:21:08.914136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Variable selection and refined model estimation","metadata":{}},{"cell_type":"code","source":"# Variable selection using initial model estimate and permutation score\nperm1 = PermutationImportance(clf, random_state=1).fit(train_final[feature_columns], train_final[\"target\"])\nfeature_importance_recalc=pd.DataFrame({\n                        \"feature\": feature_columns,\n                        \"feature_importance\":perm1.feature_importances_*100\n                    })\nfeature_importance_recalc.sort_values(by=[\"feature_importance\"],ascending=False,inplace=True)\nbest_features=list(set(set(feature_importance_recalc.head(best_feature_numb)[\"feature\"]).union(set(feat_imp.index[0:best_feature_numb]))))","metadata":{"execution":{"iopub.status.busy":"2022-12-17T18:21:08.917401Z","iopub.execute_input":"2022-12-17T18:21:08.918178Z","iopub.status.idle":"2022-12-17T19:45:08.667701Z","shell.execute_reply.started":"2022-12-17T18:21:08.918125Z","shell.execute_reply":"2022-12-17T19:45:08.666571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_features=list(set(set(feature_importance_recalc.head(best_feature_numb)[\"feature\"]).union(set(feat_imp.index[0:best_feature_numb]))))","metadata":{"execution":{"iopub.status.busy":"2022-12-17T19:45:08.669611Z","iopub.execute_input":"2022-12-17T19:45:08.670007Z","iopub.status.idle":"2022-12-17T19:45:08.677466Z","shell.execute_reply.started":"2022-12-17T19:45:08.669972Z","shell.execute_reply":"2022-12-17T19:45:08.676048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,6))\nax = sns.barplot(x=\"feature\",\n                 y=\"feature_importance\", \n                 data=feature_importance_recalc[feature_importance_recalc[\"feature_importance\"]>0.01].tail(30), \n                 palette=PALETTE)\nplt.title('Comparison of feature importance scores', fontsize=20);\nplt.ylabel('Feature importance', fontsize=14);\nplt.xlabel('Feature name', fontsize=14);\nplt.xticks(rotation = 90);","metadata":{"execution":{"iopub.status.busy":"2022-12-17T19:45:08.678795Z","iopub.execute_input":"2022-12-17T19:45:08.679115Z","iopub.status.idle":"2022-12-17T19:45:09.183319Z","shell.execute_reply.started":"2022-12-17T19:45:08.679087Z","shell.execute_reply":"2022-12-17T19:45:09.182451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of total features\nlen(best_features)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T19:45:09.184846Z","iopub.execute_input":"2022-12-17T19:45:09.185472Z","iopub.status.idle":"2022-12-17T19:45:09.192695Z","shell.execute_reply.started":"2022-12-17T19:45:09.185433Z","shell.execute_reply":"2022-12-17T19:45:09.191283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf2 = CatBoostClassifier(verbose=False)\n\nclf2, feat_imp=fit_model_using_classifier(clf2, \n                                           dtrain=train_final, \n                                           predictors=best_features,\n                                           repeat=repeat_numb,\n                                           scoring=scoref)\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-12-17T19:45:09.194321Z","iopub.execute_input":"2022-12-17T19:45:09.195149Z","iopub.status.idle":"2022-12-17T20:16:52.871820Z","shell.execute_reply.started":"2022-12-17T19:45:09.195104Z","shell.execute_reply":"2022-12-17T20:16:52.870859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"catboost_submission2=pd.DataFrame({\n    \"id\": test_final[\"id\"],\n    \"target\": clf2.predict_proba(test_final[best_features])[:,1]\n})\ncatboost_submission2.to_csv(\"catboost_submission2.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-17T20:16:52.873286Z","iopub.execute_input":"2022-12-17T20:16:52.874311Z","iopub.status.idle":"2022-12-17T20:16:55.791969Z","shell.execute_reply.started":"2022-12-17T20:16:52.874273Z","shell.execute_reply":"2022-12-17T20:16:55.790826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_params={\n    \"iterations\":1200,\n    \"learning_rate\":0.025,\n    'loss_function' : 'Logloss',\n    \"eval_metric\":\"AUC\",\n    \"verbose\":False\n}\nclf3 = CatBoostClassifier(**cat_params)\n\nclf3, feat_imp=fit_model_using_classifier(clf3, \n                                           dtrain=train_final, \n                                           predictors=best_features,\n                                           repeat=repeat_numb,\n                                           scoring=scoref)\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-12-17T20:16:55.793481Z","iopub.execute_input":"2022-12-17T20:16:55.793861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"catboost_submission3=pd.DataFrame({\n    \"id\": test_final[\"id\"],\n    \"target\": clf3.predict_proba(test_final[best_features])[:,1]\n})\ncatboost_submission3.to_csv(\"catboost_submission3.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}