{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ДЗ 7 Гладышев В.В.\n\nИспользую датасет Kaggle соревнования: https://www.kaggle.com/c/geekbrains-competitive-data-analysis/overview\n<br>В данной работе нужно было построить модель для выявления случаев мошенничества (fraud detection)\n- 0 no fraud\n- 1 fraud","metadata":{"cellId":"9inh4gkdkqaenugl01xwb","id":"vKMNdupZ6Y-_"}},{"cell_type":"code","source":"# %pip install shap","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:06:00.347677Z","iopub.execute_input":"2021-05-24T14:06:00.348132Z","iopub.status.idle":"2021-05-24T14:06:00.352753Z","shell.execute_reply.started":"2021-05-24T14:06:00.348039Z","shell.execute_reply":"2021-05-24T14:06:00.351761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport math\n\nfrom tqdm import tqdm\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import precision_recall_curve\nfrom sklearn.metrics import log_loss\n\nfrom sklearn.model_selection import GridSearchCV\nimport xgboost as xgb\nfrom scipy.stats import ttest_rel\n\nfrom sklearn.model_selection import KFold, StratifiedKFold, train_test_split, cross_val_score, cross_validate\n\nfrom typing import List, Optional, Dict, Tuple\n\nimport shap\nshap.initjs()","metadata":{"cellId":"k1z9uduxz88euvfvnyt","id":"nC5PXTCu6Y_D","outputId":"18c8743e-175e-427d-ee9c-c60432a6b017","execution":{"iopub.status.busy":"2021-05-24T14:06:00.354474Z","iopub.execute_input":"2021-05-24T14:06:00.355055Z","iopub.status.idle":"2021-05-24T14:06:10.331614Z","shell.execute_reply.started":"2021-05-24T14:06:00.355012Z","shell.execute_reply":"2021-05-24T14:06:10.330455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgbm","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:06:10.333627Z","iopub.execute_input":"2021-05-24T14:06:10.334188Z","iopub.status.idle":"2021-05-24T14:06:11.515359Z","shell.execute_reply.started":"2021-05-24T14:06:10.334121Z","shell.execute_reply":"2021-05-24T14:06:11.514281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\ndata_dir = dirname + \"/\"\nworking_dir = \"/kaggle/working/\"","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:06:11.517086Z","iopub.execute_input":"2021-05-24T14:06:11.517386Z","iopub.status.idle":"2021-05-24T14:06:11.525465Z","shell.execute_reply.started":"2021-05-24T14:06:11.517357Z","shell.execute_reply":"2021-05-24T14:06:11.523994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Загрузка данных","metadata":{"cellId":"zudi6k4eubbduiu0safsj","id":"r31123il6Y_J"}},{"cell_type":"code","source":"src_list = [\"train\", \"client_profile\"]","metadata":{"cellId":"dj64uxrs0ancmkkyccbhl","id":"3joPSrp66Y_V","execution":{"iopub.status.busy":"2021-05-24T14:06:11.528408Z","iopub.execute_input":"2021-05-24T14:06:11.528922Z","iopub.status.idle":"2021-05-24T14:06:11.540978Z","shell.execute_reply.started":"2021-05-24T14:06:11.528854Z","shell.execute_reply":"2021-05-24T14:06:11.539453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"source = {}\nfor name in src_list:\n    source[name] = pd.read_csv(dirname + \"/\" + name + \".csv\")","metadata":{"cellId":"sy50jk4tqdcqqylt6h0gt","id":"zZ8mnI636Y_b","execution":{"iopub.status.busy":"2021-05-24T14:06:11.542199Z","iopub.execute_input":"2021-05-24T14:06:11.542512Z","iopub.status.idle":"2021-05-24T14:06:13.201251Z","shell.execute_reply.started":"2021-05-24T14:06:11.542482Z","shell.execute_reply":"2021-05-24T14:06:13.199954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ключевые характеристики","metadata":{"cellId":"81r171htyy9zc3ydjkkdpb","id":"-rnLW6kA6Y_r"}},{"cell_type":"code","source":"for name in src_list:\n    print(f\"{name} \\n {source[name].shape} \\n\")","metadata":{"cellId":"o01yof1fqs9ornfs2bbhm","id":"d6H07HSJ6Y_t","outputId":"8cc3c503-f4c6-41c1-c907-7ac24857e4f1","execution":{"iopub.status.busy":"2021-05-24T14:06:13.202768Z","iopub.execute_input":"2021-05-24T14:06:13.203227Z","iopub.status.idle":"2021-05-24T14:06:13.209057Z","shell.execute_reply.started":"2021-05-24T14:06:13.203184Z","shell.execute_reply":"2021-05-24T14:06:13.207953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"source['train'].head(3)","metadata":{"cellId":"g3lbxfv5w6wecfeqnot6r","id":"-7QSPiET6Y_6","outputId":"a563576c-57b6-4e45-a19a-69e8a9494528","execution":{"iopub.status.busy":"2021-05-24T14:06:13.210439Z","iopub.execute_input":"2021-05-24T14:06:13.210766Z","iopub.status.idle":"2021-05-24T14:06:13.240574Z","shell.execute_reply.started":"2021-05-24T14:06:13.210735Z","shell.execute_reply":"2021-05-24T14:06:13.239480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"source['client_profile'].head(3)","metadata":{"cellId":"xcdc7ku7132yo8g8411lg","id":"2IvWFbs96ZAA","outputId":"e9324ce0-8b4a-4dd1-bb04-d91eaa2d322e","execution":{"iopub.status.busy":"2021-05-24T14:06:13.243162Z","iopub.execute_input":"2021-05-24T14:06:13.243454Z","iopub.status.idle":"2021-05-24T14:06:13.275642Z","shell.execute_reply.started":"2021-05-24T14:06:13.243425Z","shell.execute_reply":"2021-05-24T14:06:13.274522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for name in src_list:\n    print(f\"{name} \\n\")\n    source[name].info()\n    print(\"\\n\")","metadata":{"cellId":"1p3yw74a1kbxyqrvtp0gxk","id":"_EC_Ljeu6ZAr","outputId":"a788d578-7665-4c3e-bbf3-c9afc7b6a017","scrolled":true,"execution":{"iopub.status.busy":"2021-05-24T14:06:13.277948Z","iopub.execute_input":"2021-05-24T14:06:13.278239Z","iopub.status.idle":"2021-05-24T14:06:13.401607Z","shell.execute_reply.started":"2021-05-24T14:06:13.278211Z","shell.execute_reply":"2021-05-24T14:06:13.400507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = source['train']\ndata.tail()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:06:13.402791Z","iopub.execute_input":"2021-05-24T14:06:13.403088Z","iopub.status.idle":"2021-05-24T14:06:13.412330Z","shell.execute_reply.started":"2021-05-24T14:06:13.403059Z","shell.execute_reply":"2021-05-24T14:06:13.411499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Добавляем client_profile","metadata":{"cellId":"g3t8fscnl1tzfm8q4331","id":"P0IFgUFR6ZBc"}},{"cell_type":"code","source":"def create_client_profile_features(X: pd.DataFrame, copy: bool = True) -> pd.DataFrame:\n    \"\"\"\n    Создание признаков на основе профиля клиентов.\n\n    Parameters\n    ----------\n    X: pandas.core.frame.DataFrame\n        Матрица признаков с исходным профилем клиента.\n\n    copy: bool, optional, default = True\n        Флаг использования копии датафрейма X.\n        Опциональный параметр, по умолчанию, равен True.\n\n    Returns\n    -------\n    X_transformed: pandas.core.frame.DataFrame\n        Расширенная матрица признаков с профилем клиентов.\n\n    \"\"\"\n    if copy:\n        X = X.copy()\n\n    X[\"DAYS_ON_LAST_JOB\"] = X[\"DAYS_ON_LAST_JOB\"].replace(365243, np.nan)\n    bki_flags = [flag for flag in X.columns if \"AMT_REQ_CREDIT_BUREAU\" in flag]\n    X[\"BKI_REQUESTS_COUNT\"] = X[bki_flags].sum(axis=1)\n    X[\"BKI_KURTOSIS\"] = X[bki_flags].kurtosis(axis=1)\n\n    X[\"EXTERNAL_SCORING_PROD\"] = X[\"EXTERNAL_SCORING_RATING_1\"] * X[\"EXTERNAL_SCORING_RATING_2\"] * X[\"EXTERNAL_SCORING_RATING_3\"]\n    X[\"EXTERNAL_SCORING_WEIGHTED\"] = X.EXTERNAL_SCORING_RATING_1 * 2 + X.EXTERNAL_SCORING_RATING_2 * 1 + X.EXTERNAL_SCORING_RATING_3 * 3\n\n    for function_name in [\"min\", \"max\", \"mean\", \"nanmedian\", \"var\"]:\n        feature_name = \"EXTERNAL_SCORING_RATING_{}\".format(function_name)\n        X[feature_name] = eval(\"np.{}\".format(function_name))(\n            X[[\"EXTERNAL_SCORING_RATING_1\", \"EXTERNAL_SCORING_RATING_2\", \"EXTERNAL_SCORING_RATING_3\"]], axis=1\n        )\n\n    # Отношение между основными фин. показателями\n    X['RATIO_CREDIT_TO_ANNUITY'] = X['AMOUNT_CREDIT'] / X['AMOUNT_ANNUITY']\n    X[\"RATIO_ANNUITY_TO_SALARY\"] = X['AMOUNT_ANNUITY'] / X['TOTAL_SALARY']\n    X['RATIO_CREDIT_TO_SALARY'] = X['AMOUNT_CREDIT'] / X['TOTAL_SALARY']\n    #X[\"TOTAL_SALARY_NET\"] = X[\"TOTAL_SALARY\"] - X[\"AMOUNT_ANNUITY\"]\n\n    # Отношение фин. показателей к возрасту и временным фичам\n    X[\"RATIO_ANNUITY_TO_AGE\"] = X[\"AMOUNT_ANNUITY\"] / X[\"AGE\"]\n    X[\"RATIO_CREDIT_TO_AGE\"] = X[\"AMOUNT_CREDIT\"] / X[\"AGE\"]\n    X[\"RATIO_SALARY_TO_AGE\"] = X[\"TOTAL_SALARY\"] / X[\"AGE\"]\n    X[\"RATIO_SALARY_TO_EXPERIENCE\"] = X[\"TOTAL_SALARY\"] / X[\"DAYS_ON_LAST_JOB\"]\n    X[\"RATIO_CREDIT_TO_EXPERIENCE\"] = X[\"AMOUNT_CREDIT\"] / X[\"DAYS_ON_LAST_JOB\"]\n    X[\"RATIO_ANNUITY_TO_EXPERIENCE\"] = X[\"AMOUNT_ANNUITY\"] / X[\"DAYS_ON_LAST_JOB\"]\n\n    # Отношение врменных признаков\n    X[\"RATIO_AGE_TO_EXPERIENCE\"] = X[\"AGE\"] / X[\"DAYS_ON_LAST_JOB\"]\n    X[\"RATIO_SALARY_TO_REGION_POPULATION\"] = X[\"TOTAL_SALARY\"] * X[\"REGION_POPULATION\"]\n    X[\"RATIO_CAR_TO_EXPERIENCE\"] = X[\"OWN_CAR_AGE\"] / X[\"DAYS_ON_LAST_JOB\"]\n    X[\"RATIO_CAR_TO_AGE\"] = X[\"OWN_CAR_AGE\"] / X[\"AGE\"]\n\n    # Произведение фин. показателей кредита на вероятность дефолта\n    # Такая штука называется математическим ожиданием дефолта или ожидаемыми потерями\n    X[\"EXPECTED_TOTAL_LOSS_1\"] = X[\"EXTERNAL_SCORING_RATING_1\"] * X[\"AMOUNT_CREDIT\"]\n    X[\"EXPECTED_TOTAL_LOSS_2\"] = X[\"EXTERNAL_SCORING_RATING_2\"] * X[\"AMOUNT_CREDIT\"]\n    X[\"EXPECTED_TOTAL_LOSS_3\"] = X[\"EXTERNAL_SCORING_RATING_3\"] * X[\"AMOUNT_CREDIT\"]\n    X[\"EXPECTED_MONTHLY_LOSS_1\"] = X[\"EXTERNAL_SCORING_RATING_1\"] * X[\"AMOUNT_ANNUITY\"]\n    X[\"EXPECTED_MONTHLY_LOSS_2\"] = X[\"EXTERNAL_SCORING_RATING_2\"] * X[\"AMOUNT_ANNUITY\"]\n    X[\"EXPECTED_MONTHLY_LOSS_3\"] = X[\"EXTERNAL_SCORING_RATING_3\"] * X[\"AMOUNT_ANNUITY\"]\n\n    return X","metadata":{"cellId":"fyx4jln09ur01ki818v6smr","cellView":"both","id":"zescElkkPj0Z","execution":{"iopub.status.busy":"2021-05-24T14:06:13.413502Z","iopub.execute_input":"2021-05-24T14:06:13.413802Z","iopub.status.idle":"2021-05-24T14:06:13.431379Z","shell.execute_reply.started":"2021-05-24T14:06:13.413773Z","shell.execute_reply":"2021-05-24T14:06:13.430228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"client_profile = create_client_profile_features(source[\"client_profile\"])\nclient_profile.head(n=2)","metadata":{"cellId":"7m6gtysxbfmch79p4dnlg5","id":"pTFlEpSpPl0K","outputId":"af5aac36-54b4-4402-9281-b4ecc0621b8e","execution":{"iopub.status.busy":"2021-05-24T14:06:13.432658Z","iopub.execute_input":"2021-05-24T14:06:13.433055Z","iopub.status.idle":"2021-05-24T14:06:14.128372Z","shell.execute_reply.started":"2021-05-24T14:06:13.433022Z","shell.execute_reply":"2021-05-24T14:06:14.127666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.merge(data, client_profile, on='APPLICATION_NUMBER', how='left')\ndata.head()","metadata":{"cellId":"4r65gtof29ky0775kdblqn","id":"0C-AIKuW6ZBd","outputId":"12ff403d-d85d-45a6-c0c4-2e0c4726e68d","execution":{"iopub.status.busy":"2021-05-24T14:06:14.129421Z","iopub.execute_input":"2021-05-24T14:06:14.129897Z","iopub.status.idle":"2021-05-24T14:06:14.621381Z","shell.execute_reply.started":"2021-05-24T14:06:14.129847Z","shell.execute_reply":"2021-05-24T14:06:14.620725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.tail()","metadata":{"cellId":"8w5lyu5bu0oqlzptc01th","id":"XvCNAfOD6ZBi","outputId":"9d8dc77e-8ff0-4bb6-d291-879ca322a758","execution":{"iopub.status.busy":"2021-05-24T14:06:14.622534Z","iopub.execute_input":"2021-05-24T14:06:14.622808Z","iopub.status.idle":"2021-05-24T14:06:14.653390Z","shell.execute_reply.started":"2021-05-24T14:06:14.622782Z","shell.execute_reply":"2021-05-24T14:06:14.652266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"cellId":"sr2e8wp97s9b6uli6mbwc","id":"k1mysemA6ZBm","outputId":"dfe3a446-f9c1-4994-9f6d-2a4f557cdf3c","execution":{"iopub.status.busy":"2021-05-24T14:06:14.654826Z","iopub.execute_input":"2021-05-24T14:06:14.655242Z","iopub.status.idle":"2021-05-24T14:06:14.666289Z","shell.execute_reply.started":"2021-05-24T14:06:14.655197Z","shell.execute_reply":"2021-05-24T14:06:14.665326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Заполняем пропуски","metadata":{"cellId":"7jtays2hrfb83rc5tmj60x","id":"alvPcQLe6ZD-"}},{"cell_type":"code","source":"obj_col_names = data.select_dtypes(include=['object']).columns\nobj_col_names","metadata":{"cellId":"b00kbhhkvxju629gdeiju","id":"eJb5h11c6ZD-","outputId":"6d2a4543-bcb4-435e-86c8-37a041192c39","execution":{"iopub.status.busy":"2021-05-24T14:06:14.667556Z","iopub.execute_input":"2021-05-24T14:06:14.668154Z","iopub.status.idle":"2021-05-24T14:06:14.716650Z","shell.execute_reply.started":"2021-05-24T14:06:14.668113Z","shell.execute_reply":"2021-05-24T14:06:14.715665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.loc[data[obj_col_names[1]].isnull(), obj_col_names[1]]","metadata":{"cellId":"2oq7wdwzf85r4lex577as","id":"tSbDPvMw6ZEB","outputId":"12233d9c-1f3b-4de8-d495-e6213b4b165b","execution":{"iopub.status.busy":"2021-05-24T14:06:14.717816Z","iopub.execute_input":"2021-05-24T14:06:14.718390Z","iopub.status.idle":"2021-05-24T14:06:14.738149Z","shell.execute_reply.started":"2021-05-24T14:06:14.718346Z","shell.execute_reply":"2021-05-24T14:06:14.737194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in obj_col_names:\n    data.loc[data[col].isnull(), col] = 'No'","metadata":{"cellId":"jmfkhc1wi8hgwumewtl4zs","id":"Yi85A4lv6ZEE","execution":{"iopub.status.busy":"2021-05-24T14:06:14.739478Z","iopub.execute_input":"2021-05-24T14:06:14.739779Z","iopub.status.idle":"2021-05-24T14:06:14.800703Z","shell.execute_reply.started":"2021-05-24T14:06:14.739750Z","shell.execute_reply":"2021-05-24T14:06:14.799484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_col_names = data.select_dtypes(exclude=['object']).columns\nnum_col_names","metadata":{"cellId":"swwxl03mh3digq1nplpjh","id":"yY1spBlT6ZEH","outputId":"c217d362-b1e2-4e48-9106-cfd86d8407d3","execution":{"iopub.status.busy":"2021-05-24T14:06:14.802124Z","iopub.execute_input":"2021-05-24T14:06:14.802430Z","iopub.status.idle":"2021-05-24T14:06:14.826001Z","shell.execute_reply.started":"2021-05-24T14:06:14.802400Z","shell.execute_reply":"2021-05-24T14:06:14.825062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:06:14.827406Z","iopub.execute_input":"2021-05-24T14:06:14.827853Z","iopub.status.idle":"2021-05-24T14:06:14.945667Z","shell.execute_reply.started":"2021-05-24T14:06:14.827808Z","shell.execute_reply":"2021-05-24T14:06:14.944725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in num_col_names:\n    data.loc[data[col].isnull(), col] = -1","metadata":{"cellId":"wdr10j4f537y2amr6grh5","id":"QUAvfN--6ZEL","execution":{"iopub.status.busy":"2021-05-24T14:06:14.947974Z","iopub.execute_input":"2021-05-24T14:06:14.948412Z","iopub.status.idle":"2021-05-24T14:06:15.025407Z","shell.execute_reply.started":"2021-05-24T14:06:14.948365Z","shell.execute_reply":"2021-05-24T14:06:15.024396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"cellId":"z0zp7ntfxwi02r5mv8anqah","id":"IRy-XrGh6ZEO","outputId":"d44a848f-4739-4899-8422-641b1fbf109c","execution":{"iopub.status.busy":"2021-05-24T14:06:15.026936Z","iopub.execute_input":"2021-05-24T14:06:15.027397Z","iopub.status.idle":"2021-05-24T14:06:15.058078Z","shell.execute_reply.started":"2021-05-24T14:06:15.027349Z","shell.execute_reply":"2021-05-24T14:06:15.056984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Преобразуем категориальные признаки в числовые","metadata":{"cellId":"zoocibov4g3hpedqqhbik","id":"d9NhoAWs6ZFA"}},{"cell_type":"code","source":"data_cat = data.copy()","metadata":{"cellId":"7t47a3xfykl2th0rsldo6","id":"hpW01veNUSNg","execution":{"iopub.status.busy":"2021-05-24T14:06:15.063205Z","iopub.execute_input":"2021-05-24T14:06:15.063536Z","iopub.status.idle":"2021-05-24T14:06:15.086317Z","shell.execute_reply.started":"2021-05-24T14:06:15.063507Z","shell.execute_reply":"2021-05-24T14:06:15.085212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"obj_col = data.select_dtypes(exclude=[np.number])","metadata":{"cellId":"m7ervb0souraeckch9zfh","id":"07PZp9mh6ZFB","execution":{"iopub.status.busy":"2021-05-24T14:06:15.088215Z","iopub.execute_input":"2021-05-24T14:06:15.088529Z","iopub.status.idle":"2021-05-24T14:06:15.097514Z","shell.execute_reply.started":"2021-05-24T14:06:15.088498Z","shell.execute_reply":"2021-05-24T14:06:15.096391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_obj_discribe(data: pd.DataFrame):\n# если столбец имеет тип 'object' и содержит менее 30 уникальных \n# распечатывает значения столбцов и подсчитывает количество значений\n\n    # data: pd.DataFrame исходный датафрейм\n    \n    obj_fit = data.select_dtypes(include='object')\n    print('*' * 100)\n    for cat_colname in obj_fit.columns:\n        if len(obj_fit[cat_colname].unique()) < 30:\n            print(str(cat_colname) + '\\n\\n' + str(obj_fit[cat_colname].unique()) + '\\n\\n')\n            print(obj_fit.groupby(cat_colname)[cat_colname].count())\n            print('*' * 100 + '\\n')","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:06:15.098789Z","iopub.execute_input":"2021-05-24T14:06:15.099124Z","iopub.status.idle":"2021-05-24T14:06:15.110676Z","shell.execute_reply.started":"2021-05-24T14:06:15.099095Z","shell.execute_reply":"2021-05-24T14:06:15.109787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not obj_col.empty:\n    data_obj_discribe(data)","metadata":{"cellId":"z2o2438b1brrcb2ha2hqvn","id":"0EVrvSo86ZFH","outputId":"9c7c81d6-2360-4e53-f28d-51215381107e","execution":{"iopub.status.busy":"2021-05-24T14:06:15.112175Z","iopub.execute_input":"2021-05-24T14:06:15.112591Z","iopub.status.idle":"2021-05-24T14:06:15.283695Z","shell.execute_reply.started":"2021-05-24T14:06:15.112543Z","shell.execute_reply":"2021-05-24T14:06:15.282448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_obj_map(data: pd.DataFrame):\n# если столбец имеет тип 'object' и содержит менее 30 уникальных \n# преобразует категориальные признак в числовые\n\n    # data: pd.DataFrame исходный датафрейм\n    \n    # изменяет датафрейм источник\n    \n    obj_fit = data.select_dtypes(include='object')\n    for cat_colname in obj_fit.columns:\n        map_tr = {}\n        un_val = obj_fit[cat_colname].unique()\n        if len(un_val) < 30:\n            print(str(cat_colname) + '\\n')\n            for i in range(len(un_val)):\n                map_tr[un_val[i]] = i\n            print(map_tr)\n            data[cat_colname] = data[cat_colname].map(map_tr)\n            print('*' * 100 + '\\n')","metadata":{"cellId":"7z33uf858zglolhhv629zo","id":"m7dqiS-z6ZFL","execution":{"iopub.status.busy":"2021-05-24T14:06:15.285068Z","iopub.execute_input":"2021-05-24T14:06:15.285437Z","iopub.status.idle":"2021-05-24T14:06:15.292559Z","shell.execute_reply.started":"2021-05-24T14:06:15.285405Z","shell.execute_reply":"2021-05-24T14:06:15.291314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not obj_col.empty:\n    data_obj_map(data)","metadata":{"cellId":"ddmdzy680ptpahszl845o","id":"y0SSKLeq6ZFP","outputId":"ada58880-d2c6-4fb3-e216-9a2a777ad4f2","execution":{"iopub.status.busy":"2021-05-24T14:06:15.294290Z","iopub.execute_input":"2021-05-24T14:06:15.294722Z","iopub.status.idle":"2021-05-24T14:06:15.402022Z","shell.execute_reply.started":"2021-05-24T14:06:15.294680Z","shell.execute_reply":"2021-05-24T14:06:15.400727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"cellId":"zqqevbd1ly9eknbbdttzd","id":"6Sz3xYb-6ZFS","outputId":"b9b133d4-50d9-47fc-cbeb-a0dbb0948583","execution":{"iopub.status.busy":"2021-05-24T14:06:15.403529Z","iopub.execute_input":"2021-05-24T14:06:15.404039Z","iopub.status.idle":"2021-05-24T14:06:15.439063Z","shell.execute_reply.started":"2021-05-24T14:06:15.404001Z","shell.execute_reply":"2021-05-24T14:06:15.437980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in data.columns:\n    data.loc[data[col].isnull(), col] = -1","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:06:15.440298Z","iopub.execute_input":"2021-05-24T14:06:15.440583Z","iopub.status.idle":"2021-05-24T14:06:15.494920Z","shell.execute_reply.started":"2021-05-24T14:06:15.440554Z","shell.execute_reply":"2021-05-24T14:06:15.493943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## XGBoost","metadata":{"cellId":"6svp7frdnv3xvjn03ethda","id":"tZ5MdPvLDU3M"}},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(data.drop(\"TARGET\", axis=1), \n                                                      data[\"TARGET\"], \n                                                      test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:06:15.496134Z","iopub.execute_input":"2021-05-24T14:06:15.496445Z","iopub.status.idle":"2021-05-24T14:06:15.571303Z","shell.execute_reply.started":"2021-05-24T14:06:15.496414Z","shell.execute_reply":"2021-05-24T14:06:15.570232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_xgb = xgb.XGBClassifier(objective=\"binary:logistic\",\n                              nthread=2, \n                              max_depth=5, \n                              n_estimators=1000, \n                              subsample=0.5, \n                              learning_rate=0.3, \n                              seed=1400, \n                              random_state=42,\n                              eval_metric='auc')\n\nmodel_xgb.fit(X_train, y_train)\n\npreds = model_xgb.predict(X_valid)\n\naccuracy_score(y_valid, preds)","metadata":{"cellId":"ky95s46m9naerfwu0rn13q","id":"MfCb8FxW_PT3","execution":{"iopub.status.busy":"2021-05-24T14:06:15.572700Z","iopub.execute_input":"2021-05-24T14:06:15.573037Z","iopub.status.idle":"2021-05-24T14:09:37.958590Z","shell.execute_reply.started":"2021-05-24T14:06:15.573005Z","shell.execute_reply":"2021-05-24T14:09:37.957521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XGBoost prediction","metadata":{}},{"cell_type":"code","source":"rw_clissifiers = ['XGBoost']\n\ncl_met = ['Best Threshold', 'F-Score', 'Precision', \n          'Recall', 'roc_auc_s', 'log_loss_s', 'TPR', 'FPR', 'TNR', \"TN\", \"FN\", \"TP\", \"FP\"]\n\nres_tab = pd.DataFrame(columns=cl_met)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:37.959971Z","iopub.execute_input":"2021-05-24T14:09:37.960331Z","iopub.status.idle":"2021-05-24T14:09:37.969467Z","shell.execute_reply.started":"2021-05-24T14:09:37.960299Z","shell.execute_reply":"2021-05-24T14:09:37.967916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"precision, recall, thresholds = precision_recall_curve(y_valid, preds)\n\nfscore = (2 * precision * recall) / (precision + recall)\n# locate the index of the largest f score\nix = np.argmax(fscore)\nprint('Best Threshold=%f, F-Score=%.3f, Precision=%.3f, Recall=%.3f' % (thresholds[ix], \n                                                                        fscore[ix],\n                                                                        precision[ix],\n                                                                        recall[ix]))","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:37.971060Z","iopub.execute_input":"2021-05-24T14:09:37.971457Z","iopub.status.idle":"2021-05-24T14:09:37.984308Z","shell.execute_reply.started":"2021-05-24T14:09:37.971406Z","shell.execute_reply":"2021-05-24T14:09:37.982935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r_auc = roc_auc_score(y_true=y_valid, y_score=preds)\nl_los = log_loss(y_true=y_valid, y_pred=preds)\n\nprint(\"roc auc score: {}\".format(r_auc))\nprint(\"log loss score: {}\".format(l_los))","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:37.985681Z","iopub.execute_input":"2021-05-24T14:09:37.986022Z","iopub.status.idle":"2021-05-24T14:09:38.004269Z","shell.execute_reply.started":"2021-05-24T14:09:37.985991Z","shell.execute_reply":"2021-05-24T14:09:38.003070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnf_matrix = confusion_matrix(y_valid, preds>thresholds[ix])\n\nTN = cnf_matrix[0][0]\nFN = cnf_matrix[1][0]\nTP = cnf_matrix[1][1]\nFP = cnf_matrix[0][1]\n\nTPR = TP/(TP+FN)\nFPR = FP/(FP+TN)\nTNR = TN/(FP+TN)\nTPR, FPR, TNR","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:38.005526Z","iopub.execute_input":"2021-05-24T14:09:38.005812Z","iopub.status.idle":"2021-05-24T14:09:38.108249Z","shell.execute_reply.started":"2021-05-24T14:09:38.005786Z","shell.execute_reply":"2021-05-24T14:09:38.107157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_tab.loc['XGBoost', :] = [thresholds[ix], fscore[ix], precision[ix], recall[ix], \n                             r_auc, l_los, TPR, FPR, TNR, TN, FN, TP, FP]\nres_tab","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:38.109596Z","iopub.execute_input":"2021-05-24T14:09:38.109931Z","iopub.status.idle":"2021-05-24T14:09:38.132012Z","shell.execute_reply.started":"2021-05-24T14:09:38.109900Z","shell.execute_reply":"2021-05-24T14:09:38.130825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature importance","metadata":{}},{"cell_type":"code","source":"lgbm_params =  {\n    'boosting_type': 'gbdt',\n    'objective': 'binary',\n    'metric': 'auc',\n    \"learning_rate\": 0.1,\n    \"num_leaves\": 31,\n    \"random_state\": 42 \n    }","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:38.133295Z","iopub.execute_input":"2021-05-24T14:09:38.133599Z","iopub.status.idle":"2021-05-24T14:09:38.138693Z","shell.execute_reply.started":"2021-05-24T14:09:38.133568Z","shell.execute_reply":"2021-05-24T14:09:38.137456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = lgbm.Dataset(X_train, label=y_train)\ntest_data = lgbm.Dataset(X_valid, label=y_valid)\n\nmodel = lgbm.train(lgbm_params,\n                   train_data,\n                   valid_sets=test_data,\n                   num_boost_round=1000,\n                   early_stopping_rounds=50,\n                   verbose_eval=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:38.140363Z","iopub.execute_input":"2021-05-24T14:09:38.140818Z","iopub.status.idle":"2021-05-24T14:09:40.827858Z","shell.execute_reply.started":"2021-05-24T14:09:38.140771Z","shell.execute_reply":"2021-05-24T14:09:40.826854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = lgbm.plot_importance(model, max_num_features=355, figsize=(5,45))   #, figsize=(45,5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:40.829273Z","iopub.execute_input":"2021-05-24T14:09:40.829671Z","iopub.status.idle":"2021-05-24T14:09:42.401656Z","shell.execute_reply.started":"2021-05-24T14:09:40.829631Z","shell.execute_reply":"2021-05-24T14:09:42.400635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Permutation importance","metadata":{}},{"cell_type":"code","source":"def calculate_permutation_importance(estimator, \n                                     metric: callable,\n                                     x_valid: pd.DataFrame,\n                                     y_valid: pd.Series) -> pd.Series:\n    \"\"\"\n    Расчет пермутированной важности признаков.\n    \"\"\"\n    scores = {}\n    y_pred = estimator.predict(x_valid)\n    base_score = metric(y_valid, y_pred)\n\n    for feature in tqdm(x_valid.columns):\n        x_valid_copy = x_valid.copy()\n        x_valid_copy[feature] = np.random.permutation(x_valid_copy[feature])\n\n        y_pred = estimator.predict(x_valid_copy)\n        score = metric(y_valid, y_pred)\n        scores[feature] = base_score - score\n\n    scores = pd.Series(scores)\n    scores = scores.sort_values(ascending=False)\n\n    return scores","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:42.403048Z","iopub.execute_input":"2021-05-24T14:09:42.403372Z","iopub.status.idle":"2021-05-24T14:09:42.410615Z","shell.execute_reply.started":"2021-05-24T14:09:42.403340Z","shell.execute_reply":"2021-05-24T14:09:42.409619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = lgbm.Dataset(X_train, label=y_train)\ntest_data = lgbm.Dataset(X_valid, label=y_valid)\n\nmodel = lgbm.train(lgbm_params,\n                   train_data,\n                   valid_sets=test_data,\n                   num_boost_round=1000,\n                   early_stopping_rounds=50,\n                   verbose_eval=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:42.411939Z","iopub.execute_input":"2021-05-24T14:09:42.412464Z","iopub.status.idle":"2021-05-24T14:09:45.022453Z","shell.execute_reply.started":"2021-05-24T14:09:42.412418Z","shell.execute_reply":"2021-05-24T14:09:45.021474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"perm_importance = calculate_permutation_importance(\n    estimator=model, metric=roc_auc_score, x_valid=X_valid, y_valid=y_valid\n)\nperm_importance","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:45.023916Z","iopub.execute_input":"2021-05-24T14:09:45.024595Z","iopub.status.idle":"2021-05-24T14:09:49.446374Z","shell.execute_reply.started":"2021-05-24T14:09:45.024546Z","shell.execute_reply":"2021-05-24T14:09:49.445356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SHAP","metadata":{}},{"cell_type":"code","source":"# explain the model's predictions using SHAP\n# (same syntax works for LightGBM, CatBoost, scikit-learn and spark models)\nexplainer = shap.TreeExplainer(model_xgb)\nshap_values = explainer.shap_values(X_train)\n\n# visualize the first prediction's explanation (use matplotlib=True to avoid Javascript)\nshap.force_plot(explainer.expected_value, shap_values[0,:], X_train.iloc[0,:])","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:09:49.447810Z","iopub.execute_input":"2021-05-24T14:09:49.448414Z","iopub.status.idle":"2021-05-24T14:14:09.573548Z","shell.execute_reply.started":"2021-05-24T14:09:49.448368Z","shell.execute_reply":"2021-05-24T14:14:09.572505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Из 53 признаков наибольший вклад: \n   - в пользу fraud=1 внесли DAYS_ON_LAST_JOB и RATIO_CAR_TO_AGE и т.д.\n   - в пользу fraud=0 внесли (по убыванию): APPLICATION_NUMBER, AGE, EXTERNAL_SCORING_RATING_min и т.д.\n<br>Результирующее значение -7 (не fraud)","metadata":{}},{"cell_type":"code","source":"client = 2048\nshap.force_plot(explainer.expected_value, shap_values[client,:], X_train.iloc[client,:])","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:31:25.923983Z","iopub.execute_input":"2021-05-24T14:31:25.924505Z","iopub.status.idle":"2021-05-24T14:31:25.934683Z","shell.execute_reply.started":"2021-05-24T14:31:25.924471Z","shell.execute_reply":"2021-05-24T14:31:25.933126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Из 53 признаков наибольший вклад: \n   - в пользу fraud=1 внесли (по убыванию): AGE, APPLICATION_NUMBER и т.д.\n   - в пользу fraud=0 внесли (по убыванию): EXPECTED_TOTAL_LOSS, EDUCATION_LEVEL, EXPECTED_MONTHLY_LOSS_3 и т.д.\n<br>Результирующее значение -3.25 (не fraud, но не так как -7 пред клиента)","metadata":{}},{"cell_type":"code","source":"# summarize the effects of all the features\nshap.summary_plot(shap_values, X_train)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:14:09.575143Z","iopub.execute_input":"2021-05-24T14:14:09.575733Z","iopub.status.idle":"2021-05-24T14:14:26.283231Z","shell.execute_reply.started":"2021-05-24T14:14:09.575687Z","shell.execute_reply":"2021-05-24T14:14:26.282145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"При повышении значения EXTERNAL_SCORING_RATING_mean (самого важного признака) снижается вероятность fraud (зависимость не линейная (см. ниже), но связь есть), аналогично ведут себя подобные деривативы от EXTERNAL_SCORING_RATING - max, min, nanmedian\n\nAPPLICATION_NUMBER грязный, т.е. неоднозначный признак . да ещё основная масса значений около 0 надо дополнительно исследовать. Зависимость не очевидна. Кстати, RATIO_CRDIT_TO_ANNUITY ведёт себя аналогично.\n\nAGE клиенты более сташего возраста менее склоныы к fraud (не без исключений конечно)","metadata":{}},{"cell_type":"code","source":"shap.summary_plot(shap_values, X_train, plot_type=\"bar\")","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:14:26.284544Z","iopub.execute_input":"2021-05-24T14:14:26.284849Z","iopub.status.idle":"2021-05-24T14:14:26.661931Z","shell.execute_reply.started":"2021-05-24T14:14:26.284820Z","shell.execute_reply":"2021-05-24T14:14:26.660938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Список самых важных признаков (в порядке по убыванию)","metadata":{}},{"cell_type":"code","source":"shap.dependence_plot('EXTERNAL_SCORING_RATING_mean', shap_values, X_train)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:17:35.562818Z","iopub.execute_input":"2021-05-24T14:17:35.563330Z","iopub.status.idle":"2021-05-24T14:17:37.852090Z","shell.execute_reply.started":"2021-05-24T14:17:35.563297Z","shell.execute_reply":"2021-05-24T14:17:37.850953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Наглядная демострация влияния EXTERNAL_SCORING_RATING_mean на целевую переменную (вероятность fraud). Зависимость обратная, чим больше EXTERNAL_SCORING_RATING_mean, тем реже fraud. Цветом обозначено влияние EXTERNAL_SCORING_WEIGHTED на EXTERNAL_SCORING_RATING_mean - чем больше оба эти признака тем реже fraud","metadata":{}},{"cell_type":"code","source":"shap.dependence_plot('AGE', shap_values, X_train)","metadata":{"execution":{"iopub.status.busy":"2021-05-24T14:17:57.296086Z","iopub.execute_input":"2021-05-24T14:17:57.296705Z","iopub.status.idle":"2021-05-24T14:17:59.642200Z","shell.execute_reply.started":"2021-05-24T14:17:57.296666Z","shell.execute_reply":"2021-05-24T14:17:59.641191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Очень лохматый признак. Заметно, что fraud чаще случается среди самых молодых клиентов, а с возрастом вероятность несколко снижается. Также заметно, что чем выше стаж, тем ниже веротность fraud.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}