{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport xgboost as xgb\nimport lightgbm as lgb\nimport catboost as cb\nimport matplotlib.pyplot as plt\n\nfrom scipy.stats import gmean, rankdata\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.13598Z","start_time":"2021-04-28T09:27:56.895662Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:02.099447Z","iopub.execute_input":"2021-10-18T10:16:02.099857Z","iopub.status.idle":"2021-10-18T10:16:05.006861Z","shell.execute_reply.started":"2021-10-18T10:16:02.099766Z","shell.execute_reply":"2021-10-18T10:16:05.005744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def profiling(df):\n    \n    df['DAYS_ON_LAST_JOB'].replace(365243, np.nan, inplace= True)\n    df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMOUNT_CREDIT'] / df['AMOUNT_ANNUITY']\n    df['NEW_STATUS'] = df['GENDER'] + df['FAMILY_STATUS'].astype(str)\n    \n    df, cat_cols = get_encoded(df, nan_as_category=False)\n    \n    return df","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.143554Z","start_time":"2021-04-28T09:27:58.137571Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.008967Z","iopub.execute_input":"2021-10-18T10:16:05.009461Z","iopub.status.idle":"2021-10-18T10:16:05.015555Z","shell.execute_reply.started":"2021-10-18T10:16:05.009388Z","shell.execute_reply":"2021-10-18T10:16:05.014532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_client_profile_features(X):\n\n    X[\"RATIO_ANNUITY_TO_AGE\"] = X[\"AMOUNT_ANNUITY\"] / X[\"AGE\"]\n    X[\"RATIO_CREDIT_TO_AGE\"] = X[\"AMOUNT_CREDIT\"] / X[\"AGE\"]\n    X[\"RATIO_SALARY_TO_AGE\"] = X[\"TOTAL_SALARY\"] / X[\"AGE\"]\n    X[\"RATIO_AGE_TO_EXPERIENCE\"] = X[\"AGE\"] / X[\"DAYS_ON_LAST_JOB\"]\n    X[\"RATIO_CAR_TO_EXPERIENCE\"] = X[\"OWN_CAR_AGE\"] / X[\"DAYS_ON_LAST_JOB\"]\n    X[\"RATIO_CAR_TO_AGE\"] = X[\"OWN_CAR_AGE\"] / X[\"AGE\"]\n\n    df, cat_cols = get_encoded(X, nan_as_category=False)\n    \n    return df","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.15253Z","start_time":"2021-04-28T09:27:58.146546Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.017602Z","iopub.execute_input":"2021-10-18T10:16:05.018189Z","iopub.status.idle":"2021-10-18T10:16:05.027562Z","shell.execute_reply.started":"2021-10-18T10:16:05.018142Z","shell.execute_reply":"2021-10-18T10:16:05.026794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bki(num_rows = None, nan_as_category = True):\n    bureau = pd.read_csv(PATH + 'bki.csv', nrows = num_rows)\n    bureau, bureau_cat = get_encoded(bureau, nan_as_category)\n\n    bureau.drop(['BUREAU_ID'], axis=1, inplace= True)\n    \n    num_aggregations = {\n        'CREDIT_DAY_OVERDUE': ['mean'],\n        'AMT_CREDIT_MAX_OVERDUE': ['mean'],\n        'AMT_CREDIT_SUM_OVERDUE': ['mean']\n    }\n    cat_aggregations = {}\n    for cat in bureau_cat: cat_aggregations[cat] = ['mean']\n    \n    bureau_agg = bureau.groupby(COLUMN_ID).agg({**num_aggregations, **cat_aggregations})\n    bureau_agg.columns = pd.Index(['BURO_' + e[0] + \"_\" + e[1].upper() for e in bureau_agg.columns.tolist()])\n    \n    active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]\n    active_agg = active.groupby(COLUMN_ID).agg(num_aggregations)\n    active_agg.columns = pd.Index(['ACTIVE_' + e[0] + \"_\" + e[1].upper() for e in active_agg.columns.tolist()])\n    bureau_agg = bureau_agg.join(active_agg, how='left', on=COLUMN_ID)\n\n    closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]\n    closed_agg = closed.groupby(COLUMN_ID).agg(num_aggregations)\n    closed_agg.columns = pd.Index(['CLOSED_' + e[0] + \"_\" + e[1].upper() for e in closed_agg.columns.tolist()])\n    bureau_agg = bureau_agg.join(closed_agg, how='left', on=COLUMN_ID)\n\n    return bureau_agg","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.164564Z","start_time":"2021-04-28T09:27:58.154524Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.029177Z","iopub.execute_input":"2021-10-18T10:16:05.029650Z","iopub.status.idle":"2021-10-18T10:16:05.044036Z","shell.execute_reply.started":"2021-10-18T10:16:05.029606Z","shell.execute_reply":"2021-10-18T10:16:05.042809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_encoded(df, nan_as_category = True):\n    original_columns = list(df.columns)\n    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n    df = pd.get_dummies(df, columns = categorical_columns, dummy_na = nan_as_category)\n    new_columns = [c for c in df.columns if c not in original_columns]\n    return df, new_columns","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.183775Z","start_time":"2021-04-28T09:27:58.166492Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.045555Z","iopub.execute_input":"2021-10-18T10:16:05.045876Z","iopub.status.idle":"2021-10-18T10:16:05.063726Z","shell.execute_reply.started":"2021-10-18T10:16:05.045837Z","shell.execute_reply":"2021-10-18T10:16:05.062237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_scores(folds_scores):\n    print(f\"Score by each fold: {folds_scores}\")\n    print(f\"Mean score by each fold:{np.mean(folds_scores):.5f}\")\n    print(\"*\" * 50)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.191762Z","start_time":"2021-04-28T09:27:58.185786Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.065545Z","iopub.execute_input":"2021-10-18T10:16:05.065960Z","iopub.status.idle":"2021-10-18T10:16:05.076322Z","shell.execute_reply.started":"2021-10-18T10:16:05.065926Z","shell.execute_reply":"2021-10-18T10:16:05.074530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lightgbm_cross_validation(params, X, y, cv, categorical = None):\n\n    estimators, folds_scores = [], []\n    \n    if not categorical:\n        categorical = \"auto\"\n\n    oof_preds = np.zeros(X.shape[0])\n    print(f\"{time.ctime()}, Cross-Validation, {X.shape[0]} rows, {X.shape[1]} cols\")\n\n    for fold, (train_idx, valid_idx) in enumerate(cv.split(X, y)):\n        x_train, x_valid = X.loc[train_idx], X.loc[valid_idx]\n        y_train, y_valid = y[train_idx], y[valid_idx]\n        \n        model = lgb.LGBMClassifier(**params)\n        model.fit(\n            x_train, y_train,\n            eval_set=[(x_valid, y_valid)],\n            eval_metric=\"auc\", \n            verbose=500, \n            early_stopping_rounds=STOPPING_ROUNDS\n        )\n        \n        oof_preds[valid_idx] = model.predict_proba(x_valid)[:, 1]\n        score = roc_auc_score(y_valid, oof_preds[valid_idx])\n        folds_scores.append(round(score, 5))\n        print(f\"Fold {fold + 1}, Valid score = {score:.5f}\")\n        estimators.append(model)\n\n    print_scores(folds_scores)\n    return estimators, oof_preds","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.201739Z","start_time":"2021-04-28T09:27:58.193747Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.077893Z","iopub.execute_input":"2021-10-18T10:16:05.078495Z","iopub.status.idle":"2021-10-18T10:16:05.089737Z","shell.execute_reply.started":"2021-10-18T10:16:05.078423Z","shell.execute_reply":"2021-10-18T10:16:05.088362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mean_scores(scores, target):\n    scores_list = [\n        (scores.mean(axis=1), 'AMean score: '),\n        (gmean(scores, axis=1), 'GMean score: '),\n        (scores.rank().mean(axis=1), 'Rank  score: '),\n        (gmean(scores.rank(), axis=1), 'GMean  rank: ')\n    ]\n\n    for scores_mean in scores_list:\n        score = roc_auc_score(target, scores_mean[0])\n        print(f\"{scores_mean[1]}{score:.5f}\")","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.220686Z","start_time":"2021-04-28T09:27:58.20278Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.093278Z","iopub.execute_input":"2021-10-18T10:16:05.093777Z","iopub.status.idle":"2021-10-18T10:16:05.107221Z","shell.execute_reply.started":"2021-10-18T10:16:05.093723Z","shell.execute_reply":"2021-10-18T10:16:05.105864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cross_validation(model, X, y, cv):\n    estimators, folds_scores = [], []\n    oof_preds = np.zeros(X.shape[0])\n\n    for fold, (train_idx, valid_idx) in enumerate(cv.split(X, y)):\n\n        x_train, x_valid = X.loc[train_idx], X.loc[valid_idx]\n        y_train, y_valid = y[train_idx], y[valid_idx]\n\n        model.fit(X=x_train, y=y_train)\n    \n        oof_preds[valid_idx] = model.predict_proba(x_valid)[:, 1]\n        score = roc_auc_score(y_valid, oof_preds[valid_idx])\n        print(f\"Fold {fold+1}, Valid score = {score:.5f}\")\n        folds_scores.append(round(score, 5))\n        estimators.append(model)\n\n    print_scores(folds_scores)\n    return estimators, oof_preds, np.mean(folds_scores)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:58.229057Z","start_time":"2021-04-28T09:27:58.223073Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.109519Z","iopub.execute_input":"2021-10-18T10:16:05.109935Z","iopub.status.idle":"2021-10-18T10:16:05.124115Z","shell.execute_reply.started":"2021-10-18T10:16:05.109886Z","shell.execute_reply":"2021-10-18T10:16:05.122721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_SPLITS = 5\nSTOPPING_ROUNDS = 50\nN_ESTIMATORS = 1000\nRANDOM = 42\nTARGET = 'TARGET'\nCOLUMN_ID = 'APPLICATION_NUMBER'\nPATH = '/kaggle/input/geekbrains-competitive-data-analysis/'\n\nraw_train = pd.read_csv(PATH + 'train.csv')\nraw_test = pd.read_csv(PATH + 'test.csv')\nraw_cp = pd.read_csv(PATH + 'client_profile.csv')\n\ntrain_index = raw_train[COLUMN_ID]\ntest_index = raw_test[COLUMN_ID]\n\ntrain_cp = raw_train.merge(raw_cp, on=COLUMN_ID, how='left')\ntest_cp = raw_test.merge(raw_cp, on=COLUMN_ID, how='left')\n\ntarget = train_cp[TARGET]\n\ntrain_df = train_cp.drop([COLUMN_ID, TARGET], axis=1)\ntest_df = test_cp.drop([COLUMN_ID], axis=1)\n\ntrain = profiling(train_df)\ntest = profiling(test_df)\n\ntrain['NEW_STATUS_FUnknown'] = 0\ntrain['NEW_STATUS_XNACivil marriage'] = 0\ntest['NEW_STATUS_MUnknown'] = 0\n\ncols = train.columns.to_list()\ntest = test[cols]","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:59.793555Z","start_time":"2021-04-28T09:27:58.23205Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:05.125847Z","iopub.execute_input":"2021-10-18T10:16:05.126185Z","iopub.status.idle":"2021-10-18T10:16:07.935700Z","shell.execute_reply.started":"2021-10-18T10:16:05.126155Z","shell.execute_reply":"2021-10-18T10:16:07.934876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_strategy = KFold(n_splits=N_SPLITS, random_state=RANDOM, shuffle=True)\n\nn_estimators_lgb = N_ESTIMATORS","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:27:59.80096Z","start_time":"2021-04-28T09:27:59.795577Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:07.936874Z","iopub.execute_input":"2021-10-18T10:16:07.937378Z","iopub.status.idle":"2021-10-18T10:16:07.941012Z","shell.execute_reply.started":"2021-10-18T10:16:07.937313Z","shell.execute_reply":"2021-10-18T10:16:07.940303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_params = {\n    'metric':\"auc\", \n    'boosting_type': 'gbdt',\n    'n_estimators': n_estimators_lgb,\n    'learning_rate': 0.03,\n    'num_leaves': 42,\n    'max_depth': 10,\n    'reg_alpha': 0.05,\n    'reg_lambda': 0.06,\n    'subsample': 0.8,\n    'min_child_weight': 60,\n    'random_state': RANDOM\n}\n\nestimators_lgbm, oof_preds_lgbm = lightgbm_cross_validation(\n    lgbm_params, train, target, cv=cv_strategy)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:28:13.778221Z","start_time":"2021-04-28T09:27:59.803605Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:07.943018Z","iopub.execute_input":"2021-10-18T10:16:07.943386Z","iopub.status.idle":"2021-10-18T10:16:32.373289Z","shell.execute_reply.started":"2021-10-18T10:16:07.943354Z","shell.execute_reply":"2021-10-18T10:16:32.372455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_params1 = {\n    'metric':\"auc\", \n    'boosting_type': 'gbdt',\n    'n_estimators': n_estimators_lgb,\n    'learning_rate': 0.02,\n    'max_depth': 8,\n    'random_state': RANDOM\n}\n\nestimators_lgbm1, oof_score_lgbm1 = lightgbm_cross_validation(\n    lgbm_params1, train, target, cv=cv_strategy)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:28:36.45221Z","start_time":"2021-04-28T09:28:13.779218Z"},"execution":{"iopub.status.busy":"2021-10-18T10:16:32.374727Z","iopub.execute_input":"2021-10-18T10:16:32.375303Z","iopub.status.idle":"2021-10-18T10:17:02.197776Z","shell.execute_reply.started":"2021-10-18T10:16:32.375261Z","shell.execute_reply":"2021-10-18T10:17:02.196904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_params2 = {\n    'metric':\"auc\", \n    'boosting_type': 'gbdt',\n    'n_estimators': n_estimators_lgb,\n    'learning_rate': 0.05,\n    'num_leaves': 32,\n    'max_depth': 5,\n    'min_child_weight': 42,\n    'random_state': RANDOM\n}\n\nestimators_lgbm2, oof_preds_lgbm2 = lightgbm_cross_validation(\n    lgbm_params2, train, target, cv=cv_strategy)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:28:49.949349Z","start_time":"2021-04-28T09:28:36.454205Z"},"execution":{"iopub.status.busy":"2021-10-18T10:17:02.200118Z","iopub.execute_input":"2021-10-18T10:17:02.200503Z","iopub.status.idle":"2021-10-18T10:17:18.915121Z","shell.execute_reply.started":"2021-10-18T10:17:02.200469Z","shell.execute_reply":"2021-10-18T10:17:18.914028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = pd.DataFrame({\n    \"lgbm\": oof_preds_lgbm,\n    \"lgbm1\": oof_score_lgbm1,\n    \"lgbm2\": oof_preds_lgbm2 \n})\n\nprediction_lgbm1 = np.zeros(test.shape[0])\nfor model in estimators_lgbm1:\n    prediction_lgbm1 += model.predict_proba(test)[:, 1]\nprediction_lgbm1 = prediction_lgbm1/N_SPLITS\n\nprediction_lgbm = np.zeros(test.shape[0])\nfor model in estimators_lgbm:\n    prediction_lgbm += model.predict_proba(test)[:, 1]\nprediction_lgbm = prediction_lgbm/N_SPLITS\n\nprediction_lgbm2 = np.zeros(test.shape[0])\nfor model in estimators_lgbm2:\n    prediction_lgbm2 += model.predict_proba(test)[:, 1]\nprediction_lgbm2 = prediction_lgbm2/N_SPLITS\n\nscores_preds = pd.DataFrame({\n    \"lgbm\": prediction_lgbm,\n    \"lgbm1\": prediction_lgbm1,\n    \"lgbm2\": prediction_lgbm2  \n})\n\nmean_scores(scores, target)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:29:17.688395Z","start_time":"2021-04-28T09:28:49.951344Z"},"scrolled":true,"execution":{"iopub.status.busy":"2021-10-18T10:17:18.917506Z","iopub.execute_input":"2021-10-18T10:17:18.917963Z","iopub.status.idle":"2021-10-18T10:17:44.217254Z","shell.execute_reply.started":"2021-10-18T10:17:18.917915Z","shell.execute_reply":"2021-10-18T10:17:44.215763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lda = LinearDiscriminantAnalysis(\n    solver='eigen',\n    shrinkage='auto',\n    store_covariance=True,\n    n_components=1\n)\n\nestimators_lda, oof_preds_lda, mean_score = cross_validation(model=lda,\n                                                                X=scores,\n                                                                y=target,\n                                                                cv=cv_strategy)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:29:18.32431Z","start_time":"2021-04-28T09:29:17.691382Z"},"execution":{"iopub.status.busy":"2021-10-18T10:17:44.219082Z","iopub.execute_input":"2021-10-18T10:17:44.219404Z","iopub.status.idle":"2021-10-18T10:17:45.044241Z","shell.execute_reply.started":"2021-10-18T10:17:44.219372Z","shell.execute_reply":"2021-10-18T10:17:45.043078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_lda = np.zeros(test.shape[0])\nfor model in estimators_lda:\n    prediction_lda += model.predict_proba(scores_preds)[:, 1]\nprediction_lda = prediction_lda/len(estimators_lda)\n\nsubmission_chek = pd.DataFrame(\n    data={COLUMN_ID: test_index,\n          TARGET: prediction_lda})\nsubmission_chek.head()","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:29:18.377128Z","start_time":"2021-04-28T09:29:18.326222Z"},"scrolled":true,"execution":{"iopub.status.busy":"2021-10-18T10:17:45.045982Z","iopub.execute_input":"2021-10-18T10:17:45.046659Z","iopub.status.idle":"2021-10-18T10:17:45.142006Z","shell.execute_reply.started":"2021-10-18T10:17:45.046613Z","shell.execute_reply":"2021-10-18T10:17:45.140986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"alert alert-block alert-info\">\n    <h3>Second TURN</h3>\n    </div>","metadata":{"ExecuteTime":{"end_time":"2021-04-11T17:47:30.307508Z","start_time":"2021-04-11T17:47:30.297533Z"}}},{"cell_type":"code","source":"train_df = train_cp.drop([COLUMN_ID, TARGET], axis=1)\ntest_df = test_cp.drop([COLUMN_ID], axis=1)\n\ntrain = create_client_profile_features(train_df)\ntest = create_client_profile_features(test_df)\n\ncols = train.columns.to_list()\ntest = test[cols]\n\nestimators_lgbm, oof_preds_lgbm = lightgbm_cross_validation(\n    lgbm_params, train, target, cv=cv_strategy)\n\nestimators_lgbm1, oof_score_lgbm1 = lightgbm_cross_validation(\n    lgbm_params1, train, target, cv=cv_strategy)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:29:55.717196Z","start_time":"2021-04-28T09:29:18.379126Z"},"execution":{"iopub.status.busy":"2021-10-18T10:17:45.143487Z","iopub.execute_input":"2021-10-18T10:17:45.144153Z","iopub.status.idle":"2021-10-18T10:18:32.053859Z","shell.execute_reply.started":"2021-10-18T10:17:45.144108Z","shell.execute_reply":"2021-10-18T10:18:32.052510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores['lgbm3'] = oof_preds_lgbm\nscores['lgbm4'] = oof_score_lgbm1\n\ncorr = scores.corr()\n\nfig, axes = plt.subplots(1, 1, figsize=(8, 8))\nsns.heatmap(corr, annot=True, fmt=\".4g\", square=True, cmap=\"coolwarm\", ax=axes)\n\nscores.head()","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:29:56.341528Z","start_time":"2021-04-28T09:29:55.722185Z"},"execution":{"iopub.status.busy":"2021-10-18T10:18:32.055668Z","iopub.execute_input":"2021-10-18T10:18:32.055993Z","iopub.status.idle":"2021-10-18T10:18:32.423499Z","shell.execute_reply.started":"2021-10-18T10:18:32.055960Z","shell.execute_reply":"2021-10-18T10:18:32.422490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_lgbm1 = np.zeros(test.shape[0])\nfor model in estimators_lgbm1:\n    prediction_lgbm1 += model.predict_proba(test)[:, 1]\nprediction_lgbm1 = prediction_lgbm1/N_SPLITS\n\nprediction_lgbm = np.zeros(test.shape[0])\nfor model in estimators_lgbm:\n    prediction_lgbm += model.predict_proba(test)[:, 1]\nprediction_lgbm = prediction_lgbm/N_SPLITS\n\nscores_preds['lgbm3'] = prediction_lgbm\nscores_preds['lgbm4'] = prediction_lgbm1\n\ncorr = scores_preds.corr()\n\nfig, axes = plt.subplots(1, 1, figsize=(8, 8))\nsns.heatmap(corr, annot=True, fmt=\".4g\", square=True, cmap=\"coolwarm\", ax=axes)\n\nscores_preds.head()","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:30:13.160575Z","start_time":"2021-04-28T09:29:56.34452Z"},"execution":{"iopub.status.busy":"2021-10-18T10:18:32.424874Z","iopub.execute_input":"2021-10-18T10:18:32.425180Z","iopub.status.idle":"2021-10-18T10:18:46.594107Z","shell.execute_reply.started":"2021-10-18T10:18:32.425147Z","shell.execute_reply":"2021-10-18T10:18:46.593214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_scores(scores, target)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:30:13.677189Z","start_time":"2021-04-28T09:30:13.164522Z"},"scrolled":true,"execution":{"iopub.status.busy":"2021-10-18T10:18:46.595748Z","iopub.execute_input":"2021-10-18T10:18:46.596462Z","iopub.status.idle":"2021-10-18T10:18:46.945172Z","shell.execute_reply.started":"2021-10-18T10:18:46.596396Z","shell.execute_reply":"2021-10-18T10:18:46.944215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lda = LinearDiscriminantAnalysis(\n    solver='eigen',\n    shrinkage='auto',\n    store_covariance=True,\n    n_components=1\n)\n\nestimators_lda, oof_preds_lda, mean_score = cross_validation(model=lda,\n                                                                X=scores,\n                                                                y=target,\n                                                                cv=cv_strategy)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:30:14.283528Z","start_time":"2021-04-28T09:30:13.680143Z"},"execution":{"iopub.status.busy":"2021-10-18T10:18:46.946561Z","iopub.execute_input":"2021-10-18T10:18:46.947021Z","iopub.status.idle":"2021-10-18T10:18:47.859790Z","shell.execute_reply.started":"2021-10-18T10:18:46.946987Z","shell.execute_reply":"2021-10-18T10:18:47.858607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_lda_base = np.zeros(test.shape[0])\nfor model in estimators_lda:\n    prediction_lda_base += model.predict_proba(scores_preds)[:, 1]\nprediction_lda_base = prediction_lda_base/len(estimators_lda)\n\nsubmission_base = pd.DataFrame(\n    data={COLUMN_ID: test_index,\n          TARGET: prediction_lda_base})\nsubmission_base.head()","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:30:14.340376Z","start_time":"2021-04-28T09:30:14.284525Z"},"execution":{"iopub.status.busy":"2021-10-18T10:18:47.864992Z","iopub.execute_input":"2021-10-18T10:18:47.865852Z","iopub.status.idle":"2021-10-18T10:18:47.955649Z","shell.execute_reply.started":"2021-10-18T10:18:47.865794Z","shell.execute_reply":"2021-10-18T10:18:47.954413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"alert alert-block alert-info\">\n    <h3>BKI</h3>\n    </div>","metadata":{"ExecuteTime":{"end_time":"2021-04-11T17:47:30.307508Z","start_time":"2021-04-11T17:47:30.297533Z"}}},{"cell_type":"code","source":"train = profiling(train_cp)\ntest = profiling(test_cp)\n\ntrain['NEW_STATUS_FUnknown'] = 0\ntrain['NEW_STATUS_XNACivil marriage'] = 0\ntest['NEW_STATUS_MUnknown'] = 0\n\nbureau = bki()\nprint(\"Bureau df shape:\", bureau.shape)\ntrain = train.join(bureau, how='left', on=COLUMN_ID)\ntest = test.join(bureau, how='left', on=COLUMN_ID)\n\ntrain = train.drop([COLUMN_ID, TARGET], axis=1)\ntest = test.drop([COLUMN_ID], axis=1)\n\ncols = train.columns.to_list()\ntest = test[cols]","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:30:18.2774Z","start_time":"2021-04-28T09:30:14.342371Z"},"execution":{"iopub.status.busy":"2021-10-18T10:18:47.957716Z","iopub.execute_input":"2021-10-18T10:18:47.958190Z","iopub.status.idle":"2021-10-18T10:18:53.495914Z","shell.execute_reply.started":"2021-10-18T10:18:47.958143Z","shell.execute_reply":"2021-10-18T10:18:53.494749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"estimators_lgbm1, oof_score_lgbm1 = lightgbm_cross_validation(\n    lgbm_params1, train, target, cv=cv_strategy)\n\nestimators_lgbm, oof_preds_lgbm = lightgbm_cross_validation(\n    lgbm_params, train, target, cv=cv_strategy)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:31:05.41027Z","start_time":"2021-04-28T09:30:18.279352Z"},"execution":{"iopub.status.busy":"2021-10-18T10:18:53.497602Z","iopub.execute_input":"2021-10-18T10:18:53.498139Z","iopub.status.idle":"2021-10-18T10:19:55.685275Z","shell.execute_reply.started":"2021-10-18T10:18:53.498092Z","shell.execute_reply":"2021-10-18T10:19:55.684211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores['lgbm3'] = oof_preds_lgbm\nscores['lgbm4'] = oof_score_lgbm1\n\nprediction_lgbm1 = np.zeros(test.shape[0])\nfor model in estimators_lgbm1:\n    prediction_lgbm1 += model.predict_proba(test)[:, 1]\nprediction_lgbm1 = prediction_lgbm1/N_SPLITS\n\nprediction_lgbm = np.zeros(test.shape[0])\nfor model in estimators_lgbm:\n    prediction_lgbm += model.predict_proba(test)[:, 1]\nprediction_lgbm = prediction_lgbm/N_SPLITS\n\nscores_preds['lgbm3'] = prediction_lgbm\nscores_preds['lgbm4'] = prediction_lgbm1","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:31:24.3137Z","start_time":"2021-04-28T09:31:05.412265Z"},"execution":{"iopub.status.busy":"2021-10-18T10:19:55.686601Z","iopub.execute_input":"2021-10-18T10:19:55.687093Z","iopub.status.idle":"2021-10-18T10:20:13.917740Z","shell.execute_reply.started":"2021-10-18T10:19:55.687047Z","shell.execute_reply":"2021-10-18T10:20:13.916623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lda = LinearDiscriminantAnalysis(\n    solver='eigen',\n    shrinkage='auto',\n    store_covariance=True,\n    n_components=1\n)\n\nestimators_lda, oof_preds_lda, mean_score = cross_validation(model=lda,\n                                                                X=scores,\n                                                                y=target,\n                                                                cv=cv_strategy)","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:31:25.040979Z","start_time":"2021-04-28T09:31:24.320683Z"},"execution":{"iopub.status.busy":"2021-10-18T10:20:13.919103Z","iopub.execute_input":"2021-10-18T10:20:13.919718Z","iopub.status.idle":"2021-10-18T10:20:14.830048Z","shell.execute_reply.started":"2021-10-18T10:20:13.919676Z","shell.execute_reply":"2021-10-18T10:20:14.828312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction_lda = np.zeros(test.shape[0])\nfor model in estimators_lda:\n    prediction_lda += model.predict_proba(scores_preds)[:, 1]\nprediction_lda = prediction_lda/len(estimators_lda)\n\nsubmission_bki = pd.DataFrame(\n    data={COLUMN_ID: test_index,\n          TARGET: prediction_lda})\nsubmission_bki.head()","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:31:25.094834Z","start_time":"2021-04-28T09:31:25.042973Z"},"execution":{"iopub.status.busy":"2021-10-18T10:20:14.831924Z","iopub.execute_input":"2021-10-18T10:20:14.832607Z","iopub.status.idle":"2021-10-18T10:20:14.914281Z","shell.execute_reply.started":"2021-10-18T10:20:14.832550Z","shell.execute_reply":"2021-10-18T10:20:14.913074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_join = submission_bki.copy()\ncp_index = raw_cp[COLUMN_ID].to_list()\n\nsubmission_join.loc[submission_join[COLUMN_ID].isin(\n    cp_index)] = submission_base.loc[submission_base[COLUMN_ID].isin(cp_index)]\n\nsubmission_join.to_csv('submission.csv', index= False)\nsubmission_join.head()","metadata":{"ExecuteTime":{"end_time":"2021-04-28T09:31:25.595403Z","start_time":"2021-04-28T09:31:25.09683Z"},"execution":{"iopub.status.busy":"2021-10-18T10:20:14.916245Z","iopub.execute_input":"2021-10-18T10:20:14.917029Z","iopub.status.idle":"2021-10-18T10:20:15.843558Z","shell.execute_reply.started":"2021-10-18T10:20:14.916981Z","shell.execute_reply":"2021-10-18T10:20:15.842193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}