{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LightGBM GPU Support","metadata":{"_kg_hide-input":true,"_kg_hide-output":true}},{"cell_type":"code","source":"!pip uninstall -y lightgbm\n!apt-get install -y libboost-all-dev\n!git clone --recursive https://github.com/Microsoft/LightGBM","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-12-02T14:06:21.975667Z","iopub.execute_input":"2022-12-02T14:06:21.976158Z","iopub.status.idle":"2022-12-02T14:06:58.770001Z","shell.execute_reply.started":"2022-12-02T14:06:21.976045Z","shell.execute_reply":"2022-12-02T14:06:58.768836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%bash\ncd LightGBM\nrm -r build\nmkdir build\ncd build\ncmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/ ..\nmake -j$(nproc)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-12-02T14:06:58.773616Z","iopub.execute_input":"2022-12-02T14:06:58.773932Z","iopub.status.idle":"2022-12-02T14:10:07.311205Z","shell.execute_reply.started":"2022-12-02T14:06:58.773899Z","shell.execute_reply":"2022-12-02T14:10:07.310014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cd LightGBM/python-package/;python setup.py install --precompile","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-12-02T14:10:07.313477Z","iopub.execute_input":"2022-12-02T14:10:07.313852Z","iopub.status.idle":"2022-12-02T14:10:09.179019Z","shell.execute_reply.started":"2022-12-02T14:10:07.313809Z","shell.execute_reply":"2022-12-02T14:10:09.177800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /etc/OpenCL/vendors && echo \"libnvidia-opencl.so.1\" > /etc/OpenCL/vendors/nvidia.icd\n!rm -r LightGBM","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-12-02T14:10:09.182329Z","iopub.execute_input":"2022-12-02T14:10:09.182939Z","iopub.status.idle":"2022-12-02T14:10:11.275406Z","shell.execute_reply.started":"2022-12-02T14:10:09.182895Z","shell.execute_reply":"2022-12-02T14:10:11.273930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import functools\nimport os\nimport random\nimport warnings\n\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport optuna\nimport pandas as pd\nimport seaborn as sns\nimport xgboost as xgb\nfrom sklearn import metrics\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer, SimpleImputer\nfrom sklearn.preprocessing import MinMaxScaler, OneHotEncoder, OrdinalEncoder\nfrom sklearnex import patch_sklearn\n\npatch_sklearn()\nwarnings.filterwarnings(\"ignore\")\n\nsns.set_theme()\nsns.set_style(\"ticks\")\nsns.despine()\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-02T14:10:11.277587Z","iopub.execute_input":"2022-12-02T14:10:11.278262Z","iopub.status.idle":"2022-12-02T14:10:14.337595Z","shell.execute_reply.started":"2022-12-02T14:10:11.278216Z","shell.execute_reply":"2022-12-02T14:10:14.336472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    \nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:14.339043Z","iopub.execute_input":"2022-12-02T14:10:14.340352Z","iopub.status.idle":"2022-12-02T14:10:14.346194Z","shell.execute_reply.started":"2022-12-02T14:10:14.340314Z","shell.execute_reply":"2022-12-02T14:10:14.344907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# General Configuration","metadata":{}},{"cell_type":"code","source":"class Config:\n    DATA_DIR = \"../input/home-loan-prediction-prepared-datasets\"\n    \n    MODEL_TRIALS = 100\n    ENSEMBLE_TRIALS = 100\n    \n    MODEL_NAMES = {\n        \"xgb\": \"XGBoost\",\n        \"lgb\": \"LightGBM\",\n    }\n    \n    MODEL_CLASSES = {\n        \"xgb\": xgb.XGBClassifier,\n        \"lgb\": lgb.LGBMClassifier,\n    }\n    \n    @classmethod\n    def filepath(cls, filename):\n        return os.path.join(cls.DATA_DIR, filename)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:14.347457Z","iopub.execute_input":"2022-12-02T14:10:14.348308Z","iopub.status.idle":"2022-12-02T14:10:14.358551Z","shell.execute_reply.started":"2022-12-02T14:10:14.348272Z","shell.execute_reply":"2022-12-02T14:10:14.357494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration for Models","metadata":{}},{"cell_type":"code","source":"class XGBConfig:\n    EVAL_METRIC = \"logloss\"\n    \n    DEFAULT_VALUES = {\n        \"max_depth\": 6,\n        \"n_estimators\": 100,\n        \"alpha\": 0.0,\n        \"lambda\": 1.0,\n        \"learning_rate\": 0.3,\n        \"colsample_bytree\": 1.0,\n        \"colsample_bylevel\": 1.0,\n        \"min_child_weight\": 1.0,\n        \"sampling_method\": \"uniform\",\n        \"early_stopping_rounds\": None,\n    }\n    \n    STATIC_PARAMS = {\n        \"tree_method\": \"gpu_hist\",\n        \"use_label_encoder\": False,\n        \"n_jobs\": -1,\n        \"predictor\": \"gpu_predictor\",\n        \"max_bin\": 1024,\n        \"eval_metric\": EVAL_METRIC,\n        \"enable_categorical\": True,\n    }\n    \n    USE_PRUNER = True\n    \n    @classmethod\n    def get_fit_params(cls, X_train, y_train, X_val, y_val, params):\n        return {\n            \"eval_set\": [(X_train, y_train), (X_val, y_val)],\n            \"verbose\": False,\n        }","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:14.371343Z","iopub.execute_input":"2022-12-02T14:10:14.371727Z","iopub.status.idle":"2022-12-02T14:10:14.383772Z","shell.execute_reply.started":"2022-12-02T14:10:14.371692Z","shell.execute_reply":"2022-12-02T14:10:14.382824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LGBConfig:\n    DEFAULT_VALUES = {\n        \"num_leaves\": 31,\n        \"max_depth\": -1,\n        \"learning_rate\": 0.1,\n        \"n_estimators\": 100,\n        \"reg_alpha\": 0.0,\n        \"reg_lambda\": 0.0,\n        \"min_child_samples\": 20,\n        \"subsample_for_bin\": 200000,\n    }\n    \n    STATIC_PARAMS = {\n        \"n_jobs\": -1,\n        \"verbose\": -1,\n        \"objective\": \"binary\",\n        \"device\": \"gpu\",\n    }\n    \n    USE_PRUNER = True\n    \n    @classmethod\n    def get_fit_params(cls, X_train, y_train, X_val, y_val, params):\n        # To suppress training output\n        callbacks = params.get(\"callbacks\", []) + [lgb.log_evaluation(period=0)]\n        return {\n            \"eval_set\": [(X_train, y_train), (X_val, y_val)],\n            \"eval_metric\": [\"logloss\", \"auc\"],\n            \"callbacks\": callbacks,\n        }","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:14.388502Z","iopub.execute_input":"2022-12-02T14:10:14.388759Z","iopub.status.idle":"2022-12-02T14:10:14.397966Z","shell.execute_reply.started":"2022-12-02T14:10:14.388735Z","shell.execute_reply":"2022-12-02T14:10:14.396987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG_MAP = {\n    \"xgb\": XGBConfig,\n    \"lgb\": LGBConfig,\n}","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:14.399262Z","iopub.execute_input":"2022-12-02T14:10:14.399555Z","iopub.status.idle":"2022-12-02T14:10:14.410290Z","shell.execute_reply.started":"2022-12-02T14:10:14.399530Z","shell.execute_reply":"2022-12-02T14:10:14.409328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Dataset","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(Config.filepath(\"train_with_new_features.csv\"))\ntest_df = pd.read_csv(Config.filepath(\"test_with_new_features.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:14.411610Z","iopub.execute_input":"2022-12-02T14:10:14.412396Z","iopub.status.idle":"2022-12-02T14:10:20.957320Z","shell.execute_reply.started":"2022-12-02T14:10:14.412363Z","shell.execute_reply":"2022-12-02T14:10:20.956298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:20.958924Z","iopub.execute_input":"2022-12-02T14:10:20.959324Z","iopub.status.idle":"2022-12-02T14:10:20.995012Z","shell.execute_reply.started":"2022-12-02T14:10:20.959278Z","shell.execute_reply":"2022-12-02T14:10:20.993925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:20.996583Z","iopub.execute_input":"2022-12-02T14:10:20.996966Z","iopub.status.idle":"2022-12-02T14:10:21.025049Z","shell.execute_reply.started":"2022-12-02T14:10:20.996920Z","shell.execute_reply":"2022-12-02T14:10:21.024076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Loop","metadata":{}},{"cell_type":"code","source":"def train(df, test_df, model, params, verbose=True):\n    df = df.copy()\n    df[\"preds\"] = pd.NA\n    \n    drop = [\"target\", \"preds\", \"kfold\"]\n    \n    # Get the initializer class and the configuration class\n    klass = Config.MODEL_CLASSES[model]\n    config = CONFIG_MAP[model]\n   \n    # Add static params - Parameters that are not tuned\n    params.update(config.STATIC_PARAMS)\n    \n    # For storing total accuracy across folds for averaging\n    total_f1 = 0.0\n    \n    # Empty list for storing test predictions in each fold\n    test_preds = []\n    \n    for fold in range(5):\n        train = df[df[\"kfold\"] != fold]\n        \n        # Get training features and labels\n        y_train = train[\"target\"]\n        X_train = train.drop(drop, axis=1)\n        \n        val = df[df[\"kfold\"] == fold]\n        \n        # Get validation features and labels\n        y_val = val[\"target\"]\n        X_val = val.drop(drop, axis=1)\n        \n        # Initialize model\n        clf = klass(**params)\n        \n        # Get parameters for .fit() other than X and y\n        fit_params = config.get_fit_params(\n            X_train=X_train,\n            y_train=y_train,\n            X_val=X_val,\n            y_val=y_val,\n            params=params\n        )\n        \n        # Train model on training set\n        clf.fit(\n            X=X_train,\n            y=y_train,\n            **fit_params,\n        )\n        \n        # Make predictions on validation set\n        val_pred = clf.predict(X_val)\n        f1 = metrics.f1_score(y_val, val_pred, average=\"macro\")\n        \n        # Report accuracy if verbose is True\n        if verbose is True:\n            print(f\"\\tFold {fold + 1} - Macro F1 = {f1: .4f}\")\n        \n        # Add to total accuracy\n        total_f1 += f1\n        \n        # Make predictions on validation set again\n        # But this time in terms of probabilities\n        # And store in the df\n        # These will be used in the meta model\n        df.loc[val.index, \"preds\"] = clf.predict_proba(X_val)[:, 1]\n        \n        # Get the test predictions for this fold in terms of probability\n        test_preds.append(clf.predict_proba(test_df)[:, 1])\n        \n    avg_f1 = total_f1 / 5\n    \n    if verbose is True:\n        print(f\"\\tOverall Macro F1 = {avg_f1: .4f}\")   \n    \n    # Calculate final test predictions\n    # These will be used in the meta model\n    test_preds = np.vstack(test_preds)\n    test_preds = test_preds.mean(axis=0)\n    \n    # Return val preds, test preds and overall accuracy\n    return df[\"preds\"].values, test_preds, avg_f1","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:21.027362Z","iopub.execute_input":"2022-12-02T14:10:21.027706Z","iopub.status.idle":"2022-12-02T14:10:21.039881Z","shell.execute_reply.started":"2022-12-02T14:10:21.027672Z","shell.execute_reply":"2022-12-02T14:10:21.038989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optuna Objectives","metadata":{}},{"cell_type":"code","source":"def xgb_objective(trial, train_df, test_df):\n    params = {\n        \"max_depth\": trial.suggest_int(\"max_depth\", 1, 11),\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 5, 500),\n        \"alpha\": trial.suggest_uniform(\"alpha\", 0.0, 5.0),\n        \"lambda\": trial.suggest_float(\"lambda\", 1.0, 5.0, log=True),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.03, 0.8, log=True),\n        \"scale_pos_weight\": trial.suggest_float(\"scale_pos_weight\", 1, 20, log=True),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.2, 1.0),\n        \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.2, 1.0),\n        \"min_child_weight\": trial.suggest_uniform(\"min_child_weight\", 1, 100),\n        \"sampling_method\": trial.suggest_categorical(\"sampling_method\", [\"uniform\", \"gradient_based\"]),\n        \"early_stopping_rounds\": trial.suggest_int(\"early_stopping_rounds\", 5, 20, step=5)\n    }\n    \n    obs_k = f\"validation_1-{XGBConfig.EVAL_METRIC}\"\n    params[\"callbacks\"] = [optuna.integration.XGBoostPruningCallback(trial, obs_k)]\n    \n    _, _, f1 = train(df=train_df, test_df=test_df, model=\"xgb\", params=params, verbose=False)\n    return f1","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:21.055053Z","iopub.execute_input":"2022-12-02T14:10:21.055790Z","iopub.status.idle":"2022-12-02T14:10:21.066805Z","shell.execute_reply.started":"2022-12-02T14:10:21.055744Z","shell.execute_reply":"2022-12-02T14:10:21.065518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lgb_objective(trial, train_df, test_df):\n    params = {\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 31, 100, log=True),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 1, 100, log=True),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-4, 1.0),\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 100, 500),\n        \"scale_pos_weight\": trial.suggest_float(\"scale_pos_weight\", 1, 20, log=True),\n        \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 0.0, 5.0),\n        \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0.0, 5.0),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 20, 50, log=True),\n        \"subsample_for_bin\": trial.suggest_int(\"subsample_for_bin\", 2000, 8000),\n    }\n    \n    params[\"callbacks\"] = [optuna.integration.LightGBMPruningCallback(trial, \"auc\", \"valid_1\")]\n    \n    _, _, f1 = train(df=train_df, test_df=test_df, model=\"lgb\", params=params, verbose=False)\n    return f1","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:21.069037Z","iopub.execute_input":"2022-12-02T14:10:21.069752Z","iopub.status.idle":"2022-12-02T14:10:21.078707Z","shell.execute_reply.started":"2022-12-02T14:10:21.069718Z","shell.execute_reply":"2022-12-02T14:10:21.077635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OBJECTIVE_MAP = {\n    \"xgb\": xgb_objective,\n    \"lgb\": lgb_objective,\n}","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:21.080324Z","iopub.execute_input":"2022-12-02T14:10:21.080712Z","iopub.status.idle":"2022-12-02T14:10:21.092138Z","shell.execute_reply.started":"2022-12-02T14:10:21.080676Z","shell.execute_reply":"2022-12-02T14:10:21.091126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameter Search","metadata":{}},{"cell_type":"code","source":"def hyperparameter_search(train_df, test_df, model, n_trials=Config.MODEL_TRIALS, verbose=True):\n    if verbose is False:\n        v = optuna.logging.get_verbosity()\n        optuna.logging.set_verbosity(optuna.logging.WARNING)\n    \n    objective = OBJECTIVE_MAP[model]\n    objective = functools.partial(objective, train_df=train_df, test_df=test_df)\n    \n    # Check if pruning is required\n    pruner = optuna.pruners.HyperbandPruner() if CONFIG_MAP[model].USE_PRUNER is True else None\n    \n    sampler = optuna.samplers.TPESampler(seed=42)\n    study = optuna.create_study(\n        direction=\"maximize\",\n        pruner=pruner,\n        sampler=sampler,\n    )\n    \n    study.optimize(objective, n_trials=n_trials, n_jobs=2, gc_after_trial=True)\n    \n    if verbose is False:\n        optuna.logging.set_verbosity(v)\n\n    return study.best_params","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:21.093526Z","iopub.execute_input":"2022-12-02T14:10:21.093971Z","iopub.status.idle":"2022-12-02T14:10:21.103612Z","shell.execute_reply.started":"2022-12-02T14:10:21.093936Z","shell.execute_reply":"2022-12-02T14:10:21.102737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"cat_feats = pd.read_csv(Config.filepath(\"categorical_features.csv\"))\ncat_feats = cat_feats[\"column\"].to_list()\n\nnum_feats = pd.read_csv(Config.filepath(\"numerical_features.csv\"))\nnum_feats = num_feats[\"column\"].to_list()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:21.105000Z","iopub.execute_input":"2022-12-02T14:10:21.105501Z","iopub.status.idle":"2022-12-02T14:10:21.133461Z","shell.execute_reply.started":"2022-12-02T14:10:21.105464Z","shell.execute_reply":"2022-12-02T14:10:21.132480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imputer = IterativeImputer(max_iter=100)\nscaler = MinMaxScaler()\ncat_imputer = SimpleImputer(strategy=\"most_frequent\")\nencoder = OrdinalEncoder(handle_unknown=\"use_encoded_value\", unknown_value=-1)\n\ntrain_df[num_feats] = imputer.fit_transform(train_df[num_feats])\ntrain_df[num_feats] = scaler.fit_transform(train_df[num_feats])\ntrain_df[cat_feats] = cat_imputer.fit_transform(train_df[cat_feats])\ntrain_df[cat_feats] = encoder.fit_transform(train_df[cat_feats])\ntrain_df[cat_feats] = train_df[cat_feats].astype(\"category\")","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:10:21.136646Z","iopub.execute_input":"2022-12-02T14:10:21.136906Z","iopub.status.idle":"2022-12-02T14:11:42.750661Z","shell.execute_reply.started":"2022-12-02T14:10:21.136882Z","shell.execute_reply":"2022-12-02T14:11:42.749553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[num_feats] = imputer.transform(test_df[num_feats])\ntest_df[num_feats] = scaler.transform(test_df[num_feats])\ntest_df[cat_feats] = cat_imputer.transform(test_df[cat_feats])\ntest_df[cat_feats] = encoder.transform(test_df[cat_feats])\ntest_df[cat_feats] = test_df[cat_feats].astype(\"category\")","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:11:42.752382Z","iopub.execute_input":"2022-12-02T14:11:42.752774Z","iopub.status.idle":"2022-12-02T14:11:55.070376Z","shell.execute_reply.started":"2022-12-02T14:11:42.752735Z","shell.execute_reply":"2022-12-02T14:11:55.069361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop(\"sk_id_curr\", axis=1)\n\ntest_sk_id = test_df[\"sk_id_curr\"].to_numpy()\ntest_df = test_df.drop(\"sk_id_curr\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:11:55.071959Z","iopub.execute_input":"2022-12-02T14:11:55.072349Z","iopub.status.idle":"2022-12-02T14:11:55.127783Z","shell.execute_reply.started":"2022-12-02T14:11:55.072301Z","shell.execute_reply":"2022-12-02T14:11:55.126655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:11:55.129230Z","iopub.execute_input":"2022-12-02T14:11:55.130133Z","iopub.status.idle":"2022-12-02T14:11:55.165387Z","shell.execute_reply.started":"2022-12-02T14:11:55.130076Z","shell.execute_reply":"2022-12-02T14:11:55.164485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-02T14:11:55.166895Z","iopub.execute_input":"2022-12-02T14:11:55.167366Z","iopub.status.idle":"2022-12-02T14:11:55.202485Z","shell.execute_reply.started":"2022-12-02T14:11:55.167331Z","shell.execute_reply":"2022-12-02T14:11:55.201435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Traininig Individual Models","metadata":{}},{"cell_type":"code","source":"models = [\"xgb\", \"lgb\"]\n\ntrain_meta_dict = {}\ntest_meta_dict = {}\n\nfor model in models:\n    print(f\"{Config.MODEL_NAMES[model]}:\")\n            \n    print(\"\\tFinding optimal hyperparameters using Optuna...\")\n    params = hyperparameter_search(\n        train_df=train_df, test_df=test_df, model=model, verbose=False\n    )\n\n    print(f\"\\n\\tBest params: {params}\\n\")\n\n    print(\"\\tTraining model with optimal parameters...\\n\")\n    val_preds, test_preds, acc = train(\n        df=train_df,\n        test_df=test_df,\n        model=model,\n        params=params\n    )\n    \n    train_meta_dict[f\"{model}_preds\"] = val_preds\n    test_meta_dict[f\"{model}_preds\"] = test_preds\n\n    print(\"\\tDone!\\n\")","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-12-02T14:11:55.204042Z","iopub.execute_input":"2022-12-02T14:11:55.204422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Dataset for Ensemble","metadata":{}},{"cell_type":"code","source":"train_meta_df = pd.DataFrame(data=train_meta_dict, dtype=np.float64)\ntest_meta_df = pd.DataFrame(data=test_meta_dict, dtype=np.float64)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_columns = [\n    \"ext_source_1\",\n    \"ext_source_2\",\n    \"ext_source_3\",\n    \"amt_income_total\",\n    \"amt_credit\",\n    \"loan_rate\",\n    \"loan_income_ratio\",\n    \"es1_is_missing\",\n    \"es3_is_missing\",\n    \"application_is_incomplete\",\n]\n\ntrain_meta_df[extra_columns] = train_df[extra_columns]\ntrain_meta_df[[\"target\", \"kfold\"]] = train_df[[\"target\", \"kfold\"]]\n\ntest_meta_df[extra_columns] = test_df[extra_columns]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_meta_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train XGBoost as Ensemble Model","metadata":{}},{"cell_type":"code","source":"params = hyperparameter_search(\n    train_df=train_meta_df,\n    test_df=test_meta_df,\n    model=\"xgb\",\n    n_trials=Config.ENSEMBLE_TRIALS,\n)","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds, test_preds, acc = train(\n    df=train_meta_df,\n    test_df=test_meta_df,\n    model=\"xgb\",\n    params=params\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\"SK_ID_CURR\": test_sk_id, \"TARGET\": (test_preds >= 0.5).astype(int)})\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"./submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}