{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import os\nfrom collections import defaultdict\nfrom itertools import count, product\nimport random\nimport warnings\n\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport xgboost as xgb\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer, KNNImputer, SimpleImputer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import (\n    MinMaxScaler,\n    OneHotEncoder,\n    OrdinalEncoder,\n    StandardScaler,\n)\nfrom sklearnex import patch_sklearn\n\nsns.set_theme()\nsns.set_style(\"ticks\")\nsns.despine()\n\npatch_sklearn()\n\nwarnings.filterwarnings(\"ignore\")\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-15T21:27:34.045988Z","iopub.execute_input":"2022-11-15T21:27:34.047129Z","iopub.status.idle":"2022-11-15T21:27:34.062764Z","shell.execute_reply.started":"2022-11-15T21:27:34.047085Z","shell.execute_reply":"2022-11-15T21:27:34.061494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything():\n    np.random.seed(42)\n    random.seed(42)\n    os.environ[\"PYTHONHASHSEED\"] = str(42)\n    \nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T21:27:34.065507Z","iopub.execute_input":"2022-11-15T21:27:34.066191Z","iopub.status.idle":"2022-11-15T21:27:34.077809Z","shell.execute_reply.started":"2022-11-15T21:27:34.066150Z","shell.execute_reply":"2022-11-15T21:27:34.076603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"class Config:\n    DATA_DIR = \"../input/home-loan-prediction-prepared-datasets\"\n\n    LOG_FILE = \"./log_file.csv\"\n\n    MODEL_CLASSES = {\n        \"lr\": LogisticRegression,\n        \"ada\": AdaBoostClassifier,\n        \"rf\": RandomForestClassifier,\n        \"xgb\": xgb.XGBClassifier,\n        \"lgb\": lgb.LGBMClassifier,\n    }\n\n    MODEL_NAMES = {\n        \"lr\": \"Logistic Regression\",\n        \"ada\": \"AdaBoost\",\n        \"rf\": \"Random Forest\",\n        \"xgb\": \"XGBoost\",\n        \"lgb\": \"LightGBM\",\n    }\n\n    @classmethod\n    def get_filepath(cls, filename):\n        return os.path.join(cls.DATA_DIR, filename)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-15T21:27:34.078905Z","iopub.execute_input":"2022-11-15T21:27:34.079252Z","iopub.status.idle":"2022-11-15T21:27:34.089771Z","shell.execute_reply.started":"2022-11-15T21:27:34.079224Z","shell.execute_reply":"2022-11-15T21:27:34.088690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions to Build Experiments","metadata":{}},{"cell_type":"code","source":"def get_imputer(name, impute_strategy= \"median\"):\n    return {\n        \"simple\": SimpleImputer(strategy=impute_strategy),\n        \"iterative\": IterativeImputer(max_iter=100),\n    }.get(name)\n\n\ndef get_scaler(name):\n    return {\n        \"standard\": StandardScaler(),\n        \"minmax\": MinMaxScaler(),\n    }.get(name)\n\n\ndef get_encoder(name):\n    return {\n        \"ohe\": OneHotEncoder(handle_unknown=\"ignore\", drop=\"if_binary\"),\n        \"label\": OrdinalEncoder(handle_unknown=\"use_encoded_value\", unknown_value=-1),\n    }.get(name)\n\n\ndef make_pipeline(\n    numeric_features,\n    categorical_features,\n    impute_method,\n    impute_strategy,\n    scale_method,\n    encode_method,\n    estimator,\n    estimator_params=None,\n):\n    imputer = get_imputer(name=impute_method, impute_strategy=impute_strategy)\n    encoder = get_encoder(name=encode_method)\n    scaler = get_scaler(name=scale_method)\n\n    numeric_transformer = Pipeline(steps=[(\"imputer\", imputer), (\"scaler\", scaler)])\n    categorical_transformer = Pipeline(\n        steps=[\n            (\"imputer\", SimpleImputer(strategy=\"most_frequent\")),\n            (\"encoder\", encoder),\n        ]\n    )\n\n    preprocessor = ColumnTransformer(\n        transformers=[\n            (\"numeric\", numeric_transformer, numeric_features),\n            (\"categorical\", categorical_transformer, categorical_features),\n        ]\n    )\n\n    if estimator_params is None:\n        estimator_params = {}\n\n    model_cls = Config.MODEL_CLASSES.get(estimator)\n\n    if model_cls is None:\n        valid = \", \".join(Config.MODEL_CLASSES.keys())\n        raise ValueError(f\"Unrecognized model. Should be one of {valid}\")\n\n    model = model_cls(**estimator_params)\n\n    pipeline = Pipeline(steps=[(\"preprocessor\", preprocessor), (\"model\", model)])\n\n    return pipeline","metadata":{"execution":{"iopub.status.busy":"2022-11-15T21:27:34.092331Z","iopub.execute_input":"2022-11-15T21:27:34.093420Z","iopub.status.idle":"2022-11-15T21:27:34.106761Z","shell.execute_reply.started":"2022-11-15T21:27:34.093381Z","shell.execute_reply":"2022-11-15T21:27:34.105629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment Class","metadata":{}},{"cell_type":"code","source":"class Experiment:\n    _id = count(0)\n\n    def __init__(\n        self,\n        name=\"exp\",\n        imputer=\"simple\",\n        impute_strategy=None,\n        scaler=\"minmax\",\n        encoder=\"label\",\n        estimator=\"lr\",\n    ) -> None:\n        if estimator not in Config.MODEL_CLASSES:\n            valid = \", \".join(Config.MODEL_CLASSES.keys())\n            raise ValueError(\n                f\"Unrecognized estimator {estimator}. Should be one of {valid}.\"\n            )\n            \n        if imputer == \"simple\" and impute_strategy is None:\n            raise ValuError(f\"impute_strategy cannot be none when using {imputer} imputer.\")\n\n        exp_id = next(self._id)\n\n        self.exp_id = exp_id\n        self.name = f\"{name}_{exp_id}\"\n        self.imputer = imputer\n        self.impute_strategy = impute_strategy\n        self.scaler = scaler\n        self.encoder = encoder\n        self.estimator = estimator\n        self._experiment_details = None\n\n        self.train_scores = []\n        self.val_scores = []\n        self.avg_train_score = []\n        self.avg_val_score = []\n\n    @property\n    def experiment_details(self):\n        if self._experiment_details is None:\n            details = [f\"Imputer: {self.imputer}\"]\n\n            if self.imputer == \"simple\":\n                details.append(f\"Impute Strategy: {self.impute_strategy}\")\n\n            details.extend(\n                [\n                    f\"Scaler: {self.scaler}\",\n                    f\"Encoder: {self.encoder}\",\n                    f\"Estimator: {Config.MODEL_NAMES.get(self.estimator)}\",\n                ]\n            )\n\n            self._experiment_details = details\n\n        return \"\\n\".join(self._experiment_details)\n\n    def run(\n        self,\n        df,\n        numeric_features,\n        categorical_features,\n        estimator_params=None,\n        fit_params=None,\n        n_folds=5,\n        *,\n        verbose=True,\n    ):\n        print(f\"Running experiment {self.name}\")\n\n        if verbose is True:\n            print(self.experiment_details)\n\n        if fit_params is None:\n            fit_params = {}\n\n        tot_train_score = tot_val_score = 0\n\n        train_scores = []\n        val_scores = []\n\n        for fold in range(n_folds):\n            train = df[df[\"kfold\"] != fold]\n            X_train = train.drop([\"target\", \"kfold\"], axis=1)\n            y_train = train[\"target\"].to_numpy()\n\n            val = df[df[\"kfold\"] == fold]\n            X_valid = val.drop([\"target\", \"kfold\"], axis=1)\n            y_valid = val[\"target\"].to_numpy()\n\n            pipeline = make_pipeline(\n                numeric_features=numeric_features,\n                categorical_features=categorical_features,\n                impute_method=self.imputer,\n                impute_strategy=self.impute_strategy,\n                scale_method=self.scaler,\n                encode_method=self.encoder,\n                estimator=self.estimator,\n                estimator_params=estimator_params,\n            )\n\n            pipeline.fit(X=X_train, y=y_train, **fit_params)\n\n            train_pred = pipeline.predict(X=X_train)\n            train_score = f1_score(y_train, train_pred, average=\"macro\")\n\n            val_pred = pipeline.predict(X=X_valid)\n            val_score = f1_score(y_valid, val_pred, average=\"macro\")\n\n            if verbose is True:\n                print(\n                    f\"Fold {fold + 1} - Train F1 = {train_score: .4f}; Valid F1 = {val_score: .4f}\"\n                )\n\n            tot_train_score += train_score\n            tot_val_score += val_score\n\n            train_scores.append(train_score)\n            val_scores.append(val_score)\n\n        avg_train_score = tot_train_score / n_folds\n        avg_val_score = tot_val_score / n_folds\n\n        if verbose is True:\n            print(f\"Overall train F1: {avg_train_score: .4f}\")\n            print(f\"Overall validation F1: {avg_val_score: .4f}\")\n\n        self.avg_train_score = avg_train_score\n        self.avg_val_score = avg_val_score\n        self.train_scores = train_scores\n        self.val_scores = val_scores\n\n        return self","metadata":{"execution":{"iopub.status.busy":"2022-11-15T21:27:34.108197Z","iopub.execute_input":"2022-11-15T21:27:34.108876Z","iopub.status.idle":"2022-11-15T21:27:34.130898Z","shell.execute_reply.started":"2022-11-15T21:27:34.108827Z","shell.execute_reply":"2022-11-15T21:27:34.129703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Datasets","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(Config.get_filepath(\"train_with_new_features.csv\"))\ndf = df.drop(\"sk_id_curr\", axis=1)\n\nnum_feats = pd.read_csv(Config.get_filepath(\"numerical_features.csv\"))\nnum_feats = num_feats[\"column\"].to_list()\n\ncat_feats = pd.read_csv(Config.get_filepath(\"catgeorical_features.csv\"))\ncat_feats = cat_feats[\"column\"].to_list()","metadata":{"execution":{"iopub.status.busy":"2022-11-15T21:27:34.232825Z","iopub.execute_input":"2022-11-15T21:27:34.235224Z","iopub.status.idle":"2022-11-15T21:27:36.246829Z","shell.execute_reply.started":"2022-11-15T21:27:34.235186Z","shell.execute_reply":"2022-11-15T21:27:36.245898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions to Obtain Model Parameters","metadata":{}},{"cell_type":"code","source":"def lr_params():\n    return {\"max_iter\": 10000}\n\n\ndef ada_params():\n    return {}\n\n\ndef rf_params():\n    return {\"max_depth\": 7}\n\ndef xgb_params():\n    return {\"tree_method\": \"gpu_hist\"}\n\ndef lgb_params():\n    return {}\n\n\nPARAM_MAP = {\n    \"lr\": lr_params,\n    \"ada\": ada_params,\n    \"rf\": rf_params,\n    \"xgb\": xgb_params,\n    \"lgb\": lgb_params,\n}","metadata":{"execution":{"iopub.status.busy":"2022-11-15T21:27:36.248893Z","iopub.execute_input":"2022-11-15T21:27:36.249339Z","iopub.status.idle":"2022-11-15T21:27:36.256395Z","shell.execute_reply.started":"2022-11-15T21:27:36.249302Z","shell.execute_reply":"2022-11-15T21:27:36.255228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Experiments","metadata":{}},{"cell_type":"code","source":"imputers = [\"simple,mean\", \"simple,median\", \"iterative\"]\nscalers = [\"minmax\", \"standard\"]\nencoders = [\"label\", \"ohe\"]\nestimators = Config.MODEL_CLASSES.keys()\n\nexperiments = product(imputers, scalers, encoders, estimators)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T21:27:36.258027Z","iopub.execute_input":"2022-11-15T21:27:36.258447Z","iopub.status.idle":"2022-11-15T21:27:36.267593Z","shell.execute_reply.started":"2022-11-15T21:27:36.258411Z","shell.execute_reply":"2022-11-15T21:27:36.266530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment Loop","metadata":{}},{"cell_type":"code","source":"log = defaultdict(list)\n\nfor i, (imputer_params, scaler, encoder, estimator) in enumerate(experiments):\n    imp_comps = imputer_params.split(\",\")\n    imputer = imp_comps[0]\n    \n    imp_strategy = None if len(imp_comps) == 1 else imp_comps[1]\n    \n    exp = Experiment(\n        imputer=imputer,\n        impute_strategy=imp_strategy,\n        scaler=scaler,\n        encoder=encoder,\n        estimator=estimator,\n    )\n\n    estimator_params = PARAM_MAP.get(estimator)()\n    \n    exp.run(df=df, numeric_features=num_feats, categorical_features=cat_feats, estimator_params=estimator_params)\n\n    log[\"exp_id\"].append(i)\n    log[\"imputer\"].append(imputer_params)\n    log[\"scaler\"].append(scaler)\n    log[\"encoder\"].append(encoder)\n    log[\"estimator\"].append(estimator)\n    \n    for fold in range(5):\n        log[f\"train_score_fold{fold + 1}\"].append(exp.train_scores[fold])\n        log[f\"val_score_fold{fold + 1}\"].append(exp.val_scores[fold])\n\n    log[\"avg_train_score\"].append(exp.avg_train_score)\n    log[\"avg_val_score\"].append(exp.avg_val_score)\n    \n    print(\"\\n\")\n\nlog_df = pd.DataFrame(data=log)\nlog_df.to_csv(Config.LOG_FILE, index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-15T21:27:36.269368Z","iopub.execute_input":"2022-11-15T21:27:36.269744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}