{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":82611,"databundleVersionId":9553358,"sourceType":"competition"}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **IMPORTS**","metadata":{}},{"cell_type":"code","source":"%%writefile requirements.txt\n\nlightgbm==4.5.0\nxgboost==2.1.1\nscikit-learn==1.4.2","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:19:03.161394Z","iopub.execute_input":"2024-09-29T08:19:03.162004Z","iopub.status.idle":"2024-09-29T08:19:03.199252Z","shell.execute_reply.started":"2024-09-29T08:19:03.161948Z","shell.execute_reply":"2024-09-29T08:19:03.197989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture \n\n!pip install -r requirements.txt","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:19:03.201780Z","iopub.execute_input":"2024-09-29T08:19:03.202209Z","iopub.status.idle":"2024-09-29T08:19:44.458066Z","shell.execute_reply.started":"2024-09-29T08:19:03.202166Z","shell.execute_reply":"2024-09-29T08:19:44.455951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile -a myimports.py\n\nprint(f\"\\n---> Commencing imports\")\n\nfrom gc import collect\nfrom warnings import filterwarnings\nfilterwarnings('ignore')\nfrom IPython.display import display_html, clear_output\nclear_output()\nimport os, sys, logging, re, joblib, ctypes, shutil\nfrom copy import deepcopy\n\n# General library imports:-\nfrom os import path, walk, getpid\nfrom psutil import Process\nfrom collections import Counter\nfrom itertools import product\nimport ctypes\nlibc = ctypes.CDLL(\"libc.so.6\")\n\nfrom IPython.display import display_html, clear_output\nfrom pprint import pprint\nfrom functools import partial\nfrom copy import deepcopy\nimport pandas as pd, numpy as np\nfrom scipy.optimize import minimize\nfrom numpy.typing import ArrayLike, NDArray\nimport polars as pl\nimport polars.selectors as cs\nfrom polars.testing import assert_frame_equal\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom colorama import Fore, Style, init\nfrom tqdm.notebook import tqdm","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:19:44.460364Z","iopub.execute_input":"2024-09-29T08:19:44.460959Z","iopub.status.idle":"2024-09-29T08:19:44.472389Z","shell.execute_reply.started":"2024-09-29T08:19:44.460891Z","shell.execute_reply":"2024-09-29T08:19:44.470731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile -a myimports.py\n\n# Importing model and pipeline specifics:-\nfrom category_encoders import OrdinalEncoder, OneHotEncoder\n\n# Pipeline specifics:-\nfrom sklearn.preprocessing import (RobustScaler,\n                                   MinMaxScaler,\n                                   StandardScaler,\n                                   FunctionTransformer as FT,\n                                   PowerTransformer,\n                                  )\nfrom sklearn.impute import SimpleImputer as SI\nfrom sklearn.model_selection import (RepeatedStratifiedKFold as RSKF,\n                                     StratifiedKFold as SKF,\n                                     StratifiedGroupKFold as SGKF,\n                                     KFold,\n                                     GroupKFold as GKF,\n                                     RepeatedKFold as RKF,\n                                     PredefinedSplit as PDS,\n                                     cross_val_score,\n                                     cross_val_predict,\n                                    )\nfrom sklearn.inspection import permutation_importance\nfrom sklearn.feature_selection import VarianceThreshold as VT\nfrom sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.base import (BaseEstimator, TransformerMixin, RegressorMixin, clone)\nfrom sklearn.compose import ColumnTransformer, make_column_selector\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import (root_mean_squared_error as rmse, \n                             mean_absolute_error as mae,\n                            )\nfrom sklearn.ensemble import (RandomForestRegressor as RFR,\n                              ExtraTreesRegressor as ETR,\n                              HistGradientBoostingRegressor as HGBR,\n                             )\n\n# Importing model packages\nimport xgboost as xgb, lightgbm as lgb\nfrom xgboost import QuantileDMatrix, XGBRegressor as XGBR\nfrom lightgbm import log_evaluation, early_stopping, LGBMRegressor as LGBMR\nfrom catboost import CatBoostRegressor as CBR, Pool\n\n# Importing Ensemble and tuning packages\nimport optuna\nfrom optuna import Trial, trial, create_study\nfrom optuna.pruners import HyperbandPruner\nfrom optuna.samplers import TPESampler, CmaEsSampler\noptuna.logging.disable_default_handler()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:19:44.475700Z","iopub.execute_input":"2024-09-29T08:19:44.476475Z","iopub.status.idle":"2024-09-29T08:19:44.489615Z","shell.execute_reply.started":"2024-09-29T08:19:44.476423Z","shell.execute_reply":"2024-09-29T08:19:44.488186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile -a myimports.py\n\n# Setting rc parameters in seaborn for plots and graphs-\nsns.set({\"axes.facecolor\"       : \"#ffffff\",\n         \"figure.facecolor\"     : \"#ffffff\",\n         \"axes.edgecolor\"       : \"#000000\",\n         \"grid.color\"           : \"#ffffff\",\n         \"font.family\"          : ['Cambria'],\n         \"axes.labelcolor\"      : \"#000000\",\n         \"xtick.color\"          : \"#000000\",\n         \"ytick.color\"          : \"#000000\",\n         \"grid.linewidth\"       : 0.75,\n         \"grid.linestyle\"       : \"--\",\n         \"axes.titlecolor\"      : '#0099e6',\n         'axes.titlesize'       : 8.5,\n         'axes.labelweight'     : \"bold\",\n         'legend.fontsize'      : 7.0,\n         'legend.title_fontsize': 7.0,\n         'font.size'            : 7.5,\n         'xtick.labelsize'      : 7.5,\n         'ytick.labelsize'      : 7.5,\n        }\n       )\n\n# Color printing\ndef PrintColor(text: str, color = Fore.BLUE, style = Style.BRIGHT):\n    \"Prints color outputs using colorama using a text F-string\"\n    print(style + color + text + Style.RESET_ALL)\n \n# Checking package versions\nimport xgboost as xgb, lightgbm as lgb, catboost as cb, sklearn as sk\nprint(f\"---> XGBoost = {xgb.__version__} | LightGBM = {lgb.__version__} | Catboost = {cb.__version__}\")\nprint(f\"---> Sklearn = {sk.__version__}| Pandas = {pd.__version__} | Polars = {pl.__version__}\")\ncollect()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:19:44.491786Z","iopub.execute_input":"2024-09-29T08:19:44.492922Z","iopub.status.idle":"2024-09-29T08:19:44.511237Z","shell.execute_reply.started":"2024-09-29T08:19:44.492860Z","shell.execute_reply":"2024-09-29T08:19:44.509757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile -a myimports.py\n\nclass MyLogger:\n    \"\"\"\n    This class helps to suppress logs in lightgbm and Optuna\n    Source - https://github.com/microsoft/LightGBM/issues/6014\n    \"\"\"\n\n    def init(self, logging_lbl: str):\n        self.logger = logging.getLogger(logging_lbl)\n        self.logger.setLevel(logging.ERROR)\n\n    def info(self, message):\n        pass\n\n    def warning(self, message):\n        pass\n\n    def error(self, message):\n        self.logger.error(message)\n        \n        \n# Customizing logging for XGBoost\nfor handler in logging.root.handlers[:]:\n    logging.root.removeHandler(handler)\n\nlogger = logging.getLogger(__name__)\nlogger.setLevel(logging.ERROR)\nformatter = logging.Formatter('%(asctime)s | %(levelname)s | %(message)s')\n\nstdout_handler = logging.StreamHandler(sys.stdout)\nstdout_handler.setLevel(logging.INFO)\nstdout_handler.setFormatter(formatter)\n\nfile_handler = logging.FileHandler(f'xgb_optimize.log')\nfile_handler.setLevel(logging.ERROR)\nfile_handler.setFormatter(formatter)\n\nlogger.addHandler(file_handler)\nlogger.addHandler(stdout_handler)\n\nclass XGBLogging(xgb.callback.TrainingCallback):\n    \"\"\"\n    This class designs the custom logging for XGboost \n    This is to be used inside the XGboost callback\n    \"\"\"\n\n    def __init__(self, epoch_log_interval=100):\n        self.epoch_log_interval = epoch_log_interval\n\n    def after_iteration(\n        self, model, epoch:int, evals_log:xgb.callback.TrainingCallback.EvalsLog\n    ):\n        if self.epoch_log_interval <= 0:\n            pass\n        \n        elif (epoch %  self.epoch_log_interval == 0):\n            for data, metric in evals_log.items():\n                for metric_name, log in metric.items():\n                    score = log[-1][0] if isinstance(log[-1], tuple) else log[-1]\n                    logger.info(f\"XGBLogging epoch {epoch} dataset {data} {metric_name} {score}\")\n\n        return False","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:19:44.512949Z","iopub.execute_input":"2024-09-29T08:19:44.513401Z","iopub.status.idle":"2024-09-29T08:19:44.528897Z","shell.execute_reply.started":"2024-09-29T08:19:44.513355Z","shell.execute_reply":"2024-09-29T08:19:44.527400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nexec(open('myimports.py','r').read())\nprint()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T08:19:44.530918Z","iopub.execute_input":"2024-09-29T08:19:44.531462Z","iopub.status.idle":"2024-09-29T08:19:49.408799Z","shell.execute_reply.started":"2024-09-29T08:19:44.531411Z","shell.execute_reply":"2024-09-29T08:19:49.407226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **APPROACH DETAILS**","metadata":{}},{"cell_type":"markdown","source":"This is a baseline kernel for the community competition [here](https://www.kaggle.com/competitions/brist1d) <br>\nThis is a tabular regression problem with the RMSE metric. This needs to be minimized <br>\n\nThis kernel is based on the simple preprocesssing done [here](https://www.kaggle.com/code/sierra022/blood-glucose-prediction) and my ML model framework. Let's add more models as we move along and refine the process! <br>\n\n### **CV STRATEGY** <br>\nWe use a group k-fold strategy here, with the patient number being the grouper column. We also resort to a 5-fold split <br>\n\n### **INCORPORATING COMMUNITY FEEDBACK** <br>\nThis training class is my rejoinder to the community feedback. My previously designed trainer class became a bit clunky and needed revision. Hopefully, this one is better to read and more nimble. Hope it serves the purpose of the user! <br>\n\nAll the best!","metadata":{}},{"cell_type":"code","source":"%%writefile -a training.py\n\n# Configuration class:-\nclass CFG:\n    \"\"\"\n    Configuration class for parameters and CV strategy for tuning and training\n    Some parameters may be unused here as this is a general configuration class\n    \"\"\";\n\n    # Data preparation:-\n    version_nb  = 1\n    model_id    = \"V1_2\"\n    model_label = \"ML\"\n\n    test_req           = False\n    test_sample_frac   = 1000\n\n    gpu_switch         = \"OFF\"\n    state              = 42\n    target             = f'bg+1:00'\n    grouper            = f'p_num'\n\n    ip_path            = f\"/kaggle/input/brist1d\"\n    op_path            = f\"/kaggle/working\"\n\n    # Model Training:-\n    pstprcs_oof        = False\n    pstprcs_train      = False\n    pstprcs_test       = False\n    ML                 = True\n    test_preds_req     = True\n\n    pseudo_lbl_req     = False\n    pseudolbl_up       = 0.975\n    pseudolbl_low      = 0.00\n\n    n_splits           = 3 if test_req == \"Y\" else 5\n    n_repeats          = 1\n    nbrnd_erly_stp     = 100\n    mdlcv_mthd         = 'GKF'\n\n    # Ensemble:-\n    ensemble_req       = True\n    metric_obj         = 'minimize'\n\n    # Global variables for plotting:-\n    grid_specs = {'visible'  : True,\n                  'which'    : 'both',\n                  'linestyle': '--',\n                  'color'    : 'lightgrey',\n                  'linewidth': 0.75\n                 }\n\n    title_specs = {'fontsize'   : 9,\n                   'fontweight' : 'bold',\n                   'color'      : '#992600',\n                  }\n    \ncv_selector = \\\n{\n \"RKF\"   : RKF(n_splits = CFG.n_splits, n_repeats= CFG.n_repeats, random_state= CFG.state),\n \"RSKF\"  : RSKF(n_splits = CFG.n_splits, n_repeats= CFG.n_repeats, random_state= CFG.state),\n \"SKF\"   : SKF(n_splits = CFG.n_splits, shuffle = True, random_state= CFG.state),\n \"KF\"    : KFold(n_splits = CFG.n_splits, shuffle = True, random_state= CFG.state),\n \"GKF\"   : GKF(n_splits = CFG.n_splits)\n}\n\nPrintColor(f\"\\n---> Configuration done!\\n\")\ncollect()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T08:19:49.410482Z","iopub.execute_input":"2024-09-29T08:19:49.411256Z","iopub.status.idle":"2024-09-29T08:19:49.421345Z","shell.execute_reply.started":"2024-09-29T08:19:49.411203Z","shell.execute_reply":"2024-09-29T08:19:49.419539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"|Configuration parameter| Explanation| Data type| Sample values |  \n| ---------------------- | ------------------------------- | --------------------- | --------------- |\n| version_nb    | Version Number | int | 1 | \n| model_id      | Model ID    | string | V1_1 | \n| model_label   | Model Label | string | ML | \n| test_req      | Test Required| bool | True / False | \n| test_sample_frac| Test sampled fraction | int | 1000 |\n| gpu_switch      | Do we need GPU support | bool | True / False |\n| state           | Random state | int | 42 |\n| target          | Target column | str |  |\n| grouper         | CV grouper column | str |  |\n| ip_path, op_path | Data paths  | str | |\n| pstprcs_* | Do we need post-processing  | bool |True / False |\n| ML| Do we need machine learning models  | bool |True / False |\n| test_preds_req| Do we need test set predictions (training in inference kernel)  | bool |True / False |\n| pseudo_lbl_req| Pseudo label required?  | bool |True / False |\n| pseudo_lbl_* | Pseudo label cutoff | float | |\n| n_splits/ n_repeats | N-splits and repeats for CV scheme | int | 3/5/10|\n| nbrnd_erly_stp | Early stopping rounds | int | 40|\n| mdlcv_mthd | Model CV method | str | RSKF|\n| ensemble_req | Do we need ensemble | bool | True / False |\n| optuna_req   | Do we need optuna | bool | True / False |\n| metric_obj   | Metric direction | str | minimize/ maximize |\n| ntrials      | Trials | int | 300 |","metadata":{}},{"cell_type":"code","source":"%%writefile -a training.py\n\nclass Utils:\n    \"\"\"\n    This class creates and uses several utility methods to be used across the code\n    \"\"\";\n\n    def __init__(self):\n        pass\n\n    def ScoreMetric(self, ytrue, ypred)-> float:\n        \"\"\"\n        This method calculates the metric for the competition\n        Inputs- ytrue, ypred:- input truth and predictions\n        Output- float:- competition metric\n        \"\"\"\n        return rmse(ytrue, ypred)\n\n\n    def CleanMemory(self):\n        \"This method cleans the memory off unused objects and displays the cleaned state RAM usage\"\n\n        collect();\n        libc.malloc_trim(0)\n        pid        = getpid()\n        py         = Process(pid)\n        memory_use = py.memory_info()[0] / 2. ** 30\n        return f\"\\nRAM usage = {memory_use :.4} GB\"\n\n    def DisplayAdjTbl(self, *args):\n        \"\"\"\n        This function displays pandas tables in an adjacent manner, sourced from the below link-\n        https://stackoverflow.com/questions/38783027/jupyter-notebook-display-two-pandas-tables-side-by-side\n        \"\"\"\n\n        html_str = ''\n        for df in args:\n            html_str += df.to_html()\n        display_html(html_str.replace('table','table style=\"display:inline\"'),raw=True)\n        collect()\n\n    def DisplayScores(\n        self, Scores: pd.DataFrame, TrainScores: pd.DataFrame, methods: list\n    ):\n        \"This method displays the scores and their means\"\n\n        args = \\\n        [Scores.style.format(precision = 5).\\\n         background_gradient(cmap = \"Blues\", subset = methods + [\"Ensemble\"]).\\\n         set_caption(f\"\\nOOF scores across methods and folds\\n\"),\n\n         TrainScores.style.format(precision = 5).\\\n         background_gradient(cmap = \"Pastel2\", subset = methods).\\\n         set_caption(f\"\\nTrain scores across methods and folds\\n\")\n        ];\n\n        PrintColor(f\"\\n\\n\\n---> OOF score across all methods and folds\\n\",\n                   color = Fore.LIGHTMAGENTA_EX\n                   )\n        self.DisplayAdjTbl(*args)\n\n        print('\\n')\n        display(Scores.mean().to_frame().\\\n                transpose().\\\n                style.format(precision = 5).\\\n                background_gradient(cmap = \"mako\", axis=1,\n                                    subset = Scores.columns\n                                   ).\\\n                set_caption(f\"\\nOOF mean scores across methods and folds\\n\")\n               )\n\n\nutils = Utils()\ncollect()\nprint()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:19:49.423440Z","iopub.execute_input":"2024-09-29T08:19:49.423984Z","iopub.status.idle":"2024-09-29T08:19:49.450277Z","shell.execute_reply.started":"2024-09-29T08:19:49.423933Z","shell.execute_reply":"2024-09-29T08:19:49.448790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **PREPROCESSING**","metadata":{}},{"cell_type":"code","source":"%%writefile -a training.py\n\ndef XformCols(df: pd.DataFrame):\n    \"This function converts the time column into time components and removes special characters in column labels\"\n    \n    df[\"hour_nb\"]   = pd.to_datetime(df['time']).dt.hour\n    df['minute_nb'] = pd.to_datetime(df['time']).dt.minute\n    df.columns      = df.columns.str.replace('\\W', '', regex = True)\n    \n    return df.drop(\"time\", axis=1, errors = \"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-09-29T08:19:49.455519Z","iopub.execute_input":"2024-09-29T08:19:49.457178Z","iopub.status.idle":"2024-09-29T08:19:49.467327Z","shell.execute_reply.started":"2024-09-29T08:19:49.457117Z","shell.execute_reply":"2024-09-29T08:19:49.465436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture \n\nexec(open('training.py','r').read())\n\ntrain    = pd.read_csv(os.path.join(CFG.ip_path, \"train.csv\"), index_col = \"id\", low_memory = False)\ntest     = pd.read_csv(os.path.join(CFG.ip_path, \"test.csv\"), index_col = \"id\", low_memory = False)\nsub_fl   = pd.read_csv(os.path.join(CFG.ip_path, \"sample_submission.csv\"), index_col = \"id\")\n\nif CFG.test_req:\n    train = train.groupby(CFG.grouper).sample(n = CFG.test_sample_frac)   \ntrain.index = range(len(train))\n\n_ = utils.CleanMemory()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:19:49.469610Z","iopub.execute_input":"2024-09-29T08:19:49.470248Z","iopub.status.idle":"2024-09-29T08:20:25.680447Z","shell.execute_reply.started":"2024-09-29T08:19:49.470195Z","shell.execute_reply":"2024-09-29T08:20:25.678760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nPrintColor(f\"---> Shapes = {train.shape} {test.shape} {sub_fl.shape}\")\n\nXtrain   = train.drop([CFG.target, CFG.grouper], axis=1, errors = \"ignore\")\nytrain   = train[CFG.target]\nygrp     = train[CFG.grouper]\nXtest    = test.drop([CFG.target, CFG.grouper], axis=1, errors = \"ignore\")\n\n# Defining the data pipeline\nxform = \\\nPipeline(\n    [(\"XformCols\", FT(XformCols)),\n     (\"Xform\",\n      ColumnTransformer(\n         [(\"SI\", SI(strategy = \"mean\"), make_column_selector(dtype_include = np.number)), \n          (\"OE\", \n           make_pipeline(\n               *[SI(strategy = \"constant\", fill_value = \"missing\"),\n                 OrdinalEncoder(return_df = False)\n                ]\n           ), \n           make_column_selector(pattern = \"activity\")\n          )\n         ],\n         remainder = \"passthrough\", \n         verbose_feature_names_out= False,\n      )\n     )\n    ], verbose = False\n).set_output(transform = \"pandas\")\n\nXtrain = xform.fit_transform(Xtrain, ytrain)\nXtest  = xform.transform(Xtest)\n\nPrintColor(f\"---> Shapes = {Xtrain.shape} {ytrain.shape} {Xtest.shape} {sub_fl.shape}\")\n\n# Initializing CV scheme\ncv = cv_selector[CFG.mdlcv_mthd]\n\n# Initializing CV folds across the training data:-\nfolds = np.zeros(len(Xtrain))\nfor fold_nb, (train_idx, dev_idx) in enumerate(cv.split(Xtrain, ytrain, ygrp)):\n    folds[dev_idx] = fold_nb\nXtrain[\"fold_nb\"] = folds\ndel folds\n\n# Adding a source column for the training algorithm- this has no impact on the model result\nXtrain[\"Source\"] = \"Competition\"\n\nPrintColor(f\"\\nData pipeline structure\\n\", color = Fore.RED)\ndisplay(xform)\n\nPrintColor(f\"---> Shapes = {Xtrain.shape} {ytrain.shape} {Xtest.shape} {sub_fl.shape}\")\nPrintColor(utils.CleanMemory())","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:20:25.682716Z","iopub.execute_input":"2024-09-29T08:20:25.683208Z","iopub.status.idle":"2024-09-29T08:20:27.641830Z","shell.execute_reply.started":"2024-09-29T08:20:25.683161Z","shell.execute_reply":"2024-09-29T08:20:27.638067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **MODEL TRAINING**","metadata":{}},{"cell_type":"code","source":"%%writefile -a training.py\n\nclass ModelTrainer:\n    \"This class trains the provided model on the train-test data and returns the predictions and fitted models\"\n\n    def __init__(\n        self,\n        es_req         : bool  = False,\n        es             : int   = 100,\n        target         : str   = CFG.target,\n        metric_lbl     : str   = \"rmse\",\n        drop_cols      : list  = [\"Source\", \"id\", \"Id\", \"Label\", CFG.target, \"fold_nb\", CFG.grouper],\n    ):\n        \"\"\"\n        Key parameters-\n        es_iter - early stopping rounds for boosted trees\n        \"\"\"\n        \n        drop_cols = list(set(drop_cols + [target]))\n        \n        self.es_req         = es_req\n        self.es_iter        = es\n        self.target         = target\n        self.drop_cols      = drop_cols\n        self.metric_lbl     = metric_lbl\n        \n    def ScoreMetric(self, ytrue, ypred)->float:\n        \"\"\"\n        This is the metric function for the competition scoring\n        \"\"\"\n        return rmse(ytrue, ypred)\n        \n    def PlotFtreImp(\n        self, ftreimp: pd.Series, method: str,\n        ntop: int = 50,\n        title_specs: dict = CFG.title_specs,\n        **params,\n    ):\n        \"This function plots the feature importances for the model provided\"\n\n        print()\n        fig, ax = plt.subplots(1, 1, figsize = (25, 8))\n\n        ftreimp.sort_values(ascending = False).\\\n        head(ntop).\\\n        plot.bar(ax = ax, color = \"blue\")\n        ax.set_title(f\"Feature Importances - {method}\", **title_specs)\n\n        plt.tight_layout()\n        plt.show()\n        print()\n\n    def PostProcessPreds(self, ypred):\n        \"This method post-processes predictions optionally\"\n        return np.clip(ypred, a_min = 0, a_max = np.inf)\n\n    def MakeOfflineModel(\n        self, X, y, ygrp, Xtest, mdl, method,\n        test_preds_req   : bool = True,\n        ftreimp_plot_req : bool = True,\n        ntop             : int  = 50,\n        **params,\n    ):\n        \"\"\"\n        This function trains the provided model on the dataset and cross-validates appropriately\n\n        Inputs-\n        X, y, ygrp       - training data components\n        Xtest            - test data\n        model            - model object for training\n        method           - model method label\n        test_preds_req   - boolean flag to extract test set predictions\n        ftreimp_plot_req - boolean flag to plot tree feature importances\n        ntop             - top n features for feature importances plot\n\n        Returns-\n        oof_preds, test_preds - prediction arrays\n        fitted_models         - fitted model list for test set\n        ftreimp               - feature importances across selected features\n        mdl_best_iter         - model average best iteration across folds\n        \"\"\"\n\n        oof_preds     = np.zeros(len(X))\n        test_preds    = []\n        mdl_best_iter = []\n        ftreimp       = 0\n        \n        scores, tr_scores, fitted_models = [], [], []\n        cv = PDS(ygrp)\n        n_splits = ygrp.nunique()\n\n        for fold_nb, (train_idx, dev_idx) in tqdm(enumerate(cv.split(X, y))):\n            Xtr   = X.iloc[train_idx].drop(\"Source\", axis=1, errors = \"ignore\")\n            Xdev  = X.iloc[dev_idx].query(\"Source == 'Competition'\").drop(\"Source\", axis=1, errors = \"ignore\")\n            ytr   = y.iloc[Xtr.index]\n            ydev  = y.iloc[Xdev.index]\n            model = clone(mdl)\n\n            if \"CB\" in method and self.es_req == True:\n                model.fit(Xtr, ytr,\n                          eval_set = [(Xdev, ydev)],\n                          verbose = 0,\n                          early_stopping_rounds = self.es_iter,\n                          )\n                best_iter = model.get_best_iteration()\n\n            elif \"LGB\" in method and self.es_req == True:\n                model.fit(Xtr, ytr,\n                          eval_set = [(Xdev, ydev)],\n                          callbacks = [log_evaluation(0),\n                                       early_stopping(stopping_rounds = self.es_iter, \n                                                      verbose = False,\n                                                     ),\n                                       ],\n                          )\n                best_iter = model.best_iteration_\n\n            elif \"XGB\" in method and self.es_req == True:\n                model.fit(Xtr, ytr,\n                          eval_set = [(Xdev, ydev)],\n                          verbose  = 0,\n                          )\n                best_iter = model.best_iteration\n\n            else:\n                model.fit(Xtr, ytr)\n                best_iter = -1\n\n            fitted_models.append(model)\n\n            try:\n                ftreimp += model.feature_importances_\n            except:\n                pass\n\n            dev_preds = self.PostProcessPreds(model.predict(Xdev))\n            oof_preds[Xdev.index] = dev_preds\n\n            train_preds  = self.PostProcessPreds(model.predict(Xtr))\n            tr_score     = self.ScoreMetric(ytr.values.flatten(), train_preds)\n            score        = self.ScoreMetric(ydev.values.flatten(), dev_preds)\n\n            scores.append(score)\n            tr_scores.append(tr_score)\n\n            nspace = 15 - len(method) - 2 if fold_nb <= 9 else 15 - len(method) - 1\n            \n            if self.es_req:\n                PrintColor(f\"{method} Fold{fold_nb} {' ' * nspace} OOF = {score:.6f} | Train = {tr_score:.6f} | Iter = {best_iter:,.0f} \")\n            else:\n                PrintColor(f\"{method} Fold{fold_nb} {' ' * nspace} OOF = {score:.6f} | Train = {tr_score:.6f}\")\n                \n            mdl_best_iter.append(best_iter)\n\n            if test_preds_req:\n                test_preds.append(\n                    self.PostProcessPreds(\n                        model.predict(Xtest)\n                    )\n                )\n            else:\n                pass\n\n        test_preds    = np.mean(np.stack(test_preds, axis = 1), axis=1)\n        ftreimp       = pd.Series(ftreimp, index = Xdev.columns)\n        mdl_best_iter = np.uint16(np.amax(mdl_best_iter))\n\n        if ftreimp_plot_req == True and best_iter > 0:\n            print()\n            self.PlotFtreImp(ftreimp, method = method, ntop = ntop,)\n        else:\n            pass\n\n        PrintColor(f\"\\n---> {np.mean(scores):.6f} +- {np.std(scores):.6f} | OOF\", color = Fore.RED)\n        PrintColor(f\"---> {np.mean(tr_scores):.6f} +- {np.std(tr_scores):.6f} | Train\", color = Fore.RED)\n\n        if self.es_req == False:\n            pass\n        else:\n            PrintColor(f\"---> Max best iteration = {mdl_best_iter :,.0f}\", color = Fore.RED)\n\n        return (fitted_models, oof_preds, test_preds, ftreimp, mdl_best_iter)\n\n    def MakeOnlineModel(\n        self, X, y, Xtest, model, method,\n        test_preds_req : bool = False,\n    ):\n        \"This method refits the model on the complete train data and returns the model fitted object and predictions\"\n\n        try:\n            model.early_stopping_rounds = None\n        except:\n            pass\n\n        try:\n            model.fit(X, y, verbose = 0)\n        except:\n            model.fit(X, y,)\n\n        oof_preds  = model.predict(X)\n        if test_preds_req:\n            test_preds = model.predict(Xtest[X.columns])\n        else:\n            test_preds = 0\n        return (model, oof_preds, test_preds)\n\n    def MakeOfflinePreds(self, X, fitted_models):\n        \"This method creates test-set predictions for the offline model provided\"\n\n        test_preds = 0\n        n_splits   = len(fitted_models)\n        PrintColor(f\"---> Number of splits = {n_splits}\")\n\n        for model in fitted_models:\n            test_preds = test_preds + (model.predict(X) / n_splits)\n\n        return test_preds","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-09-29T08:20:27.647607Z","iopub.execute_input":"2024-09-29T08:20:27.649522Z","iopub.status.idle":"2024-09-29T08:20:27.689239Z","shell.execute_reply.started":"2024-09-29T08:20:27.649460Z","shell.execute_reply":"2024-09-29T08:20:27.684706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **SINGLE MODEL TRAINING**","metadata":{}},{"cell_type":"code","source":"%%capture\n\nexec(open('training.py','r').read())\n\ntry:\n    l = MyLogger()\n    l.init(logging_lbl = \"lightgbm_custom\")\n    lgb.register_logger(l)\nexcept:\n    pass\n\n# Initializing model parameters\nMdl_Master = \\\n{\n f'LGBM1R' : LGBMR(**{\"objective\"           : \"regression_l2\",\n                      \"metrics\"             : \"rmse\",\n                      'device'              : \"gpu\" if CFG.gpu_switch == \"ON\" else \"cpu\",\n                      'learning_rate'       : 0.035, \n                      'n_estimators'        : 5_000,\n                      'max_depth'           : 7, \n                      'num_leaves'          : 60, \n                      'min_data_in_leaf'    : 80,\n                      'feature_fraction'    : 0.70, \n                      'bagging_fraction'    : 0.88, \n                      'bagging_freq'        : 6, \n                      'lambda_l1'           : 0.001, \n                      'lambda_l2'           : 0.1,\n                      'verbosity'           : -1,\n                      'random_state'        : CFG.state,\n                     }\n                  ),\n\n f'LGBM2R' : LGBMR(**{\"objective\"           : \"regression_l2\",\n                      \"metrics\"             : \"rmse\",\n                      'device'              : \"gpu\" if CFG.gpu_switch == \"ON\" else \"cpu\",\n                      'learning_rate'       : 0.0325,\n                      'data_sample_strategy': 'goss', \n                      'n_estimators'        : 5_000,\n                      'max_depth'           : 6, \n                      'num_leaves'          : 45, \n                      'min_data_in_leaf'    : 100,\n                      'feature_fraction'    : 0.35, \n                      'colsample_bytree'    : 0.50,\n                      'lambda_l1'           : 0.001, \n                      'lambda_l2'           : 1.25,\n                      'verbosity'           : -1,\n                      'random_state'        : CFG.state,\n                     }\n                  ),\n    \n    \n f'XGB1R' : XGBR(**{  \"objective\"             : \"reg:squarederror\",\n                      \"eval_metric\"           : \"rmse\", \n                      'device'                : \"cuda\" if CFG.gpu_switch == \"ON\" else \"cpu\",\n                      'learning_rate'         : 0.04, \n                      'n_estimators'          : 5_000,\n                      'max_depth'             : 7, \n                      'colsample_bytree'      : 0.75, \n                      'colsample_bynode'      : 0.85,\n                      'colsample_bylevel'     : 0.45,                     \n                      'reg_alpha'             : 0.001, \n                      'reg_lambda'            : 0.25,\n                      'verbose'               : 0,\n                      'random_state'          : CFG.state,\n                      'enable_categorical'    : True,\n                      'callbacks'             : [XGBLogging(epoch_log_interval= 0)],\n                      'early_stopping_rounds' : CFG.nbrnd_erly_stp, \n                     }\n                  ),\n    \n f'CB1R' : CBR(**{'task_type'           : \"CPU\",\n                  'loss_function'       : 'RMSE',\n                  'eval_metric'         : \"RMSE\",\n                  'bagging_temperature' : 0.25,\n                  'colsample_bylevel'   : 0.40,\n                  'iterations'          : 5_000,\n                  'learning_rate'       : 0.04,\n                  'max_depth'           : 7,\n                  'l2_leaf_reg'         : 0.80,\n                  'min_data_in_leaf'    : 70,\n                  'random_strength'     : 0.25,\n                  'verbose'             : 0,\n                 }\n              ),\n}\n\n# Initializing model outputs\nOOF_Preds    = {}\nMdl_Preds    = {}\nFittedModels = {}\nFtreImp      = {}\nSelMdlCols   = {}\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-29T08:22:31.847943Z","iopub.execute_input":"2024-09-29T08:22:31.848573Z","iopub.status.idle":"2024-09-29T08:22:32.452417Z","shell.execute_reply.started":"2024-09-29T08:22:31.848521Z","shell.execute_reply":"2024-09-29T08:22:32.451033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n     \n# Creating CV scheme:-\nmd = ModelTrainer(es         = CFG.nbrnd_erly_stp, \n                  target     = CFG.target,\n                  es_req     = True,\n                  metric_lbl = \"rmse\",\n                 )\n\nfor method, mdl in tqdm(Mdl_Master.items()):\n    PrintColor(f\"\\n{'-' * 10} {method} MODEL TRAINING - {CFG.target} {'-' * 10}\\n\", \n               color = Fore.MAGENTA\n              )\n\n    fitted_models, oof_preds, test_preds, ftreimp, mdl_best_iter =  \\\n    md.MakeOfflineModel(\n        Xtrain.drop([CFG.target, \"fold_nb\", CFG.grouper], axis = 1, errors = \"ignore\"),\n        ytrain,\n        Xtrain[\"fold_nb\"],\n        Xtest.drop([CFG.target, \"fold_nb\", CFG.grouper], axis = 1, errors = \"ignore\"),\n        clone(mdl),\n        method,\n        test_preds_req   = True,\n        ftreimp_plot_req = True,\n        ntop = 50,\n    ) \n\n    # Integrating data    \n    OOF_Preds[f\"{method}\"]  = oof_preds\n    Mdl_Preds[f\"{method}\"]  = test_preds\n    FtreImp[f\"{method}\"]    = ftreimp\n\n    del fitted_models, oof_preds, test_preds, ftreimp, mdl_best_iter\n    _ = utils.CleanMemory()\n    \nPrintColor(utils.CleanMemory())","metadata":{"execution":{"iopub.status.busy":"2024-09-29T08:22:37.080896Z","iopub.execute_input":"2024-09-29T08:22:37.081482Z","iopub.status.idle":"2024-09-29T08:26:54.434458Z","shell.execute_reply.started":"2024-09-29T08:22:37.081432Z","shell.execute_reply":"2024-09-29T08:26:54.432928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **ENSEMBLE**","metadata":{}},{"cell_type":"code","source":"%%time \n\noof_preds  = pd.DataFrame.from_dict(OOF_Preds, orient = \"columns\")\nmdl_preds  = pd.DataFrame.from_dict(Mdl_Preds, orient = \"columns\")\ntest_preds = 0\nscores     = 0\nens_preds  = np.zeros(len(oof_preds))\nmycv       = PDS(Xtrain[\"fold_nb\"])\n\nPrintColor(f\"\\n ----- Ridge Ensemble ----- \\n\") \n\nfor fold_nb, (train_idx, dev_idx) in enumerate(mycv.split(oof_preds, ytrain)):\n    Xtr  = oof_preds.iloc[train_idx]\n    ytr  = ytrain.loc[train_idx]\n    Xdev = oof_preds.iloc[dev_idx]\n    ydev = ytrain.loc[dev_idx]\n    \n    model = Ridge()\n    model.fit(Xtr, ytr)\n    \n    dev_preds  = model.predict(Xdev)\n    test_preds = test_preds + (model.predict(mdl_preds) / CFG.n_splits)\n    score      = utils.ScoreMetric(ydev, dev_preds)\n    PrintColor(f\"---> Fold{fold_nb} Score = {score:.6f}\", color = Fore.CYAN)\n    scores = scores + (score / CFG.n_splits)\n    \n    ens_preds[dev_idx] = dev_preds\n    \nPrintColor(f\"\\n---> Mean Score = {scores:.6f}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-29T08:21:49.353038Z","iopub.execute_input":"2024-09-29T08:21:49.353501Z","iopub.status.idle":"2024-09-29T08:21:49.439659Z","shell.execute_reply.started":"2024-09-29T08:21:49.353455Z","shell.execute_reply":"2024-09-29T08:21:49.438254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **SUBMISSION**","metadata":{}},{"cell_type":"code","source":"%%time \n\noof_preds.assign(**{\"Ensemble\": ens_preds}).\\\nto_csv(\n    os.path.join(CFG.op_path, f\"OOF_Preds_{CFG.model_label}{CFG.model_id}.csv\")\n)\n\nmdl_preds.assign(**{\"Ensemble\": test_preds}).\\\nto_csv(\n    os.path.join(CFG.op_path, f\"Mdl_Preds_{CFG.model_label}{CFG.model_id}.csv\")\n)\n\nsub_fl[CFG.target] = test_preds\nsub_fl.to_csv(\n    os.path.join(CFG.op_path, f\"submission.csv\")\n)\n\nprint()\n!ls\nprint()\n!head submission.csv\n\n_ = utils.CleanMemory()\nprint()","metadata":{"execution":{"iopub.status.busy":"2024-09-29T08:21:49.441437Z","iopub.execute_input":"2024-09-29T08:21:49.441914Z","iopub.status.idle":"2024-09-29T08:21:52.409414Z","shell.execute_reply.started":"2024-09-29T08:21:49.441868Z","shell.execute_reply":"2024-09-29T08:21:52.407425Z"},"trusted":true},"execution_count":null,"outputs":[]}]}