Here is a short code snippet that I am using to optimize hyperparametrs. Notice that I am using comet.ml as a hp logging and visualization server but it is optional of course. I am planning to add early stopping step and use the cv method of ligthgbm. Let me know if you find it useful. If so, I will make a kernel out of this code snippet.
\n\n[EDIT] I have added early stopping and the cv method.
\n\n```
\n\nimport numpy as np\nimport pandas as pd
\n\nfrom comet_ml import Experiment\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\nfrom lightgbm import LGBMRegressor, cv, Dataset\nfrom sklearn.model_selection import KFold, cross_val_score\nfrom tqdm import tqdm
\n\nN_FOLDS = 3\nSEED = 314\nMAX_EVALS = 1000\nCV = KFold(N_FOLDS, random_state=SEED, shuffle=True)\nTARGET_COL = \"\"\nTO_DROP_COLS = \"\"
\n\nCOMET_ML_API_KEY = \"\"\nWORKSPACE = \"\"\nPROJECT_NAME = \"\"\ncomet_experiment = Experiment(api_key=COMET_ML_API_KEY,\n project_name=PROJECT_NAME,\n workspace=WORKSPACE,\n auto_param_logging=True)
\n\nclass HPOptimizer(object):
\n\ndef __init__(self, X, y debug=False, max_evals=MAX_EVALS, use_comet=False,\n early_stopping=True):\n if debug:\n X = X[:100]\n y = y[:100]\n self.X = X\n self.y = y\n self.seed = SEED\n self.trials = Trials()\n self.pbar = tqdm(total=MAX_EVALS, desc=\"Hyperopt for LGBMRegressor\")\n self.max_evals = max_evals\n self.model_class = LGBMRegressor\n self.train_dataset = Dataset(X, label=y)\n\ndef score(self, params):\n print(params)\n model = self.model_class(**params)\n\n\n if early_stopping:\n cv_scores = cv(params, self.train_dataset, stratified=False,\n num_boost_round=10000, early_stopping_rounds=1000, verbose_eval=1000,\n nfold=N_FOLDS)\n loss = np.log(min(cv_scores['l1-mean']))\n else:\n loss_cv = cross_val_score(model, X=self.X, y=self.y, cv=CV,\n scoring=\"neg_mean_absolute_error\")\n loss = np.log((-1 * loss_cv).mean())\n print(\"Loss for choosing hp: \", loss)\n self.pbar.update()\n if use_comet:\n comet_experiment.log_parameters(params)\n comet_experiment.log_metrics({\"loss\": loss, \"loss_cv\": loss_cv.tolist()})\n\n return {'loss': loss, 'status': STATUS_OK}\n\ndef optimize(self):\n \"\"\"\n This is the optimization function that given a space (space here) of\n hyperparameters and a scoring function (score here), finds the best hyperparameters.\n \"\"\"\n # TODO: choose a better hp space. \n space = {\"boosting_type\": \"gbdt\",\n \"objective\": \"huber\",\n \"metrics\": \"mae\",\n \"n_estimators\": 1000 + hp.randint('n_estimators', 10000),\n \"learning_rate\": hp.loguniform('learning_rate', np.log(1e-4), np.log(1.0)),\n \"seed\": self.seed}\n\n best = fmin(self.score, space, algo=tpe.suggest, max_evals=self.max_evals, trials=self.trials)\n return best\ndef run(debug=False, use_comet=False, frac=0.1, early_stopping=True):\n # Read the train dataset\n PATH_TO_DATASET = \"\"\n df = pd.read_parquet(PATH_TO_DATASET)\n df = reduce_mem_usage(df)\n # Sample a fraction to speed up things\n if frac:\n df = df.sample(frac=frac, random_state=SEED)\n FEATURES_COLS = df.drop(TO_DROP_COLS).columns.tolist()\n X = df.loc[:, FEATURES_COLS].values\n y = df[TARGET_COL].values\n optimizer = HPOptimizer(X, y, debug=debug, use_comet=use_comet, early_stopping=early_stopping)\n optimal_hp = optimizer.optimize()\n print(optimal_hp)
\n\nif name == \"main\":\n run()
\n\n```
", "messages": [ { "id": "580519", "postDate": "07/20/2019 09:20:08", "content": "Here is a short code snippet that I am using to optimize hyperparametrs. Notice that I am using comet.ml as a hp logging and visualization server but it is optional of course. I am planning to add early stopping step and use the cv method of ligthgbm. Let me know if you find it useful. If so, I will make a kernel out of this code snippet.
\n\n[EDIT] I have added early stopping and the cv method.
\n\n```
\n\nimport numpy as np\nimport pandas as pd
\n\nfrom comet_ml import Experiment\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\nfrom lightgbm import LGBMRegressor, cv, Dataset\nfrom sklearn.model_selection import KFold, cross_val_score\nfrom tqdm import tqdm
\n\nN_FOLDS = 3\nSEED = 314\nMAX_EVALS = 1000\nCV = KFold(N_FOLDS, random_state=SEED, shuffle=True)\nTARGET_COL = \"\"\nTO_DROP_COLS = \"\"
\n\nCOMET_ML_API_KEY = \"\"\nWORKSPACE = \"\"\nPROJECT_NAME = \"\"\ncomet_experiment = Experiment(api_key=COMET_ML_API_KEY,\n project_name=PROJECT_NAME,\n workspace=WORKSPACE,\n auto_param_logging=True)
\n\nclass HPOptimizer(object):
\n\ndef __init__(self, X, y debug=False, max_evals=MAX_EVALS, use_comet=False,\n early_stopping=True):\n if debug:\n X = X[:100]\n y = y[:100]\n self.X = X\n self.y = y\n self.seed = SEED\n self.trials = Trials()\n self.pbar = tqdm(total=MAX_EVALS, desc=\"Hyperopt for LGBMRegressor\")\n self.max_evals = max_evals\n self.model_class = LGBMRegressor\n self.train_dataset = Dataset(X, label=y)\n\ndef score(self, params):\n print(params)\n model = self.model_class(**params)\n\n\n if early_stopping:\n cv_scores = cv(params, self.train_dataset, stratified=False,\n num_boost_round=10000, early_stopping_rounds=1000, verbose_eval=1000,\n nfold=N_FOLDS)\n loss = np.log(min(cv_scores['l1-mean']))\n else:\n loss_cv = cross_val_score(model, X=self.X, y=self.y, cv=CV,\n scoring=\"neg_mean_absolute_error\")\n loss = np.log((-1 * loss_cv).mean())\n print(\"Loss for choosing hp: \", loss)\n self.pbar.update()\n if use_comet:\n comet_experiment.log_parameters(params)\n comet_experiment.log_metrics({\"loss\": loss, \"loss_cv\": loss_cv.tolist()})\n\n return {'loss': loss, 'status': STATUS_OK}\n\ndef optimize(self):\n \"\"\"\n This is the optimization function that given a space (space here) of\n hyperparameters and a scoring function (score here), finds the best hyperparameters.\n \"\"\"\n # TODO: choose a better hp space. \n space = {\"boosting_type\": \"gbdt\",\n \"objective\": \"huber\",\n \"metrics\": \"mae\",\n \"n_estimators\": 1000 + hp.randint('n_estimators', 10000),\n \"learning_rate\": hp.loguniform('learning_rate', np.log(1e-4), np.log(1.0)),\n \"seed\": self.seed}\n\n best = fmin(self.score, space, algo=tpe.suggest, max_evals=self.max_evals, trials=self.trials)\n return best\ndef run(debug=False, use_comet=False, frac=0.1, early_stopping=True):\n # Read the train dataset\n PATH_TO_DATASET = \"\"\n df = pd.read_parquet(PATH_TO_DATASET)\n df = reduce_mem_usage(df)\n # Sample a fraction to speed up things\n if frac:\n df = df.sample(frac=frac, random_state=SEED)\n FEATURES_COLS = df.drop(TO_DROP_COLS).columns.tolist()\n X = df.loc[:, FEATURES_COLS].values\n y = df[TARGET_COL].values\n optimizer = HPOptimizer(X, y, debug=debug, use_comet=use_comet, early_stopping=early_stopping)\n optimal_hp = optimizer.optimize()\n print(optimal_hp)
\n\nif name == \"main\":\n run()
\n\n```
", "rawMarkdown": "Here is a short code snippet that I am using to optimize hyperparametrs. Notice that I am using [comet.ml ](https://www.comet.ml/)as a hp logging and visualization server but it is optional of course. I am planning to add early stopping step and use the [**cv**](https://lightgbm.readthedocs.io/en/latest/_modules/lightgbm/engine.html#cv) method of ligthgbm. Let me know if you find it useful. If so, I will make a kernel out of this code snippet. \n\n[**EDIT**] I have added early stopping and the cv method. \n\n```\n# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pandas as pd\n# Not necessary but great to track the hp parameters\nfrom comet_ml import Experiment\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\nfrom lightgbm import LGBMRegressor, cv, Dataset\nfrom sklearn.model_selection import KFold, cross_val_score\nfrom tqdm import tqdm\n\n\nN_FOLDS = 3\nSEED = 314\nMAX_EVALS = 1000\nCV = KFold(N_FOLDS, random_state=SEED, shuffle=True)\nTARGET_COL = \"\"\nTO_DROP_COLS = \"\"\n\n# Comet optional setup\nCOMET_ML_API_KEY = \"\"\nWORKSPACE = \"\"\nPROJECT_NAME = \"\"\ncomet_experiment = Experiment(api_key=COMET_ML_API_KEY,\n project_name=PROJECT_NAME,\n workspace=WORKSPACE,\n auto_param_logging=True)\n\n\nclass HPOptimizer(object):\n\n def __init__(self, X, y debug=False, max_evals=MAX_EVALS, use_comet=False,\n early_stopping=True):\n if debug:\n X = X[:100]\n y = y[:100]\n self.X = X\n self.y = y\n self.seed = SEED\n self.trials = Trials()\n self.pbar = tqdm(total=MAX_EVALS, desc=\"Hyperopt for LGBMRegressor\")\n self.max_evals = max_evals\n self.model_class = LGBMRegressor\n self.train_dataset = Dataset(X, label=y)\n\n def score(self, params):\n print(params)\n model = self.model_class(**params)\n\n\n if early_stopping:\n cv_scores = cv(params, self.train_dataset, stratified=False,\n num_boost_round=10000, early_stopping_rounds=1000, verbose_eval=1000,\n nfold=N_FOLDS)\n loss = np.log(min(cv_scores['l1-mean']))\n else:\n loss_cv = cross_val_score(model, X=self.X, y=self.y, cv=CV,\n scoring=\"neg_mean_absolute_error\")\n loss = np.log((-1 * loss_cv).mean())\n print(\"Loss for choosing hp: \", loss)\n self.pbar.update()\n if use_comet:\n comet_experiment.log_parameters(params)\n comet_experiment.log_metrics({\"loss\": loss, \"loss_cv\": loss_cv.tolist()})\n\n return {'loss': loss, 'status': STATUS_OK}\n\n def optimize(self):\n \"\"\"\n This is the optimization function that given a space (space here) of\n hyperparameters and a scoring function (score here), finds the best hyperparameters.\n \"\"\"\n # TODO: choose a better hp space. \n space = {\"boosting_type\": \"gbdt\",\n \"objective\": \"huber\",\n \"metrics\": \"mae\",\n \"n_estimators\": 1000 + hp.randint('n_estimators', 10000),\n \"learning_rate\": hp.loguniform('learning_rate', np.log(1e-4), np.log(1.0)),\n \"seed\": self.seed}\n\n best = fmin(self.score, space, algo=tpe.suggest, max_evals=self.max_evals, trials=self.trials)\n return best\n\n\ndef run(debug=False, use_comet=False, frac=0.1, early_stopping=True):\n # Read the train dataset\n PATH_TO_DATASET = \"\"\n df = pd.read_parquet(PATH_TO_DATASET)\n df = reduce_mem_usage(df)\n # Sample a fraction to speed up things\n if frac:\n df = df.sample(frac=frac, random_state=SEED)\n FEATURES_COLS = df.drop(TO_DROP_COLS).columns.tolist()\n X = df.loc[:, FEATURES_COLS].values\n y = df[TARGET_COL].values\n optimizer = HPOptimizer(X, y, debug=debug, use_comet=use_comet, early_stopping=early_stopping)\n optimal_hp = optimizer.optimize()\n print(optimal_hp)\n\n\nif __name__ == \"__main__\":\n run()\n\n```", "votes": null }, { "id": "580520", "postDate": "07/20/2019 09:24:02", "content": "Another trick (based on this thread): try to use a random sample of the dataset to speed up things. I think around 10% is good enough. \nOnce you overfit this smaller dataset and get a good intuition about which hp work best for each bond type, you can add regularization and explore other hp. \nI hope this helps.
", "rawMarkdown": "Another trick (based on this [thread](https://www.kaggle.com/c/champs-scalar-coupling/discussion/100469#latest-579575)): try to use a random sample of the dataset to speed up things. I think around 10% is good enough. \nOnce you overfit this smaller dataset and get a good intuition about which hp work best for each bond type, you can add regularization and explore other hp. \nI hope this helps.", "votes": null } ], "comments": [ { "id": 580520, "author_name": "yassinealouini", "author_url": "", "post_date": "07/20/2019 09:24:02", "content": "Another trick (based on this thread): try to use a random sample of the dataset to speed up things. I think around 10% is good enough. \nOnce you overfit this smaller dataset and get a good intuition about which hp work best for each bond type, you can add regularization and explore other hp. \nI hope this helps.
", "votes": null, "replies": [] } ], "raw_markdown_by_id": { "580519": "Here is a short code snippet that I am using to optimize hyperparametrs. Notice that I am using [comet.ml ](https://www.comet.ml/)as a hp logging and visualization server but it is optional of course. I am planning to add early stopping step and use the [**cv**](https://lightgbm.readthedocs.io/en/latest/_modules/lightgbm/engine.html#cv) method of ligthgbm. Let me know if you find it useful. If so, I will make a kernel out of this code snippet. \n\n[**EDIT**] I have added early stopping and the cv method. \n\n```\n# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pandas as pd\n# Not necessary but great to track the hp parameters\nfrom comet_ml import Experiment\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\nfrom lightgbm import LGBMRegressor, cv, Dataset\nfrom sklearn.model_selection import KFold, cross_val_score\nfrom tqdm import tqdm\n\n\nN_FOLDS = 3\nSEED = 314\nMAX_EVALS = 1000\nCV = KFold(N_FOLDS, random_state=SEED, shuffle=True)\nTARGET_COL = \"\"\nTO_DROP_COLS = \"\"\n\n# Comet optional setup\nCOMET_ML_API_KEY = \"\"\nWORKSPACE = \"\"\nPROJECT_NAME = \"\"\ncomet_experiment = Experiment(api_key=COMET_ML_API_KEY,\n project_name=PROJECT_NAME,\n workspace=WORKSPACE,\n auto_param_logging=True)\n\n\nclass HPOptimizer(object):\n\n def __init__(self, X, y debug=False, max_evals=MAX_EVALS, use_comet=False,\n early_stopping=True):\n if debug:\n X = X[:100]\n y = y[:100]\n self.X = X\n self.y = y\n self.seed = SEED\n self.trials = Trials()\n self.pbar = tqdm(total=MAX_EVALS, desc=\"Hyperopt for LGBMRegressor\")\n self.max_evals = max_evals\n self.model_class = LGBMRegressor\n self.train_dataset = Dataset(X, label=y)\n\n def score(self, params):\n print(params)\n model = self.model_class(**params)\n\n\n if early_stopping:\n cv_scores = cv(params, self.train_dataset, stratified=False,\n num_boost_round=10000, early_stopping_rounds=1000, verbose_eval=1000,\n nfold=N_FOLDS)\n loss = np.log(min(cv_scores['l1-mean']))\n else:\n loss_cv = cross_val_score(model, X=self.X, y=self.y, cv=CV,\n scoring=\"neg_mean_absolute_error\")\n loss = np.log((-1 * loss_cv).mean())\n print(\"Loss for choosing hp: \", loss)\n self.pbar.update()\n if use_comet:\n comet_experiment.log_parameters(params)\n comet_experiment.log_metrics({\"loss\": loss, \"loss_cv\": loss_cv.tolist()})\n\n return {'loss': loss, 'status': STATUS_OK}\n\n def optimize(self):\n \"\"\"\n This is the optimization function that given a space (space here) of\n hyperparameters and a scoring function (score here), finds the best hyperparameters.\n \"\"\"\n # TODO: choose a better hp space. \n space = {\"boosting_type\": \"gbdt\",\n \"objective\": \"huber\",\n \"metrics\": \"mae\",\n \"n_estimators\": 1000 + hp.randint('n_estimators', 10000),\n \"learning_rate\": hp.loguniform('learning_rate', np.log(1e-4), np.log(1.0)),\n \"seed\": self.seed}\n\n best = fmin(self.score, space, algo=tpe.suggest, max_evals=self.max_evals, trials=self.trials)\n return best\n\n\ndef run(debug=False, use_comet=False, frac=0.1, early_stopping=True):\n # Read the train dataset\n PATH_TO_DATASET = \"\"\n df = pd.read_parquet(PATH_TO_DATASET)\n df = reduce_mem_usage(df)\n # Sample a fraction to speed up things\n if frac:\n df = df.sample(frac=frac, random_state=SEED)\n FEATURES_COLS = df.drop(TO_DROP_COLS).columns.tolist()\n X = df.loc[:, FEATURES_COLS].values\n y = df[TARGET_COL].values\n optimizer = HPOptimizer(X, y, debug=debug, use_comet=use_comet, early_stopping=early_stopping)\n optimal_hp = optimizer.optimize()\n print(optimal_hp)\n\n\nif __name__ == \"__main__\":\n run()\n\n```", "580520": "Another trick (based on this [thread](https://www.kaggle.com/c/champs-scalar-coupling/discussion/100469#latest-579575)): try to use a random sample of the dataset to speed up things. I think around 10% is good enough. \nOnce you overfit this smaller dataset and get a good intuition about which hp work best for each bond type, you can add regularization and explore other hp. \nI hope this helps." }, "source": "meta" }