{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importing libraries and merging a dataset\n\nWe will need only training set to find the optimal parameters since the goal of this kernel is not to make a submission but to optimally tune a model. Also this would allow us not to worry about memory."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nimport warnings\nimport gc\nfrom bayes_opt import BayesianOptimization\nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"building = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\ntrain = pd.read_csv('../input/ashrae-energy-prediction/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(building, on='building_id', how='left')\ntrain = train.merge(weather_train, on=['site_id', 'timestamp'], how='left')\n# Logarithmic transform of target values\ny = np.log1p(train['meter_reading'])\n\ndel building, weather_train, train['meter_reading']\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Some manipulations with data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transforming timestamp to a datetime format\ntrain[\"timestamp\"] = pd.to_datetime(train[\"timestamp\"])\n\n# To save time and memory this dict is going to be used to label encode primary_use feature.\n# This is exactly what LabelEncoder would do to the data.\nle_dict = {'Education': 0,\n           'Office': 6,\n           'Entertainment/public assembly': 1,\n           'Lodging/residential': 4,\n           'Public services': 9,\n           'Healthcare': 3,\n           'Other': 7,\n           'Parking': 8,\n           'Manufacturing/industrial': 5,\n           'Food sales and service': 2,\n           'Retail': 11,\n           'Warehouse/storage': 15,\n           'Services': 12,\n           'Technology/science': 13,\n           'Utility': 14,\n           'Religious worship': 10}\n\ntrain['primary_use'] = train['primary_use'].map(le_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some new features from timestamp\ntrain[\"month\"] = train[\"timestamp\"].dt.month\ntrain[\"day\"] = train[\"timestamp\"].dt.day\ntrain[\"day_of_week\"] = train[\"timestamp\"].dt.weekday\ntrain[\"hour\"] = train[\"timestamp\"].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Saving some memory\nd_types = {'building_id': np.int16,\n          'meter': np.int8,\n          'site_id': np.int8,\n          'primary_use': np.int8,\n          'square_feet': np.int32,\n          'year_built': np.float16,\n          'floor_count': np.float16,\n          'air_temperature': np.float32,\n          'cloud_coverage': np.float16,\n          'dew_temperature': np.float32,\n          'precip_depth_1_hr': np.float16,\n          'sea_level_pressure': np.float32,\n          'wind_direction': np.float16,\n          'wind_speed': np.float32,\n          'month': np.int8,\n          'day': np.int16,\n          'hour': np.int16,\n          'day_of_week': np.int8}\n\nfor feature in d_types:\n    train[feature] = train[feature].astype(d_types[feature])\n    \ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# building_id is useless. I am explaining it in my EDA kernel: https://www.kaggle.com/nroman/eda-for-ashrae\ndel train['building_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# By default dataset is not sorted by time. We want to train on the past data to predict future.\ntrain = train.sort_index(by='timestamp').reset_index(drop=True)\n\n# timestamp is no longer needed\ndel train['timestamp']\n\n# Cut first 80% of the training dataset and the last 20% keep as holdout\ncut_idx = int(len(train) * 0.8)\nX_train, y_train, X_test, y_test = train.iloc[:cut_idx], y.iloc[:cut_idx], train.iloc[cut_idx:], y.iloc[cut_idx:]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitting an optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"bounds = {\n    'learning_rate': (0.002, 0.2),\n    'num_leaves': (50, 500), \n    'bagging_fraction' : (0.1, 1),\n    'feature_fraction' : (0.1, 1),\n    'min_child_weight': (0.001, 0.5),   \n    'min_data_in_leaf': (20, 170),\n    'max_depth': (-1, 32),\n    'reg_alpha': (0.1, 2), \n    'reg_lambda': (0.1, 2)\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(learning_rate, \n                num_leaves,\n                bagging_fraction, \n                feature_fraction, \n                min_child_weight,\n                min_data_in_leaf,\n                max_depth,\n                reg_alpha,\n                reg_lambda):\n    \n    params = {'learning_rate': learning_rate,\n              'num_leaves': int(num_leaves), \n              'bagging_fraction' : bagging_fraction,\n              'feature_fraction' : feature_fraction,\n              'min_child_weight': min_child_weight,   \n              'min_data_in_leaf': int(min_data_in_leaf),\n              'max_depth': int(max_depth),\n              'reg_alpha': reg_alpha, \n              'reg_lambda': reg_lambda,\n              'objective': 'regression',\n              'boosting_type': 'gbdt',\n              'random_state': 47,\n              'verbosity': 0,\n              'metric': 'rmse'}\n    \n    trn_data = lgb.Dataset(X_train, y_train)\n    val_data = lgb.Dataset(X_test, y_test)\n    model = lgb.train(params, trn_data, 5000, valid_sets = [trn_data, val_data], verbose_eval=0, early_stopping_rounds=50)\n    # Returning negative rmse because optimizer tries to maximize a function\n    return -model.best_score['valid_1']['rmse']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And here we go."},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = BayesianOptimization(f=train_model, pbounds=bounds, random_state=47)\noptimizer.maximize(init_points=10, n_iter=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Best RMSE score:', -optimizer.max['target'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Best parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Best set of parameters:')\nprint(optimizer.max['params'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}