{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Stacked Regression\n\nThis notebook demonstrates a simple example of stacking regression models using [Mlxtend](http://rasbt.github.io/mlxtend/) (machine learning extensions) library. \n\nCode for data preprocessing and feature engineering is adapted from other kernels and slightly modified. \n\nThere might be some mistakes in code or logic since this is my first kernel, as well as the first Kaggle competition."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"## Introduction\n\nMultiple regression models could be combined together via so-called meta-regressor. This ensemble learning technique is called stacking regression. Each individual regression model is trained first, and then the meta-regressor is fitted based on the outputs (meta-features) of those models in the ensemble. This process is shown on the figure below.\n\n<img src=\"http://rasbt.github.io/mlxtend/user_guide/regressor/StackingRegressor_files/stackingregression_overview.png\" width=\"500px\">\n"},{"metadata":{},"cell_type":"markdown","source":"## Train data import and processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os, gc\nimport datetime\n\nimport numpy as np\nimport pandas as pd\n\nimport category_encoders\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom sklearn.linear_model import Lasso\nfrom sklearn.linear_model import Ridge\n\nfrom lightgbm import LGBMRegressor\n\nfrom mlxtend.regressor import StackingRegressor\n\nfrom pandas.api.types import is_categorical_dtype\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Memory optimization\n\n# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n# Modified to support timestamp type, categorical type\n# Modified to add option to use float16\ndef reduce_mem_usage(data, use_float16=False) -> pd.DataFrame:\n    start_mem = data.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in data.columns:\n        if is_datetime(data[col]) or is_categorical_dtype(data[col]):\n            continue\n        col_type = data[col].dtype\n\n        if col_type != object:\n            c_min = data[col].min()\n            c_max = data[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    data[col] = data[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    data[col] = data[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    data[col] = data[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    data[col] = data[col].astype(np.int64)\n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    data[col] = data[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    data[col] = data[col].astype(np.float32)\n                else:\n                    data[col] = data[col].astype(np.float64)\n        else:\n            data[col] = data[col].astype('category')\n\n    end_mem = data.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.2f}%'.format(\n        100 * (start_mem - end_mem) / start_mem))\n\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Path to data \nPATH = '../input/ashrae-energy-prediction/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import train data\ntrain = pd.read_csv(f'{PATH}train.csv')\nweather_train = pd.read_csv(f'{PATH}weather_train.csv')\n\n# Import metadata\nmetadata = pd.read_csv(f'{PATH}building_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove outliers in train data\ntrain = train[train['building_id'] != 1099]\ntrain = train.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for weather data processing\ndef weather_data_parser(weather_data) -> pd.DataFrame:\n    time_format = '%Y-%m-%d %H:%M:%S'\n    start_date = datetime.datetime.strptime(weather_data['timestamp'].min(), time_format)\n    end_date = datetime.datetime.strptime(weather_data['timestamp'].max(), time_format)\n    total_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)\n    hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format) for x in range(total_hours)]\n\n    for site_id in range(16):\n        site_hours = np.array(weather_data[weather_data['site_id'] == site_id]['timestamp'])\n        new_rows = pd.DataFrame(np.setdiff1d(hours_list, site_hours), columns=['timestamp'])\n        new_rows['site_id'] = site_id\n        weather_data = pd.concat([weather_data, new_rows], sort=True)\n        weather_data = weather_data.reset_index(drop=True)           \n\n    weather_data['datetime'] = pd.to_datetime(weather_data['timestamp'])\n    weather_data['day'] = weather_data['datetime'].dt.day\n    weather_data['week'] = weather_data['datetime'].dt.week\n    weather_data['month'] = weather_data['datetime'].dt.month\n\n    weather_data = weather_data.set_index(['site_id', 'day', 'month'])\n\n    air_temperature_filler = pd.DataFrame(weather_data.groupby(['site_id','day','month'])['air_temperature'].median(), columns=['air_temperature'])\n    weather_data.update(air_temperature_filler, overwrite=False)\n\n    cloud_coverage_filler = weather_data.groupby(['site_id', 'day', 'month'])['cloud_coverage'].median()\n    cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'), columns=['cloud_coverage'])\n\n    weather_data.update(cloud_coverage_filler, overwrite=False)\n\n    due_temperature_filler = pd.DataFrame(weather_data.groupby(['site_id','day','month'])['dew_temperature'].median(), columns=['dew_temperature'])\n    weather_data.update(due_temperature_filler, overwrite=False)\n\n    sea_level_filler = weather_data.groupby(['site_id','day','month'])['sea_level_pressure'].median()\n    sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'), columns=['sea_level_pressure'])\n\n    weather_data.update(sea_level_filler, overwrite=False)\n\n    wind_direction_filler =  pd.DataFrame(weather_data.groupby(['site_id','day','month'])['wind_direction'].median(), columns=['wind_direction'])\n    weather_data.update(wind_direction_filler, overwrite=False)\n\n    wind_speed_filler =  pd.DataFrame(weather_data.groupby(['site_id','day','month'])['wind_speed'].median(), columns=['wind_speed'])\n    weather_data.update(wind_speed_filler, overwrite=False)\n\n    precip_depth_filler = weather_data.groupby(['site_id','day','month'])['precip_depth_1_hr'].median()\n    precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'), columns=['precip_depth_1_hr'])\n\n    weather_data.update(precip_depth_filler, overwrite=False)\n\n    weather_data = weather_data.reset_index()\n    weather_data = weather_data.drop(['datetime','day','week','month'], axis=1)\n\n    return weather_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train weather data processing\nweather_train = weather_data_parser(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Memory optimization\ntrain = reduce_mem_usage(train, use_float16=True)\nweather_train = reduce_mem_usage(weather_train, use_float16=True)\nmetadata = reduce_mem_usage(metadata, use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge train data \ntrain = train.merge(metadata, on='building_id', how='left')\ntrain = train.merge(weather_train, on=['site_id', 'timestamp'], how='left')\n\ndel weather_train; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for train and test data processing\ndef data_parser(data) -> pd.DataFrame:\n    data.sort_values('timestamp')\n    data.reset_index(drop=True)\n    \n    data['timestamp'] = pd.to_datetime(data['timestamp'], format='%Y-%m-%d %H:%M:%S')\n    data['weekday'] = data['timestamp'].dt.weekday\n    data['hour'] = data['timestamp'].dt.hour\n    \n    data['square_feet'] =  np.log1p(data['square_feet']) \n    \n    data = data.drop(['timestamp', 'sea_level_pressure',\n        'wind_direction', 'wind_speed', 'year_built', 'floor_count'], axis=1)\n    \n    gc.collect()\n    \n    encoder = LabelEncoder()\n    data['primary_use'] = encoder.fit_transform(data['primary_use'])\n    \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train data processing\ntrain = data_parser(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define target and predictors\ntarget = np.log1p(train['meter_reading'])\nfeatures = train.drop(['meter_reading'], axis = 1) \n\ndel train; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Process categorical features\ncategorical_features = ['building_id', 'site_id', 'meter', 'primary_use']\n\nencoder = category_encoders.CountEncoder(cols=categorical_features)\nencoder.fit(features)\nfeatures = encoder.transform(features)\n\nfeatures_size = features.shape[0]\nfor feature in categorical_features:\n    features[feature] = features[feature] / features_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Missing data imputation\nimputer = SimpleImputer(missing_values=np.nan, strategy='median')\nimputer.fit(features)\nfeatures = imputer.transform(features)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model training \n\nAs a simple example we will create meta-regressor by stacking together regularized Ridge and Lasso regression models with LightGBM regressor. In this example there is minimum hyperparameters tuning and just three regressors are stacked together for simplicity."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Regressors\nlightgbm = LGBMRegressor(objective='regression', learning_rate=0.05, num_leaves=1024,\n    feature_fraction=0.8, bagging_fraction=0.9, bagging_freq=5) \n\nridge = Ridge(alpha=0.3)\nlasso = Lasso(alpha=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kfold = KFold(n_splits=2, shuffle=False)\n\nmodels = []\n\nfor idx, (train_idx, val_idx) in enumerate(kfold.split(features)):\n    \n    train_features, train_target = features[train_idx], target[train_idx]\n    val_features, val_target = features[val_idx], target[val_idx]\n    \n    model = StackingRegressor(regressors=(lightgbm, ridge, lasso),\n        meta_regressor=lightgbm, use_features_in_secondary=True)\n\n    model.fit(np.array(train_features), np.array(train_target))\n    models.append(model)\n\n    print('RMSE: {:.4f} of fold: {}'.format(\n        np.sqrt(mean_squared_error(val_target, model.predict(np.array(val_features)))), idx))\n\n    del train_features, train_target, val_features, val_target; gc.collect()\n\ndel features, target; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test data import and processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import test data\ntest = pd.read_csv(f'{PATH}test.csv')\nweather_test = pd.read_csv(f'{PATH}weather_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_ids = test['row_id']\ntest.drop('row_id', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test weather data processing\nweather_test = weather_data_parser(weather_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Memory optimization\ntest = reduce_mem_usage(test, use_float16=True)\nweather_test = reduce_mem_usage(weather_test, use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge test data\ntest = test.merge(metadata, on='building_id', how='left')\ntest = test.merge(weather_test, on=['site_id', 'timestamp'], how='left')\n\ndel metadata; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test data processing\ntest = data_parser(test)\n\ntest = encoder.transform(test)\nfor feature in categorical_features:\n    test[feature] = test[feature] / features_size\n\ntest = imputer.transform(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make predictions and create submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make predictions\npredictions = 0\nfor model in models:\n    predictions += np.expm1(model.predict(np.array(test))) / len(models)\n    del model; gc.collect()\n\ndel test, models; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create submission file\nsubmission = pd.DataFrame({\n    'row_id': row_ids,\n    'meter_reading': np.clip(predictions, 0, a_max=None)\n})\nsubmission.to_csv('submission.csv', index=False, float_format='%.4f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}