{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()\n\n%matplotlib inline\n\nimport random\nrandom.seed(0)\n\nimport gc # garbage collection\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import Datasets\nWe import datasets and perform preliminary optimization.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import all data\ntrain = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv')\ntest = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv')\nweather_train = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv')\nweather_test = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv')\nbuilding_meta = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\ntrain.name = 'train'\ntest.name = 'test'\nweather_train.name = 'weather_train'\nweather_test.name = 'weather_test'\nbuilding_meta.name = 'building_meta'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Take a quick look at the data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show the top 5 entries and summary of each dataframe\ndataframes = [train, test, weather_train, weather_test, building_meta]\nfor df in dataframes:\n    print(df.name)\n    print(df.head())\n    print(df.info())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The timestamps in the dataframes are represented in strings. Here we convert them to pandas datetime objects.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert string timestamp to Pandas datetime\ntrain['timestamp'] = pd.to_datetime(train['timestamp'])\ntest['timestamp'] = pd.to_datetime(test['timestamp'])\nweather_train['timestamp'] = pd.to_datetime(weather_train['timestamp'])\nweather_test['timestamp'] = pd.to_datetime(weather_test['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The original dataframes are too large to be directly manipulated in RAM, and also have unnecessarily large datatypes. Here we reduce the size of the dataframes by converting each data columns into most suitable datatypes.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function reducing dataframe size to fit into memory\ndef reduce_memory_usage(dataframe, verbose=True): \n    starting_memory = dataframe.memory_usage().sum() / 1024**2\n    numeric_types = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    for col in dataframe:\n        data_type = dataframe[col].dtype\n        if data_type in numeric_types:\n            min_val = dataframe[col].min()\n            max_val = dataframe[col].max()\n            if str(data_type)[:3] == 'int':\n                if min_val > np.iinfo('int8').min and max_val < np.iinfo('int8').max:\n                    dataframe[col] = dataframe[col].astype('int8')\n                elif min_val > np.iinfo('int16').min and max_val < np.iinfo('int16').max:\n                    dataframe[col] = dataframe[col].astype('int16')\n                elif min_val > np.iinfo('int32').min and max_val < np.iinfo('int32').max:\n                    dataframe[col] = dataframe[col].astype('int32')\n                else:\n                    dataframe[col] = dataframe[col].astype('int64') # useless line?\n            else: \n                if min_val > np.finfo('float16').min and max_val < np.finfo('float16').max:\n                    dataframe[col] = dataframe[col].astype('float16')\n                elif min_val > np.finfo('float32').min and max_val < np.finfo('float32').max:\n                    dataframe[col] = dataframe[col].astype('float32')\n                else: \n                    dataframe[col] = dataframe[col].astype('float64') # useless line?\n    end_memory = dataframe.memory_usage().sum() / 1024**2\n    if verbose:\n        print('Memory usage decreased to {:.2f} mb ({:.2f}% decrease)'.format(end_memory, 100 * (starting_memory-end_memory) / starting_memory))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Apply memory reduction\nreduce_memory_usage(train)\nreduce_memory_usage(test)\nreduce_memory_usage(weather_train)\nreduce_memory_usage(weather_test)\nreduce_memory_usage(building_meta)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Initial Data Exploration","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"First let's explore the building_meta dataset.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"building_meta.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There is a lot of NaN in floor_count column. Let's check.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check nan values\n1 - building_meta.isna().sum() / len(building_meta) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There is almost half of year_built value missing, and 75% of floor_count missing. We need to pay extra attention when performing data wrangling.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's visualize the columns to gain some initial insight. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of buildings in each site\nax = building_meta['site_id'].value_counts().plot(kind='bar', figsize=(10,5), title='Number of Buildings in Each Site')\nax.set_xlabel('Site ID')\nax.set_ylabel('Number of Buildings')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of buildings for each primary_use\nax = building_meta['primary_use'].value_counts().plot(kind='bar', figsize=(10,5), title='Number of Buildings for Each Primary_Use')\nax.set_xlabel('Primary Use')\nax.set_ylabel('Number of Buildings')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Distribution of square_feet (size of building)\n\nax = building_meta['square_feet'].hist(bins=100, figsize=(10,5))\nax.set_xlabel('Size of Building-Square Feet')\nax.set_ylabel('Number of Buildings')\nax.set_title('Histogram of Size of Buildings')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of buildings built in each year\nindex = building_meta['year_built'].sort_values().value_counts().keys()\nvalue = building_meta['year_built'].sort_values().value_counts().values\nplt.figure(figsize=(10,5))\nplt.bar(index, value)\nplt.xlabel('Year')\nplt.ylabel('Number of Buildings')\nplt.title('Number of Buildings Built in Each Year')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of buildings with each floor_count\nax = building_meta['floor_count'].value_counts().plot(kind='bar', figsize=(10,5), title='Number of Buildings with Each Floor_Count')\nax.set_xlabel('Floor Count')\nax.set_ylabel('Number of Buildings')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we take a look at the weather_train dataframe.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check for NaN value.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"1 - weather_train.isna().sum() / len(weather_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most columns have more than 90% of values. Only cloud_coverage has only 50% data, and precip_depth_1_hr has 64%. We might take the columns out of consideration when building model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.groupby('site_id').count() / len(weather_train['timestamp'].unique())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we can see that some sites don't have any entry for certain weather data. Some sites don't have data for all the times, and some entries are missing for some times. Both should be taken into consideration when performing data wrangling. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Let's plot the weather measurement vs. time, and see the change within a year.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']\n\nfor col in columns:\n    plt.figure(figsize=(20,10))\n    plt.title(col + ' vs Time')\n    labels = [0] * (max(weather_train['site_id'])+1)\n    for id in range(max(weather_train['site_id'])+1):\n        time = weather_train[weather_train['site_id']==id ]['timestamp']\n        data = weather_train[weather_train['site_id']==id][col]\n        labels[id], = plt.plot(time, data, label=id)\n    plt.legend(handles=labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Air_temperature is behaving as expected, we can see that some sites are warmer than others. \n* Dew_temperature is similar to air_temperature. One interesting thing is how site0's dew temperature is quite lower than other sites' in the summer months.\n* Site 8 has the most large rainfalls\n* Sea_level_pressure changes more radically during the colder half of the year.\n* It's hard to gain insight from cloud_coverage, wind_direction and wind_speed by using this kind of plot.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Weather readings sometimes can have correlations with each other. Let's check that out.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nsns.heatmap(weather_train.corr(), annot=True, linewidths=.5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There is a few weak correlations. And one strong correlation between air_temperature and dew_temperature.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Now let's take a look at the train dataset.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's check meter column first.\n* Meter 0 => electricity\n* Meter 1 => chilled water\n* Meter 2 => steam\n* Meter 3 => hot water","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.groupby(['meter', 'timestamp'])['building_id'].count().groupby(level=0).mean())\nprint(train.groupby(['meter', 'timestamp'])['building_id'].count().groupby(level=0).mean() / len(building_meta['building_id']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that most of the meters are electricity meters, followed by chilled water, steam, and hot water meters. Almost 95% of the building have electricity meters, and the other three meters aren't widely adopted.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('meter')['meter_reading'].count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since there's way more electricity meters, it makes sense that there's way more eletricity meter readings as well.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Now, let's look at how the target variable **meter_reading** of different meters changes with respect to time.\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Graph of each meter reading from 1/1/16 to 12/31/16\ntrain.groupby(['timestamp', 'meter'])['meter_reading'].mean().unstack().plot(subplots=True, figsize=(20,20), title='Meter reading from 1/1/16 to 12/31/16', fontsize='14', sharex=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From the graphs above, we can see some trends, something that makes sense, and something not making sense (possible anomaly).\n* For electricy, there's a high consumption during weekdays, low consumption during weekends, high consumption during the day, low consumption at night. \n* Higher electricy and chilled water consumption during summer, lower during winter. And hot water consumption behaves the other way around.\n* Steam consumption data is confusing. Why is the highest usage from March to mid-June? Why is there a sudden spike in November (Possible anomaly)?\n* Sudden spikes in other meter readings as well. Possible anomaly (meter malfunction, wrong data entry, etc.)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" **Now we combine the dataset together, and perform data wrangling.**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train.merge(building_meta, on='building_id', how='left').merge(weather_train, on=['site_id', 'timestamp'], how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modeling","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Import needed libraries\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create time features from timestamp, and drop unneeded features","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['hour'] = train_df.timestamp.dt.hour\ntrain_df['weekday'] = train_df.timestamp.dt.weekday\ntrain_df['month'] = train_df.timestamp.dt.month\ntrain_df.drop(['timestamp', 'sea_level_pressure', 'wind_direction', 'wind_speed'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Split train_df into X_train and y_train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train_df['meter_reading']\nX_train = train_df.drop('meter_reading', axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Encode primary_use","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\nX_train.primary_use = le.fit_transform(X_train.primary_use)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Normalize floor_count, square_feet, and target variable","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train['floor_count'] = np.log1p(X_train['floor_count'])\nX_train['square_feet'] = np.log1p(X_train['square_feet'])\ny_train = np.log1p(y_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Take a look at the processed dataset now","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The idea here is to split the dataset into two, train two models, and validate on the other half of the data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X_half_1 = X_train[:int(X_train.shape[0] / 2)]\nX_half_2 = X_train[int(X_train.shape[0] / 2):]\n\ny_half_1 = y_train[:int(X_train.shape[0] / 2)]\ny_half_2 = y_train[int(X_train.shape[0] / 2):]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create a list of categorical features to feed into the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_features = [\"building_id\", \"site_id\", \"meter\", \"primary_use\", \"hour\", \"weekday\", \"month\", \"year_built\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create two lightGBM datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)\nd_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create watchlists and set hyperparameters","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"watchlist_1 = [d_half_1, d_half_2]\nwatchlist_2 = [d_half_2, d_half_1]\n\nparams = {\n    \"objective\": \"regression\",\n    \"boosting\": \"gbdt\",\n    \"num_leaves\": 40,\n    \"learning_rate\": 0.01,\n    \"feature_fraction\": 0.85,\n    \"reg_lambda\": 2,\n    \"metric\": \"rmse\"\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Building model with first half and validating on second half:\")\nmodel_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=1000, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)\n\nprint(\"Building model with second half and validating on first half:\")\nmodel_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=1000, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature Importance","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Plot feature importance using built-in lightGBM function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_fimp_1 = pd.DataFrame()\ndf_fimp_1[\"feature\"] = X_train.columns.values\ndf_fimp_1[\"importance\"] = model_half_1.feature_importance()\ndf_fimp_1[\"half\"] = 1\n\ndf_fimp_2 = pd.DataFrame()\ndf_fimp_2[\"feature\"] = X_train.columns.values\ndf_fimp_2[\"importance\"] = model_half_2.feature_importance()\ndf_fimp_2[\"half\"] = 2\n\ndf_fimp = pd.concat([df_fimp_1, df_fimp_2], axis=0)\n\nplt.figure(figsize=(14, 7))\nsns.barplot(x=\"importance\", y=\"feature\", data=df_fimp.sort_values(by=\"importance\", ascending=False))\nplt.title(\"LightGBM Feature Importance\")\nplt.tight_layout()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}