{"cells":[{"metadata":{"id":"MEUlcmt9083n","colab_type":"text"},"cell_type":"markdown","source":"# ASHRAE - Great Energy Predictor III"},{"metadata":{"id":"O63WuRIQ08xf","colab_type":"text"},"cell_type":"markdown","source":"### Importing the libraries"},{"metadata":{"id":"YMtebhB0_uE1","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# !pip install tensorflow==2.0.0\n# !pip install kaggle","execution_count":null,"outputs":[]},{"metadata":{"id":"qIC6f8Er04xh","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nimport gc\nimport math\nfrom datetime import datetime","execution_count":null,"outputs":[]},{"metadata":{"id":"fK9ntu5azsHY","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.rcParams['figure.dpi'] = 64\nplt.rcParams['figure.figsize'] = [12, 8]\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"id":"JIIKhSG1_sK2","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import StandardScaler\n\nfrom sklearn.model_selection import KFold\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"id":"3ThrvHM3_jMo","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from tensorflow.keras.wrappers.scikit_learn import KerasRegressor\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization, Input, Embedding, concatenate, Flatten\nfrom tensorflow.keras.optimizers import Adam, Adamax, RMSprop\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler\nfrom tensorflow.keras.losses import mean_squared_error as mse_loss\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.utils import plot_model","execution_count":null,"outputs":[]},{"metadata":{"id":"4v38ahRqzOF6","colab_type":"text"},"cell_type":"markdown","source":"### Downloading the Kaggle competition dataset"},{"metadata":{"id":"S0U9D1I7zm9Q","colab_type":"code","outputId":"1deebbd9-fc2c-4f9c-a2f7-a37a824d6468","colab":{"base_uri":"https://localhost:8080/","height":340},"trusted":true},"cell_type":"code","source":"# !kaggle competitions download -c ashrae-energy-prediction","execution_count":null,"outputs":[]},{"metadata":{"id":"vYGqlqln2WYP","colab_type":"text"},"cell_type":"markdown","source":"### Importing the dataset\n"},{"metadata":{"id":"JN3Y_ips2grJ","colab_type":"code","outputId":"54f84b4d-edf7-4718-e65b-c00489f9e703","colab":{"base_uri":"https://localhost:8080/","height":51},"trusted":true},"cell_type":"code","source":"%%time\n!ls /kaggle/input\nroot = '/kaggle/input/ashrae-energy-prediction/'\ntrain = pd.read_csv(root + 'train.csv')\ntest = pd.read_csv(root + 'test.csv')\nweather_train = pd.read_csv(root + 'weather_train.csv')\nweather_test = pd.read_csv(root + 'weather_test.csv')\nbuilding = pd.read_csv(root + 'building_metadata.csv')\nsample_submission = pd.read_csv(root + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"id":"4Ca1k85j4fgh","colab_type":"code","outputId":"fbff72af-d261-41e7-adfc-d739743ad320","colab":{"base_uri":"https://localhost:8080/","height":102},"trusted":true},"cell_type":"code","source":"print('Size of train data', train.shape)\n# print('Size of test data', test.shape)\nprint('Size of weather_train data', weather_train.shape)\n# print('Size of weather_test data', weather_test.shape)\nprint('Size of building data', building.shape)","execution_count":null,"outputs":[]},{"metadata":{"id":"QYRvQXGtI_tU","colab_type":"text"},"cell_type":"markdown","source":"### Exploratory data analysis"},{"metadata":{"id":"9Ex0_wgXCcHl","colab_type":"text"},"cell_type":"markdown","source":"#### Helper plotting functions"},{"metadata":{"id":"GEsuQogcCebo","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def plot_dist(df, column, func=lambda x: x):\n    plt.figure(figsize=(12,8))\n    ax = sns.distplot(df[column].dropna().map(func))\n    ax.set_title(column+\" Distribution\", fontsize=16)\n    plt.xlabel(column, fontsize=12)\n    plt.ylabel(\"distribution\", fontsize=12)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"1_KO8KVaCpEo","colab_type":"text"},"cell_type":"markdown","source":"**meter_reading** - The target variable. Energy consumption in kWh (or equivalent). Note that this is real data with measurement error, which we expect will impose a baseline level of modeling error. UPDATE: as discussed here, the site 0 electric meter readings are in kBTU.\n"},{"metadata":{"id":"9wY4_Ms4Dd7H","colab_type":"code","outputId":"e8c1fcd2-834c-4fd9-b5d7-59f30d67de3e","colab":{"base_uri":"https://localhost:8080/","height":170},"trusted":true},"cell_type":"code","source":"train['meter_reading'].describe()","execution_count":null,"outputs":[]},{"metadata":{"id":"JX_X2gZiDoPN","colab_type":"code","outputId":"d03ee881-eec1-4de5-e6b0-e270a38fc12d","colab":{"base_uri":"https://localhost:8080/","height":518},"trusted":true},"cell_type":"code","source":"plot_dist(train, 'meter_reading')","execution_count":null,"outputs":[]},{"metadata":{"id":"DKNkW_Q8FQ5m","colab_type":"code","outputId":"3f938c06-773c-41d2-f5c9-bc31169edcd2","colab":{"base_uri":"https://localhost:8080/","height":518},"trusted":true},"cell_type":"code","source":"### Data is heavily skewed, we'll apply np.log1p to see how it affects the distribution\nplot_dist(train, 'meter_reading', np.log1p)","execution_count":null,"outputs":[]},{"metadata":{"id":"vKjgGFXqzRZY","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":593},"outputId":"27fa8a31-1e69-4d5c-f1b3-9bcf8f8ec45e","trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, figsize=(14, 6), dpi=100)\ntemp_df = train[['timestamp', 'meter_reading']].copy()\ntemp_df['timestamp'] = pd.to_datetime(temp_df['timestamp'])\ntemp_df.set_index('timestamp').resample('H').mean()['meter_reading'].plot(ax=axes, label='By hour', alpha=0.8).set_ylabel('Meter reading', fontsize=14);\ntemp_df.set_index('timestamp').resample('D').mean()['meter_reading'].plot(ax=axes, label='By day', alpha=1).set_ylabel('Meter reading', fontsize=14);\naxes.set_title('Mean Meter reading by hour and day', fontsize=16);\naxes.legend();\ndel temp_df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"d-9r6Kv2zbDT","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"c7ebaccd-c700-47e1-a6c6-0d1acfa83410","trusted":true},"cell_type":"code","source":"temp_df = train.copy()\ntemp_df = temp_df.merge(building, on='building_id', how='left')\ntemp_df['timestamp'] = pd.to_datetime(temp_df['timestamp'])\nfig, axes = plt.subplots(8,2,figsize=(14, 30), dpi=100)\nfor i in range(temp_df['site_id'].nunique()):\n    temp_df[temp_df['site_id'] == i][['timestamp', 'meter_reading']].set_index('timestamp').resample('H').mean()['meter_reading'].plot(ax=axes[i%8][i//8], alpha=0.8, label='By hour', color='tab:blue').set_ylabel('Mean meter reading', fontsize=13);\n    temp_df[temp_df['site_id'] == i][['timestamp', 'meter_reading']].set_index('timestamp').resample('D').mean()['meter_reading'].plot(ax=axes[i%8][i//8], alpha=1, label='By day', color='tab:orange').set_xlabel('');\n    axes[i%8][i//8].legend();\n    axes[i%8][i//8].set_title('site_id {}'.format(i), fontsize=13);\n    plt.subplots_adjust(hspace=0.45)\ndel temp_df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"daV7fax_nS0s","colab_type":"text"},"cell_type":"markdown","source":"**meter** - The meter id code. Read as {0: electricity, 1: chilledwater, 2: steam, 3: hotwater}. Not every building has all meter types.\n"},{"metadata":{"id":"x9ZxqIvjnXCz","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":514},"outputId":"6917935e-51d1-4dcd-f4e3-ea10d77b2dd6","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,8))\nsns.boxplot(x='meter', y='meter_reading', data=train, showfliers=False)","execution_count":null,"outputs":[]},{"metadata":{"id":"ffNgFGF7ofV_","colab_type":"text"},"cell_type":"markdown","source":"**primary_use** - Indicator of the primary category of activities for the building based on EnergyStar property type definitions\n"},{"metadata":{"id":"rXRTI-Lwohzx","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":623},"outputId":"d28918f2-c13c-4803-c549-15a7da890bf0","trusted":true},"cell_type":"code","source":"temp_df = train.copy()\ntemp_df = train.merge(building, on='building_id', how='left')\nplt.figure(figsize=(12,10))\nsns.boxplot(x='meter_reading', y='primary_use', data=temp_df, showfliers=False)\ndel temp_df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"sX_7ihhDp6Vv","colab_type":"text"},"cell_type":"markdown","source":"**square_feet** - Gross floor area of the building\n"},{"metadata":{"id":"Fr3zT_24qBvQ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":170},"outputId":"6499afda-3614-4fde-b10b-6e71444cfda7","trusted":true},"cell_type":"code","source":"building['square_feet'].describe()","execution_count":null,"outputs":[]},{"metadata":{"id":"gee-LU5JqGGU","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"22f338d1-0b2e-4487-b9df-2e22641126bd","trusted":true},"cell_type":"code","source":"plot_dist(building, 'square_feet')","execution_count":null,"outputs":[]},{"metadata":{"id":"hcQvm1iXqP98","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"194dd98f-7fda-46f7-d50d-b1cd17c357dc","trusted":true},"cell_type":"code","source":"### Data is heavily skewed, we'll apply np.log1p to see how it affects the distribution\nplot_dist(train, 'meter_reading', np.log1p)","execution_count":null,"outputs":[]},{"metadata":{"id":"mnBONSxbqZTi","colab_type":"text"},"cell_type":"markdown","source":"**year_built** - Year building was opened\n"},{"metadata":{"id":"aFeYb-dztBbE","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":136},"outputId":"92d4f607-39eb-437e-f371-280b7f8ba9d0","trusted":true},"cell_type":"code","source":"building.isna().sum().sort_values(ascending=False)*100 / len(building)","execution_count":null,"outputs":[]},{"metadata":{"id":"6nlpj6VNqkv_","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"36eba942-0958-41f0-b142-14bd30270941","trusted":true},"cell_type":"code","source":"plot_dist(building, 'year_built')","execution_count":null,"outputs":[]},{"metadata":{"id":"GU3LtVEhq1DI","colab_type":"text"},"cell_type":"markdown","source":"**floor_count** - Number of floors of the building\n"},{"metadata":{"id":"-17EHJCnq4fv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":340},"outputId":"57d97ff4-854f-405c-f327-9dcbd0ce7b07","trusted":true},"cell_type":"code","source":"building['floor_count'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"id":"6-Gz-McYuz87","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":515},"outputId":"4e979f64-59ef-4192-962a-c9222aa61136","trusted":true},"cell_type":"code","source":"temp_df = train.copy()\ntemp_df = train.merge(building, on='building_id', how='left')\nplt.figure(figsize=(15,8))\nsns.boxplot(x='floor_count', y='meter_reading', data=temp_df, showfliers=False)\ndel temp_df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"44Ti10fhs0zb","colab_type":"text"},"cell_type":"markdown","source":"**air_temperature** - Degrees Celsius\n"},{"metadata":{"id":"Qta4h04CtRS7","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":187},"outputId":"20984a81-950f-470b-bc4f-a478361e0b7f","trusted":true},"cell_type":"code","source":"weather_train.isna().sum().sort_values(ascending = False) * 100 / len(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"id":"lmdHj-hNs40d","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"41648d01-6e08-4f58-c9b9-e247f8862655","trusted":true},"cell_type":"code","source":"plot_dist(weather_train, 'air_temperature')","execution_count":null,"outputs":[]},{"metadata":{"id":"nG2dpXhWtaye","colab_type":"text"},"cell_type":"markdown","source":"**dew_temperature** - Degrees Celsius\n"},{"metadata":{"id":"oNqCAZ4dtfn9","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"c9286a24-ab30-4a84-a0aa-f381ec856f26","trusted":true},"cell_type":"code","source":"plot_dist(weather_train, 'dew_temperature')","execution_count":null,"outputs":[]},{"metadata":{"id":"UZpGUIXCtp23","colab_type":"text"},"cell_type":"markdown","source":"**wind_speed** - Meters per second\n"},{"metadata":{"id":"qnI9zFCJtyAU","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"8c117e95-f0eb-4042-9a37-03e4bee86f1b","trusted":true},"cell_type":"code","source":"plot_dist(weather_train, 'wind_speed')","execution_count":null,"outputs":[]},{"metadata":{"id":"7NesVuQ7t_t9","colab_type":"text"},"cell_type":"markdown","source":"**wind_direction** - Compass direction (0-360)"},{"metadata":{"id":"PdYgzj1ouDhL","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"8e62e56e-1557-4b89-e989-cfdab8e61d83","trusted":true},"cell_type":"code","source":"plot_dist(weather_train, 'wind_direction')","execution_count":null,"outputs":[]},{"metadata":{"id":"BnGG18l5uJjO","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"59d118e0-1b5d-4dec-ef5e-0c203890b284","trusted":true},"cell_type":"code","source":"sinus = lambda x: np.sin((2*np.pi*x)/360)\nplot_dist(weather_train, 'wind_direction', sinus)","execution_count":null,"outputs":[]},{"metadata":{"id":"IccEjPocv_m9","colab_type":"text"},"cell_type":"markdown","source":"**cloud_coverage** - Portion of the sky covered in clouds, in oktas\n"},{"metadata":{"id":"F2SjoNO8wCHn","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":623},"outputId":"b08796a6-4f28-4fb0-c603-4786214d3080","trusted":true},"cell_type":"code","source":"temp_df = train.copy()\ntemp_df = train.merge(building, on='building_id', how='left')\ntemp_df = temp_df.merge(weather_train, on=['site_id', 'timestamp'], how='left')\nplt.figure(figsize=(16,10))\nsns.boxplot(x='cloud_coverage', y='meter_reading', data=temp_df, showfliers=False)\ndel temp_df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"dV78uBjOxUoh","colab_type":"text"},"cell_type":"markdown","source":"**sea_level_pressure** - Millibar/hectopascals\n"},{"metadata":{"id":"6QAxYuUxxW3j","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"7f7b876b-52e2-4a61-96ba-ee51ad0a83eb","trusted":true},"cell_type":"code","source":"plot_dist(weather_train, 'sea_level_pressure')","execution_count":null,"outputs":[]},{"metadata":{"id":"O1PIn8GXxdQv","colab_type":"text"},"cell_type":"markdown","source":"**precip_depth_1_hr** - Millimeters\n"},{"metadata":{"id":"argTIxgHxk_T","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"d237a9d9-c3b2-4a71-a3a9-3d44367c5d5a","trusted":true},"cell_type":"code","source":"plot_dist(weather_train, 'precip_depth_1_hr')","execution_count":null,"outputs":[]},{"metadata":{"id":"UQvKf1t-xyDM","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":518},"outputId":"1b0b75ca-1cbb-46e3-8c14-24d86e04fee4","trusted":true},"cell_type":"code","source":"plot_dist(weather_train[weather_train['precip_depth_1_hr'] > 0], 'precip_depth_1_hr')","execution_count":null,"outputs":[]},{"metadata":{"id":"sh1yHq3QrLS1","colab_type":"text"},"cell_type":"markdown","source":"### Data processing pipeline\n  - Fixing timestamp\n  - Replacing missing values\n  - Label encoding\n  <!-- - Scaling continuous features -->\n  - Feature engineering\n  - Merging datasets\n  - Reducing memory usage"},{"metadata":{"id":"SZGQli1KMYXN","colab_type":"text"},"cell_type":"markdown","source":"#### Data processing for building dataset"},{"metadata":{"id":"Y2WvFxspIXTR","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def process_building_data(building_df):\n  # print('Data Cleaning: Deleting floor_count column')\n  # building_df.drop(columns=['floor_count'], inplace=True)\n  print('Data cleaning: Replacing null values for floor_count column')\n  building_df.floor_count.fillna(round(building_df.floor_count.mean(),0), inplace=True)\n  \n  print('Feature engineering: Creating building_age column')\n  current_year = datetime.now().year\n  building_df['building_age'] = current_year - building_df['year_built']\n    \n  print('Data Cleaning: Deleting floor_count column')\n  building_df.drop(columns=['year_built'], inplace=True)\n\n  print('Data cleaning: Replacing null values in building_age column')\n  building_df.building_age.fillna(round(building_df.building_age.mean(),0), inplace=True)\n\n  print('Feature engineering: Label encoding primary use')\n  le = LabelEncoder()\n  le_primary_use = le.fit_transform(building.primary_use)\n\n  print('Feature engineering: Log normalization for square_feet column')\n  building_df['log_square_feet'] = np.log(building_df['square_feet'])\n  building_df.drop(columns=['square_feet'], inplace=True)\n\n  building_df['primary_use'] = le_primary_use\n\n  del current_year, le, le_primary_use\n  gc.collect()\n\n  return building_df","execution_count":null,"outputs":[]},{"metadata":{"id":"K7-Gvgf1MdRQ","colab_type":"text"},"cell_type":"markdown","source":"#### Data processing for weather dataset"},{"metadata":{"id":"1o0vB7jaMHAn","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def convert_season(month):\n    if (month <= 2) | (month == 12):\n        return 0\n    # as winter\n    elif month <= 5:\n        return 1\n    # as spring\n    elif month <= 8:\n        return 2\n    # as summer\n    elif month <= 11:\n        return 3\n    # as fall\n\ndef convert_direction(degrees):\n    if degrees <= 90:\n        return 0\n    # as norteast direction\n    elif degrees <= 180:\n        return 1\n    # as southeast direction\n    elif degrees <= 270:\n        return 2\n    # as southwest direction\n    elif degrees <= 360:\n        return 3\n    # as northwest direction","execution_count":null,"outputs":[]},{"metadata":{"id":"YiHaXH_VIq4g","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def add_beaufort_scale(df):\n  beaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8), (6, 10.8, 13.9), \n          (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5), (11, 28.5, 33), (12, 33, 200)]\n\n  for item in beaufort:\n    df.loc[(df['wind_speed'] >= item[1]) & (df['wind_speed'] < item[2]), 'beaufort_scale'] = item[0]\n  return df","execution_count":null,"outputs":[]},{"metadata":{"id":"NXTywUScOqem","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def replace_weather_data_nans(weather_df):\n  weather_df = weather_df.set_index(['site_id','day_of_month','month'])\n                        \n  weather_features_with_nan = {\n      'air_temperature': {'round': 1, 'fill': False},\n      'dew_temperature': {'round': 1, 'fill': False},\n      'wind_speed': {'round': 1, 'fill': False},\n      'wind_direction': {'round': 0, 'fill': False},\n      'sea_level_pressure': {'round': 1, 'fill': True},\n      'precip_depth_1_hr': {'round': 0, 'fill': True},\n      'cloud_coverage': {'round': 0, 'fill': True}\n  }\n\n  for feature, params in weather_features_with_nan.items():\n    print('Data cleaning: Replacing NAN values for ' + feature)\n    filler_df = pd.DataFrame(weather_df.groupby(['site_id','day_of_month','month'])[feature].mean().round(params['round']),\n                             columns=[feature])\n\n    if params['fill']:\n      filler_df_mean = round(filler_df[feature].mean(),params['round'])\n      filler_df.fillna(filler_df_mean, inplace=True)\n    # create dataframe of air_temperatures to fill\n    temporary_df = pd.DataFrame({feature : weather_df[feature]})\n\n    # update NA air_temperature values\n    temporary_df.update(filler_df, overwrite=False)\n\n    # update in the weather dataset\n    weather_df[feature] = temporary_df[feature]\n  weather_df = weather_df.reset_index()\n  return weather_df","execution_count":null,"outputs":[]},{"metadata":{"id":"Y1QG5iauKIJU","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def process_weather_data(weather_df):\n  print('Data cleaning: Converting timestamp column')\n  weather_df[\"timestamp\"] = pd.to_datetime(weather_df[\"timestamp\"])\n\n  print('Feature engineering: Adding month, day of week, day of month and hour columns')\n  weather_df['month'] = weather_df['timestamp'].dt.month\n  weather_df['day_of_week'] = weather_df['timestamp'].dt.dayofweek\n  weather_df['day_of_month']= weather_df['timestamp'].dt.day\n  weather_df['hour'] = weather_df['timestamp'].dt.hour\n\n  print('Feature engineering: Adding is_weekend feature')\n  weather_df['is_weekend'] = weather_df.day_of_week.apply(lambda x: 1 if x>=5 else 0)\n\n  print('Feature engineering: Adding season feature')\n  weather_df['season'] = weather_df.month.apply(convert_season)\n\n  print('Data cleaning: Replacing NAN values')\n  weather_df = replace_weather_data_nans(weather_df)\n\n  print('Feature engineering: Adding beaufort_scale feature')\n  weather_df = add_beaufort_scale(weather_df)\n\n  print('Feature engineering: adding wind_compass_direction feature')\n  weather_df['wind_direction'+'_compass'] = weather_df.wind_direction.apply(convert_direction)\n\n  print('Feature engineering: adding wind_direction_sin features')\n  weather_df['wind_direction'+'_sin'] = np.sin((2*np.pi*weather_df['wind_direction'])/360)\n\n  print('Data cleaning: Dropping wind_direction feature')\n  weather_df.drop(columns=['wind_direction'], inplace=True)\n\n  return weather_df","execution_count":null,"outputs":[]},{"metadata":{"id":"QWFFbNQTeeWC","colab_type":"text"},"cell_type":"markdown","source":"#### Train dataset processing\n\n*   Merging datasets\n*   Reducing memory usage\n\n\n\n"},{"metadata":{"id":"g0FJiNmA0COZ","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def reduce_memory_usage(df):\n  d_types = {'building_id': np.uint16,\n            'site_id': np.uint8,\n            'meter': np.uint8,\n            'primary_use': np.uint8,\n            'log_square_feet': np.float16,\n            'floor_count': np.uint8,\n            'building_age': np.uint8,\n            'air_temperature': np.float16,\n            'dew_temperature': np.float16,\n            'wind_speed': np.float16,\n            'cloud_coverage': np.uint8,\n            'precip_depth_1_hr': np.int16,\n            'sea_level_pressure': np.float32,\n            'wind_direction_sin': np.float16,\n            'wind_direction_compass': np.uint8,\n            'beaufort_scale': np.uint8,\n            'day_of_month': np.uint8,\n            'month': np.uint8,\n            'day_of_week': np.uint8,\n            'hour': np.uint8,\n            'is_weekend': np.uint8,\n            'season': np.uint8\n            }\n\n  for feature in d_types:\n    df[feature] = df[feature].astype(d_types[feature])\n  return df\n","execution_count":null,"outputs":[]},{"metadata":{"id":"WiwV4DhPrUYS","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def data_processing_pipeline(df, weather_df, building_df, target_variable='meter_reading'):\n\n  print('Data cleaning: Fixing timestamp')\n  df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n  \n  print('\\nProcessing building data:\\n')\n  building_df_copy = building_df.copy()\n  building_df_copy = process_building_data(building_df_copy)\n\n  print('\\nProcessing weather data:\\n')\n  weather_df_copy = weather_df.copy()\n  weather_df_copy = process_weather_data(weather_df_copy)\n\n  print('\\nMerging datasets:\\n')\n  print('Merging with building dataset')\n  df = df.merge(building_df_copy, on='building_id', how='left')\n  print('Merging with weather dataset')\n  df = df.merge(weather_df_copy, on=['site_id', 'timestamp'], how='left')\n\n  print('\\nData cleaning: Imputing missing data after merging datasets')\n  columns_with_nan = df.columns[df.isna().any()]\n  for feature in columns_with_nan:\n    df[feature] = df[feature].fillna(method='ffill')\n\n  print('\\nReducing memory usage:\\n')\n  start_mem = df.memory_usage().sum() / 1024**2\n  print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n  df = reduce_memory_usage(df)\n  end_mem = df.memory_usage().sum() / 1024**2\n  print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n  print('Decreased by {:.1f}%\\n'.format(100 * (start_mem - end_mem) / start_mem))\n\n  del weather_df_copy, building_df_copy, start_mem, end_mem\n  gc.collect()\n  return df","execution_count":null,"outputs":[]},{"metadata":{"id":"Raeb_jKmwnge","colab_type":"code","outputId":"4d14574b-774c-4d2d-d919-0b4848fdfc92","colab":{"base_uri":"https://localhost:8080/","height":782},"trusted":true},"cell_type":"code","source":"%%time\ntrain_copy = data_processing_pipeline(train, weather_train, building)\ndel train, weather_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"id":"a4d6eru-yJqR","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"2c97ff4d-e7e2-4aff-c246-0c9d3e480d4d","trusted":true},"cell_type":"code","source":"corr_mat = train_copy.corr()\nfig, ax = plt.subplots(figsize=(20,20))\nsns.heatmap(corr_mat, annot=True, annot_kws={'size': 12})","execution_count":null,"outputs":[]},{"metadata":{"id":"UXBBMuow0EOk","colab_type":"text"},"cell_type":"markdown","source":"### Modeling\n  - Trying Gradient Boosting ML Model\n  - Trying Deep Neural Network\n  - Hyperparameter tuning and K-Fold pipeline\n  - Trying LSTM Network (WIP)\n"},{"metadata":{"id":"P3FPXJlj1Djd","colab_type":"text"},"cell_type":"markdown","source":"#### Train-test split"},{"metadata":{"id":"49V9aTUMHMlN","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"unnecessary_features = ['timestamp', 'day_of_month']\n\ncategoricals = ['building_id', 'site_id', 'meter', 'primary_use', 'floor_count',\n                'cloud_coverage', 'beaufort_scale', 'wind_direction_compass',\n                'day_of_week', 'hour', 'is_weekend', 'season']\n\nnumericals = [\"building_age\", \"log_square_feet\", \"air_temperature\", \"dew_temperature\",\n              \"precip_depth_1_hr\", \"sea_level_pressure\", \"wind_speed\", \"wind_direction_sin\"]\n\ntarget = \"meter_reading\"\n\nfeatures = categoricals + numericals","execution_count":null,"outputs":[]},{"metadata":{"id":"PoD_7dn1_ZHF","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# TRAIN_SIZE = 1047552\n# train_sample = train.sample(TRAIN_SIZE)\n\nX = train_copy[features]\ny = train_copy[target].map(np.log1p)","execution_count":null,"outputs":[]},{"metadata":{"id":"hLSP5eQg_2_B","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"id":"zWGJBUr5G4hj","colab_type":"text"},"cell_type":"markdown","source":"#### LightGBM Model"},{"metadata":{"id":"E7yVxn2EG78u","colab_type":"code","outputId":"b2301c04-56fd-40e1-8d6c-69bbb964a059","colab":{"base_uri":"https://localhost:8080/","height":326},"trusted":true},"cell_type":"code","source":"params = {\n            'boosting_type': 'gbdt',\n            'objective': 'regression',\n            'metric': {'rmse'},\n            'subsample_freq': 1,\n            'learning_rate': 0.3,\n            'bagging_freq': 5,\n            'num_leaves': 330,\n            'feature_fraction': 0.9,\n            'lambda_l1': 1,  \n            'lambda_l2': 1\n            }\n\nseed = 42\nshuffle = False\n\n\nlgb_train = lgb.Dataset(X_train, y_train, categorical_feature=categoricals)\nlgb_eval = lgb.Dataset(X_test, y_test, categorical_feature=categoricals)\n\ngbm = lgb.train(params,\n                lgb_train,\n                num_boost_round=500,\n                valid_sets=(lgb_train, lgb_eval),\n                early_stopping_rounds=50,\n                verbose_eval = 50\n)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"rBY-hiUMbJ4r","colab_type":"code","outputId":"4adb8859-b883-48c7-a129-64b90c4453fe","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"lgb.create_tree_digraph(gbm)","execution_count":null,"outputs":[]},{"metadata":{"id":"sxmb4cEBEWjR","colab_type":"text"},"cell_type":"markdown","source":"#### Neural Network Model"},{"metadata":{"id":"Rn3OQM13_-wg","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))\n\ndef create_model(dense_layer_sizes, opt=\"adam\", dropout=0.1, nbr_features=20, dense_nparams=256, lr=0.01):\n    optimizers = {\n        'adam': Adam(lr),\n        'rmsprop': RMSprop(lr)\n    }\n    model = Sequential()\n    model.add(Dense(dense_nparams, activation='relu', input_shape=(nbr_features,))) \n    model.add(BatchNormalization())\n    model.add(Dropout(dropout))\n    for layer_size in dense_layer_sizes:\n        model.add(Dense(layer_size, activation='relu'))\n        model.add(BatchNormalization())\n        model.add(Dropout(dropout))\n    model.add(Dense(1, activation='relu'))\n    model.compile(loss='mse', optimizer=optimizers[opt], metrics=[root_mean_squared_error])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"id":"0DXXpm6bABL4","colab_type":"code","outputId":"39299597-eb9b-4433-c0c4-be033981c8ff","colab":{"base_uri":"https://localhost:8080/","height":88},"trusted":true},"cell_type":"code","source":"model = create_model(dense_layer_sizes=[256, 128, 64, 32], dense_nparams=512, dropout=0.2)","execution_count":null,"outputs":[]},{"metadata":{"id":"JchRTpGPKTcc","colab_type":"code","outputId":"09a44318-abeb-44cc-a31e-3666fb984ce5","colab":{"base_uri":"https://localhost:8080/","height":697},"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"id":"5LlaZX4qK5zv","colab_type":"code","outputId":"30f2e32d-5015-4cfc-8bc1-f9a1709865fe","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"plot_model(model)","execution_count":null,"outputs":[]},{"metadata":{"id":"Gz1RppZYJSa4","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"EPOCHS = 50\nBATCH_SIZE = 1024","execution_count":null,"outputs":[]},{"metadata":{"id":"9OPkDbfUJKvd","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def step_decay(epoch):\n   initial_lrate = 0.01\n   drop = 0.9\n   lrate = initial_lrate * math.pow(drop, math.floor((epoch)))\n   return lrate\n\nbest_model_file = \"/kaggle/working/Models/epochs:{epoch:03d}-val_loss:{val_loss:.3f}.h5\"\n# best_model_file = \"epochs:{epoch:03d}-val_loss:{val_loss:.3f}.h5\"\n\nmc = ModelCheckpoint(best_model_file, monitor='val_loss', mode='auto',verbose=True, save_best_only=True)\nes = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=5, verbose=True, mode='auto', restore_best_weights=True)\nrlr = ReduceLROnPlateau(monitor='val_loss', patience=2, factor=0.5, verbose=True, min_delta=0.001, min_lr=1e-5)\nlrs = LearningRateScheduler(step_decay, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"Bo80DLQqAIOs","colab_type":"code","outputId":"16a00a44-b428-439e-a6e2-15c45a50e027","colab":{"base_uri":"https://localhost:8080/","height":598},"trusted":true},"cell_type":"code","source":"hist = model.fit(X_train.values, y_train.values, epochs=EPOCHS, batch_size=BATCH_SIZE, \n                 validation_data=(X_test.values, y_test.values), verbose=True, callbacks=[es, mc, rlr])","execution_count":null,"outputs":[]},{"metadata":{"id":"y40CBIAgBAx9","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"best_model_filepath = \"/kaggle/working/Models/epochs:026-val_loss:1.351.h5\"\ndependencies = {\n    'root_mean_squared_error': root_mean_squared_error\n}\n\nbest_model = load_model(best_model_filepath, custom_objects=dependencies)","execution_count":null,"outputs":[]},{"metadata":{"id":"KkWX1RPKtFnP","colab_type":"text"},"cell_type":"markdown","source":"#### Hyperparameter tuning and KFold Cross validation pipeline"},{"metadata":{"id":"PncbhAW8tOrO","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"keras_estimator = KerasRegressor(create_model, epochs=20, batch_size=1024, verbose=2)\nkeras_estimator.get_params()","execution_count":null,"outputs":[]},{"metadata":{"id":"2PVY51tntVix","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"dense_size_candidates = [[64, 32], [128, 64, 32], [256, 128, 64, 32], \n                         [256, 128, 64, 64, 32, 16], [512, 256, 128, 128, 64, 32]]\nparam_grid = {\n    'dense_nparams': [128, 256, 384, 512, 640, 768, 896, 1024],\n    'dense_layer_sizes': dense_size_candidates,\n    'opt':['rmsprop', 'adam'],\n    'lr': [0.001, 0.002, 0.05, 0.01],\n    'dropout': [0.5, 0.3, 0.2, 0.1, 0]\n}","execution_count":null,"outputs":[]},{"metadata":{"id":"remh6nb_t1Ru","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"kfold_splits = 4\ntuner = RandomizedSearchCV(estimator=keras_estimator,\n                    verbose=True,\n                    n_iter=25,\n                    n_jobs=-1,\n                    return_train_score=True,\n                    cv=kfold_splits,\n                    param_distributions=param_grid)","execution_count":null,"outputs":[]},{"metadata":{"id":"419zayk0t-ta","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"# es = EarlyStopping(monitor='loss', min_delta=0.01, patience=5, verbose=True, mode='auto', restore_best_weights=True)\n# rlr = ReduceLROnPlateau(monitor='loss', patience=2, factor=0.75, verbose=True, min_delta=0.01, min_lr=1e-5)\n\n# tuner_result = tuner.fit(X_train, y_train, callbacks=[es, rlr])\n\n# # summarize results\n# print(\"Best: %f using %s\" % (tuner_result.best_score_, tuner_result.best_params_))\n# means = tuner_result.cv_results_['mean_test_score']\n# stds = tuner_result.cv_results_['std_test_score']\n# params = tuner_result.cv_results_['params']\n# for mean, stdev, param in zip(means, stds, params):\n#     print(\"%f (%f) with: %r\" % (mean, stdev, param))","execution_count":null,"outputs":[]},{"metadata":{"id":"s9Or5ZriuZ-v","colab_type":"text"},"cell_type":"markdown","source":"#### Trying LSTM Network (TODO)"},{"metadata":{"id":"gtQ1t8so-u0x","colab_type":"text"},"cell_type":"markdown","source":"### Testing and submitting results\n\n\n*   Applying data processing pipeline on test dataset\n*   Getting predictions of test data\n*   Submitting results using Kaggle CLI\n\n"},{"metadata":{"id":"4-9l72L-_rky","colab_type":"text"},"cell_type":"markdown","source":"#### Data processing pipeline on test dataset"},{"metadata":{"id":"c0snJizB-whP","colab_type":"code","outputId":"5a323eab-6cd3-4424-d105-fffb4ecb0f4b","colab":{"base_uri":"https://localhost:8080/","height":782},"trusted":true},"cell_type":"code","source":"%%time\ntest_copy = data_processing_pipeline(test, weather_test, building)","execution_count":null,"outputs":[]},{"metadata":{"id":"A5_Jkvi2_vAH","colab_type":"text"},"cell_type":"markdown","source":"#### Getting predictions"},{"metadata":{"id":"eVbrvWbW_Opo","colab_type":"code","outputId":"3e1ba124-7c9b-4b02-9a83-a9625022992a","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"test_data = test_copy[features]\ny_pred = best_model.predict(test_data.values, batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"id":"CvK4KQZ9AJ1Z","colab_type":"text"},"cell_type":"markdown","source":"#### Creating submission .csv and submitting results"},{"metadata":{"id":"gfdqwRFbANjd","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"sample_submission['meter_reading'] = np.expm1(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"id":"cRRACLt-AQTD","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"export_csv = sample_submission.to_csv('./my-submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"id":"Qu4UtlfaAR5Q","colab_type":"code","outputId":"10f30416-f7a8-4064-f0b5-b3c04ea92e67","colab":{"base_uri":"https://localhost:8080/","height":68},"trusted":true},"cell_type":"code","source":"# !kaggle competitions submit ashrae-energy-prediction -f my-submission.csv -m 'Final submission'","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"ASHARE Great Energy Predictor.ipynb","provenance":[],"collapsed_sections":["vYGqlqln2WYP","sh1yHq3QrLS1","SZGQli1KMYXN","K7-Gvgf1MdRQ"],"toc_visible":true,"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"TPU"},"nbformat":4,"nbformat_minor":1}