{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n%matplotlib inline\npd.set_option('display.max_columns', 500)\npd.set_option('display.max_rows', 500)\nplt.style.use('bmh')\n# mpl.rcParams['axes.labelcolor'] = 'grey'\n# mpl.rcParams['xtick.color'] = 'grey'\nmpl.rcParams['xtick.labelsize'] = 'large'\n# mpl.rcParams['ytick.color'] = 'grey'\n# mpl.rcParams['axes.labelcolor'] = 'grey'\n# mpl.rcParams['text.color'] = 'grey'\nmpl.rcParams[\"legend.loc\"] = 'best'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"buildings = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\nweather_train = pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/weather_train.csv\")\nweather_test = pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/weather_test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BUILDING"},{"metadata":{"trusted":true},"cell_type":"code","source":"buildings['age'] = 2016 - buildings['year_built']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"sns.distplot(buildings.age.dropna());","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"sns.distplot(buildings.square_feet.dropna());","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_counts(df, col, n):\n    cnt = df[col].value_counts().nlargest(n)[::-1]\n    fig, ax = plt.subplots()\n    ax.barh(cnt.index.astype(str), cnt.values)\n    ax.set_xlabel('count')\n    ax.set_title(col, color='white')\n    for i, v in enumerate(cnt):\n        ax.text(v + 3, i, str(int(v/cnt.sum() * 100)) + \"%\", color='grey', fontweight='bold')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"show_counts(buildings, 'primary_use', 5);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"show_counts(buildings, 'floor_count', 7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# age"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compare_kde(df, col, n):\n    fig, axes = plt.subplots(nrows=n, ncols=1, figsize=(9, 10), sharex=True)\n    most_use_case = buildings.primary_use.value_counts().nlargest(n).index\n    for i, use_case in enumerate(most_use_case):\n        sns.distplot(buildings.loc[buildings.primary_use==use_case, col].dropna(), ax=axes[i]);\n        axes[i].set_title(use_case)\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"compare_kde(buildings, 'age', 5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# floor count"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"compare_kde(buildings, 'floor_count', 5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# square feet"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"compare_kde(buildings, 'square_feet', 5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# WEATHER (We'll draw Train set in Blue, and Test Set in Green)"},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_sites = weather_train.site_id.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"calculate_nans = ['site_id', 'air_temperature', 'cloud_coverage', 'dew_temperature',\n                  'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction',\n                  'wind_speed']\nweather_concat = pd.concat([weather_train, weather_test], axis=0)\ncalculate_nans_df = weather_concat[calculate_nans].isna()\ncalculate_nans_df['site_id'] = weather_concat.site_id\ncalculate_nans_df = calculate_nans_df.groupby('site_id').sum() / (24 * 365 * 3) * 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add time attribute\n# Only Month, HourofDay have something to do with weather\nweather_train['timestamp'] = pd.to_datetime(weather_train['timestamp'])\nweather_test['timestamp'] = pd.to_datetime(weather_test['timestamp'])\n\nweather_train['hour'] = weather_train.timestamp.dt.hour\nweather_test['hour'] = weather_test.timestamp.dt.hour\n\nweather_train['month'] = weather_train.timestamp.dt.month\nweather_test['month'] = weather_test.timestamp.dt.month\n\nweather_train = weather_train.set_index('timestamp')\nweather_test = weather_test.set_index('timestamp')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_weather_attribute(col, rolling_window=60):\n    fig, axes = plt.subplots(nrows=6, ncols=3, sharex=True, sharey=True, figsize=(18, 12))\n    exclude_site = [c for c in weather_train.columns if c != 'site_id']\n    rolling_weather_train = weather_train[exclude_site].rolling(rolling_window).mean()\n    rolling_weather_train['site_id'] = weather_train.site_id\n\n    rolling_weather_test = weather_test[exclude_site].rolling(rolling_window).mean()\n    rolling_weather_test['site_id'] = weather_test.site_id\n\n    for i in range(6):\n        for j in range(3):\n            site_idx = i * 3 + j\n            if (site_idx) > 15:\n                break\n            rolling_weather_train.loc[rolling_weather_train.site_id==(site_idx), col].plot(ax=axes[i, j])\n            rolling_weather_test.loc[rolling_weather_test.site_id==(site_idx), col].plot(ax=axes[i, j], color='green')\n\n            axes[i, j].set_title(f\"Site {site_idx} NULL: {int(calculate_nans_df.loc[site_idx, col])}%\")\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# air_temperature:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_attribute('air_temperature', rolling_window=60)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Site 4 is the most stable"},{"metadata":{},"cell_type":"markdown","source":"# cloud_coverage:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_attribute('cloud_coverage', rolling_window=7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Only Site 12 has the full data"},{"metadata":{},"cell_type":"markdown","source":"# dew_temperature:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_attribute('dew_temperature', rolling_window=60)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Look pretty similar to air_temperature"},{"metadata":{},"cell_type":"markdown","source":"# precip_depth_1_hr:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_attribute('precip_depth_1_hr', rolling_window=7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# sea_level_pressure:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_attribute('sea_level_pressure', rolling_window=7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# wind_direction:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_attribute('wind_direction', rolling_window=60)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Site 4 has same pattern with temparature (monsoons?)\n# Should be 0 to 360 degree"},{"metadata":{},"cell_type":"markdown","source":"# wind_speed:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_attribute('wind_speed', rolling_window=60)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Then we do EDA by HOUR OF THE DAY......"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_site_hour_mean = weather_train.groupby(['site_id', 'hour']).mean()\ntest_site_hour_mean = weather_test.groupby(['site_id', 'hour']).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_weather_by_hour(col):\n    fig, axes = plt.subplots(nrows=6, ncols=3, sharex=True, sharey=True, figsize=(18, 12))\n    for i in range(6):\n        for j in range(3):\n            site_idx = (i * 3 + j)\n            if site_idx > 15:\n                break\n            train_site_data = train_site_hour_mean.loc[(site_idx, slice(None)), col]\n            train_site_data.index = train_site_data.index.droplevel(0)\n\n            test_site_data = test_site_hour_mean.loc[(site_idx, slice(None)), col]\n            test_site_data.index = test_site_data.index.droplevel(0)\n\n            train_site_data.plot(ax=axes[i, j])\n            test_site_data.plot(ax=axes[i, j], color='green')\n\n            axes[i, j].set_title(f\"Site {site_idx} NULL: {int(calculate_nans_df.loc[site_idx, col])}%\")\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# air_temperature:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_hour('air_temperature')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# cloud_coverage:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_hour('cloud_coverage')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# There are some weird spikes...\n# Let's check if it's related to the NULLs"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"check_cloud_na = weather_train[['site_id', 'hour', 'cloud_coverage']].copy()\ncheck_cloud_na.loc[:, 'cloud_coverage'] = check_cloud_na['cloud_coverage'].isna()\ncheck_cloud_na.groupby(['site_id', 'hour']).sum().unstack('hour')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# --> Yes, the spikes is just because of the Nulls"},{"metadata":{},"cell_type":"markdown","source":"# dew_temperature:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_hour('dew_temperature')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# precip_depth_1_hr:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_hour('precip_depth_1_hr')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Though Site 7, 11 look weird\n# It's because there are around 90% null in them\n# ---> It's biased"},{"metadata":{},"cell_type":"markdown","source":"# sea_level_pressure:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_hour('sea_level_pressure')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# wind_direction:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_hour('wind_direction')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Site 2, 4, 15 might be near the ocean\n# as the wind direction is opposite in day and night"},{"metadata":{},"cell_type":"markdown","source":"# wind_speed:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_hour('wind_speed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Then we do EDA by Month...."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_site_month_mean = weather_train.groupby(['site_id', 'month']).mean()\ntest_site_month_mean = weather_test.groupby(['site_id', 'month']).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_weather_by_month(col):\n    fig, axes = plt.subplots(nrows=6, ncols=3, sharex=True, sharey=True, figsize=(18, 12))\n    for i in range(6):\n        for j in range(3):\n            site_idx = (i * 3 + j)\n            if site_idx > 15:\n                break\n            train_site_data = train_site_month_mean.loc[(site_idx, slice(None)), col]\n            train_site_data.index = train_site_data.index.droplevel(0)\n\n            test_site_data = test_site_month_mean.loc[(site_idx, slice(None)), col]\n            test_site_data.index = test_site_data.index.droplevel(0)\n\n            train_site_data.plot(ax=axes[i, j])\n            test_site_data.plot(ax=axes[i, j], color='green')\n\n            axes[i, j].set_title(f\"Site {site_idx} NULL: {int(calculate_nans_df.loc[site_idx, col])}%\")\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# air_temperature:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_month('air_temperature')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# cloud_coverage:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_month('cloud_coverage')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# dew_temperature:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_month('dew_temperature')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# precip_depth_1_hr:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_month('precip_depth_1_hr')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# sea_level_pressure:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_month('sea_level_pressure')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# wind_direction:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_month('wind_direction')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# wind_speed:"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_by_month('wind_speed')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Then Let's see how attributes interact with each other"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_weather_correlation(df):\n    fig, axes = plt.subplots(nrows=6, ncols=3, sharex=True, sharey=True, figsize=(18, 24))\n    for i in range(6):\n        for j in range(3):\n            site_idx = (i * 3 + j)\n            if site_idx > 15:\n                for tick in axes[i, j].get_xticklabels():\n                    tick.set_rotation(90)\n                continue\n                \n            sns.heatmap(df.loc[df.site_id==site_idx, \n                                          ['air_temperature', 'cloud_coverage', 'dew_temperature',\n                                          'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction',\n                                          'wind_speed']\n                                         ]\\\n                                         .corr(), ax=axes[i, j], annot=True, fmt ='.1f')\n            axes[i, j].set_title(f\"Site {site_idx}\")\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# In Train set"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(weather_train[['air_temperature', 'cloud_coverage', 'dew_temperature',\n               'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction',\n               'wind_speed']].corr(), annot=True, fmt ='.2f');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# In Test Set"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(weather_test[['air_temperature', 'cloud_coverage', 'dew_temperature',\n               'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction',\n               'wind_speed']].corr(), annot=True, fmt ='.2f');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# In Train Set by Site"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_weather_correlation(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# In Test Set by Site","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_weather_correlation(weather_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Conclusions on Buildings:\n# 1. Not sure what's the cause of the NULLs\n# 2. The Top5 most common building type might be important when performing further analysis\n\n# Conclusions on weather:\n# 1. All sites are from Northern Hemisphere\n# 2. Mostly Train and Test data share the same pattern\n# 3. Beware of the NULLs\n# 4. Sites can be identified if correctly identify the temperature, wind direction hourly/monthly\n\n# **Conclusion:**\n# Thank you for reading. Hope this help.\n# Please let me know if there's any mistake and have a good day!"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}