{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings('ignore')\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_build = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\ndf_weather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\ndf_train = pd.read_csv('../input/ashrae-energy-prediction/train.csv')\n\ndf_weather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')\ndf_test = pd.read_csv('../input/ashrae-energy-prediction/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_build.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Building Metadata', df_build.shape)\nprint('Train Data', df_train.shape)\nprint('Weather Train Data', df_weather_train.shape)\n\nprint('Test Data', df_test.shape)\nprint('Weather Test Data', df_weather_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_weather_train.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's change the datatype of timestamp column to datetime format"},{"metadata":{"trusted":true},"cell_type":"code","source":"def change_dtype(df):\n    df['timestamp'] = pd.to_datetime(df['timestamp'], format='%Y-%m-%d %H:%M:%S')\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = change_dtype(df_train)\ndf_weather_train = change_dtype(df_weather_train)\ndf_test = change_dtype(df_test)\ndf_weather_test = change_dtype(df_weather_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.merge(df_build, on='building_id', how='left')\ndf_train = df_train.merge(df_weather_train, on=['site_id', 'timestamp'], how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.merge(df_build, on='building_id', how='left')\ndf_test = df_test.merge(df_weather_test, on=['site_id', 'timestamp'], how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del df_build, df_weather_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = reduce_mem_usage(df_train)\ndf_test = reduce_mem_usage(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_weather_train = reduce_mem_usage(df_weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Let's Proceed ahead with EDA"},{"metadata":{"trusted":false},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\ncolor = sns.color_palette()\nsns.set_style(\"darkgrid\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Dataset is huge we have around 20M datapoints__"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see there are some features whose null count is quite high, we will deal with them later"},{"metadata":{"trusted":false},"cell_type":"code","source":"len(df_train['building_id'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have the data for __1449__ buildings "},{"metadata":{"trusted":false},"cell_type":"code","source":"print(f\"Max number of entries for is for building {df_train['building_id'].value_counts().keys()[0]}: \", df_train['building_id'].value_counts().values[0])\nprint(f\"Min number of entries for is for building {df_train['building_id'].value_counts().keys()[-1]}: \", df_train['building_id'].value_counts().values[-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"Data in the train set is from:\", df_train['timestamp'].min())\nprint(\"Data in the train set is till:\", df_train['timestamp'].max())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So we have one year of data in our train set."},{"metadata":{},"cell_type":"markdown","source":"#### For test set"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_test['building_id'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(f\"Max number of entries for is for building {df_test['building_id'].value_counts().keys()[0]}: \", df_test['building_id'].value_counts().values[0])\nprint(f\"Min number of entries for is for building {df_test['building_id'].value_counts().keys()[-1]}: \", df_test['building_id'].value_counts().values[-1])\nprint(f\"Data in the train set is from:\", df_test['timestamp'].min())\nprint(f\"Data in the train set is till:\", df_test['timestamp'].max())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The test set has 2 years of data"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train['meter'].value_counts().plot.bar(figsize=(8,4), color=color)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see most ereadings is for __Electricity meter__ followed by __Chilledwater, Steam and Hotwater__."},{"metadata":{},"cell_type":"markdown","source":"Let's see the distribution of different meter readings across the dataset"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 12))\n\nax1 = fig.add_subplot(221)\nsns.distplot(np.log1p(df_train[df_train['meter'] == 0]['meter_reading'].values),color='blue',ax=ax1, axlabel=\"Log of Electricity meter reading\")\nax2 = fig.add_subplot(222)\nsns.distplot(np.log1p(df_train[df_train['meter'] == 1]['meter_reading'].values),color='orange',ax=ax2, axlabel=\"Log of Chilledwater meter reading\")\nax3 = fig.add_subplot(223)\nsns.distplot(np.log1p(df_train[df_train['meter'] == 2]['meter_reading'].values),color='green',ax=ax3, axlabel=\"Log of Steam meter reading\")\nax4 = fig.add_subplot(224)\nsns.distplot(np.log1p(df_train[df_train['meter'] == 3]['meter_reading'].values),color='red',ax=ax4, axlabel=\"Log of Hotwater meter reading\")\n\nax1.title.set_text('Meter Reading Distribution for Electricity Meter')\nax2.title.set_text('Meter Reading Distribution for Chilledwater Meter')\nax3.title.set_text('Meter Reading Distribution for Steam Meter')\nax4.title.set_text('Meter Reading Distribution for Hotwater Meter')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Let's check number of meters buildings are having"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_temp = df_train.groupby(['building_id']).agg({\"meter\": [\"nunique\", \"count\"]})\ndf_temp.columns = [\"_\".join(i) for i in df_temp.columns.ravel()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_temp.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_temp['meter_nunique'].value_counts().plot.bar(figsize=(8,4), color=color)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<b> Most of the buildings have only 1 type of meter"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_temp[df_temp['meter_nunique']==4]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<b> Let's check energy consumption of some of the buildings to find energy consumtion is following a similar pattern</b>\n* We will take building 1258 as it has the maximum number of entries in the dataset and 1331"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_temp = df_train[df_train['building_id']==1258]\nfig = plt.figure(figsize=(16, 16))\n\nax1 = fig.add_subplot(411)\ndf = df_temp[df_temp['meter']==0]\nsns.lineplot(x=df['timestamp'], y=df['meter_reading'], ax=ax1, color='green')\nax2 = fig.add_subplot(412)\ndf = df_temp[df_temp['meter']==1]\nsns.lineplot(x=df['timestamp'], y=df['meter_reading'], ax=ax2, color='green')\nax3 = fig.add_subplot(413)\ndf = df_temp[df_temp['meter']==2]\nsns.lineplot(x=df['timestamp'], y=df['meter_reading'], ax=ax3, color='green')\nax4 = fig.add_subplot(414)\ndf = df_temp[df_temp['meter']==3]\nsns.lineplot(x=df['timestamp'], y=df['meter_reading'], ax=ax4, color='green')\n\nplt.tight_layout()\nplt.subplots_adjust(top=1.5)\nax1.title.set_text('Meter Reading for Electricity Meter')\nax2.title.set_text('Meter Reading for Chilledwater Meter')\nax3.title.set_text('Meter Reading for Steam Meter')\nax4.title.set_text('Meter Reading for Hotwater Meter')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Observations:__\n* The electricity meter readings are generally in the range of 800 to 1100 but becomes 0 at times in between which .\n* We can see an increase in the chill water meter from May to October of 2016 for this building and probably this might be summer time\n* We can see a dip in the steam meter from October till November\n* We can see a spike in the hot water meter from November till February, suggesting winter has come :P\n* Hot water meter reading is around 0 from May to October"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_temp = df_train[df_train['building_id']==1331]\nfig = plt.figure(figsize=(16, 16))\n\nax1 = fig.add_subplot(411)\ndf = df_temp[df_temp['meter']==0]\nsns.lineplot(x=df['timestamp'], y=df['meter_reading'], ax=ax1)\nax2 = fig.add_subplot(412)\ndf = df_temp[df_temp['meter']==1]\nsns.lineplot(x=df['timestamp'], y=df['meter_reading'], ax=ax2)\nax3 = fig.add_subplot(413)\ndf = df_temp[df_temp['meter']==2]\nsns.lineplot(x=df['timestamp'], y=df['meter_reading'], ax=ax3)\nax4 = fig.add_subplot(414)\ndf = df_temp[df_temp['meter']==3]\nsns.lineplot(x=df['timestamp'], y=df['meter_reading'], ax=ax4)\n\nplt.tight_layout()\nplt.subplots_adjust(top=1.5)\nax1.title.set_text('Meter Reading for Electricity Meter')\nax2.title.set_text('Meter Reading for Chilledwater Meter')\nax3.title.set_text('Meter Reading for Steam Meter')\nax4.title.set_text('Meter Reading for Hotwater Meter')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Observations:__\n\n* Their is sudden linear decrease in the electricity meter reading from Feburary mid to April mid, this suggest their might be a meter fault.\n* Chilled water reading meter reading reamains constant from Feburary mid to April mid\n* Steam meter reading again shows a linear decrease from Feburary mid to April mid\n* Their is sudden linear decrease in the hot water reading from Feburary mid to April mid.\n\nAbove points indicates their might be something going on in bulding 1331 from Feburary mid to April mid.\n"},{"metadata":{},"cell_type":"markdown","source":"__From the above two plots it is very clear thatt energy consumption pattern of different buildings are very different from each other__"},{"metadata":{},"cell_type":"markdown","source":"### Let's take a more deeper look on building data"},{"metadata":{},"cell_type":"markdown","source":"__Let's take a look on the age of buildings.__"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(22, 8))\ndf_train['year_built'].value_counts().sort_index().plot.bar(color=color)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__We have the building built in the year 1900(OMG!! More than 110 years old), most of the buildings were made drring 60s and 70s__."},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 8))\ndf_train['primary_use'].value_counts(dropna=False).plot.bar(color=color)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Most of buildings in the dataset are built for Educational Purpose followed by Offical and Entetaniment use.__"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 8))\ndf_train['floor_count'].value_counts(normalize=True, dropna=False).plot.bar(color=color)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Floor information for around 85% of the buildinds in our dataset dosen't have any floorr count__"},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"Smallest building Area: \", df_train['square_feet'].min())\nprint(\"largetst building Area: \", df_train['square_feet'].max())\nprint(\"Mean building Area: \", df_train['square_feet'].mean())\nprint(\"Median building Area: \", df_train['square_feet'].median())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 8))\nsns.scatterplot(x=df_train['year_built'], y=df_train['square_feet'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__We can see some buildings are quite big__"},{"metadata":{},"cell_type":"markdown","source":"### Let's explore the weather data"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_weather_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__We have around 0.13 M datapoints for weather dataset__."},{"metadata":{"trusted":false},"cell_type":"code","source":"df_weather_train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Let's check the didtribution of Air Temprature over the entire dataset.__"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(8,6))\nsns.distplot(df_weather_train['air_temperature'].dropna(), color='red', axlabel=\"Air Temprature\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Distribution of Dew Temprature over the entire weather dataset__"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(8,6))\nsns.distplot(df_weather_train['dew_temperature'].dropna(), color='green', axlabel=\"Dew Temprature\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Distribution of Sea Level Pressure over the entire weather dataset__"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(8,6))\nsns.distplot(df_weather_train['sea_level_pressure'].dropna(), color='blue', axlabel=\"Sea Level Pressure\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Distribution of Wind Speed over the entire weather dataset__"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(8,6))\nsns.distplot(df_weather_train['wind_speed'].dropna(), color='purple', axlabel=\"Wind Speed\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Cloud coverage information of majority of datapoints in the dataset is missing__.\n__Also, significantly large number of infomation regarding Precepitation Depth is missing__."},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(16,8))\ndf_train['site_id'].value_counts(normalize=True, dropna=False).plot.bar(color=color)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Majority of entries in our dataset is from site 13, 9  & 2__"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(22,11))\nsns.lineplot(x=df_weather_train['timestamp'], y=df_weather_train['air_temperature'], hue=df_weather_train['site_id'], \n             legend='full', palette=sns.color_palette('Paired',16))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Temprature range differs at different sites but all of them follows a common pattern the temprature increases till the mid of year and then starts decreasing__."},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(22,11))\nsns.lineplot(x=df_weather_train['timestamp'], y=df_weather_train['dew_temperature'], hue=df_weather_train['site_id'], \n             legend='full', palette=sns.color_palette('Paired',16))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Similar to Temprature, Dew Point Temprature among all the sites follows similar rise and fall trend__"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(22,11))\nsns.lineplot(x=df_weather_train['timestamp'], y=df_weather_train['sea_level_pressure'], hue=df_weather_train['site_id'], \n             legend='full', palette=sns.color_palette('Paired',16))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Sea Level range differs at different sites but all of them follows a common pattern it becomed minimal during the mid of year__."},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(22,11))\nsns.lineplot(x=df_weather_train['timestamp'], y=df_weather_train['wind_speed'], hue=df_weather_train['site_id'], \n             legend='full', palette=sns.color_palette('Paired',16))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"__Wind Speed among all the sites follows a similar trend__"},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":1}