{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm_notebook\nimport seaborn as sns\nimport gc\nimport plotly.graph_objects as go\n\n# /kaggle/input/ashrae-energy-prediction/weather_test.csv\n# /kaggle/input/ashrae-energy-prediction/building_metadata.csv\n# /kaggle/input/ashrae-energy-prediction/train.csv\n# /kaggle/input/ashrae-energy-prediction/test.csv\n# /kaggle/input/ashrae-energy-prediction/sample_submission.csv\n# /kaggle/input/ashrae-energy-prediction/weather_train.csv","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# Import datasets For train\n\ntrain_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv')\nweather_train_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv')\nbuilding_meta_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\n\n# Import datasets For test\n\n# test_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv')\n# weather_test_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv')\n# sample_submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage_2(df, use_float16=False):\n \"\"\"\n Iterate through all the columns of a dataframe and modify the data type to reduce memory usage. \n \"\"\"\n \n start_mem = df.memory_usage().sum() / 1024**2\n print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n \n for col in df.columns:\n if col == 'timestamp': continue\n if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n continue\n col_type = df[col].dtype\n \n if col_type != object:\n c_min = df[col].min()\n c_max = df[col].max()\n if str(col_type)[:3] == \"int\":\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64)\n else:\n df[col] = df[col].astype(\"category\")\n\n end_mem = df.memory_usage().sum() / 1024**2\n print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n \n return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# What is it all about?\n\nin this kernel for this special competition, We are working on predicting the energy consumption in KwH
\n\nFor now We just check train data\n\n\n\n\n[image source ](www.borntoengineer.com)"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = reduce_mem_usage_2(train_df ,use_float16=True)\nweather_train_df = reduce_mem_usage_2(weather_train_df ,use_float16=True)\nbuilding_meta_df = reduce_mem_usage_2(building_meta_df ,use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(building_meta_df, on='building_id', how='left')\ntrain_df = train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')\ndel weather_train_df, building_meta_df\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.timestamp = pd.to_datetime(train_df.timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['date'] = train_df.timestamp.dt.date\ntrain_df ['hour'] = train_df.timestamp.dt.hour","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# What we want to check? (Goals) : \n\n1- ) **Linear Corellation check** : At least there must be a linear correlation between meter_reading and other features.\n\n2-) **Nan Check** : this is not hypothesis but. There must be no so much NAN in train and test. But may have in metadata and weather.\n\n3-) **Time affect on meter_reading** : is monthes of the year affect the amount of energy they use?\n\n4-) **Different Type of building** : Let's see different type of buildings. By the type we can use different Feature Engineerings.\n\n5-) **Energy Usage By Building Type** : I think we must have most energy usage by Education buildings. Let's check what about others.\n\n6-) **Mode in Energy Meter Type**\n\n7-) **Energy_Type Usage Count** : let's see which energy type is used in most buildings\n\n8-) **Energy_Type Usage Count for each primary_use** : for each primary use, how many of that primary use buildings are using different energy types.\n\n8.1-) **meter_reading of different energy sources for each primary use** : KWH energy usage by different energy types in each primary use.\n \n as we can see here steam is eating money and energy\n\n9-) **Mean Of primary_use And year_built and square feet** : check some numeric data\n\n10-) **Noise Detection by hand** : Let's remove noises by hand\n\n\n"},{"metadata":{},"cell_type":"markdown","source":"# Resuluts :\n\n1-) There was no meaningful relation .\n\n2-) That was right for train. that was more right in test :|. we may have challanges for feature engineering and data filling.\n\nOne usefull way is to find mean of each building type ( primary_ use ) and fill with them for Floor_count and others.\n\n3-) It seems time is not main reason for indcreasing and decreasing of energy \n\n4-) becuase of memory problems I'll find this answer later.\n\n5-) Education has the most building. After education : Public Other Office Lodging residental.\n\n6- Education has most energy usage. after education we have Office Entertainment Service and Lodging residental\n\n7-) Most of the buildings has Electrity. But does electricity use most energy? ( NO!!! we will see later).\n\n8-) As we can see most of the buildings are using electricity in each primary use\n\n8.1-) Steam is dominating here. so We can see one very important feature here. STEAM.\n\n9-) we saw some statistic data.\n\n10-) we found that we have very very much noise in our data. and we can remove them by hand too. not so hard. but need some time."},{"metadata":{"trusted":true},"cell_type":"code","source":"# just first Let's see 5 first rows. this is my habit.\nprint(f'shape is {train_df.shape}')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Linear Corellation check"},{"metadata":{"trusted":true},"cell_type":"code","source":"# check first hypothesis :\ncorr = train_df.corr()\n# Don;t show upper triangle\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\nf, ax = plt.subplots(figsize=(11, 9))\nsns.heatmap(corr, mask=mask)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"weird!!!! No Sensible linear relation between columns."},{"metadata":{},"cell_type":"markdown","source":"# Nan check"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_df.isnull().sum() * 100 / train_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For **TRAIN** : \nSo we see some columns have many Nan Values.
\nSo bad we don't have all floor_counts | year_built . these two variable are super important( At least I think )\n\n"},{"metadata":{},"cell_type":"markdown","source":"# Time affect on meter_reading"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(figsize=(14,6))\ntrain_df[['timestamp', 'meter_reading']].set_index('timestamp').resample('H').mean()['meter_reading'].plot(ax=axes, label='By hour', alpha=0.8).set_ylabel('Meter reading', fontsize=14)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see 2 peak here\n\nThis notebook ( https://www.kaggle.com/nroman/eda-for-ashrae ) shows why we have these 2 peaks."},{"metadata":{},"cell_type":"markdown","source":"# **Different Type of building** "},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.barplot(train_df.groupby(['primary_use']).size().reset_index(name='counts')['primary_use'], train_df.groupby(['primary_use']).size().reset_index(name='counts')['counts'])\nax.set(xlabel='Primary Usage', ylabel='# of records', title='Primary Usage vs. # of records')\nax.set_xticklabels(ax.get_xticklabels(), rotation=50, ha=\"right\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Energy Usage By Building Type** "},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.barplot(train_df.groupby('primary_use').meter_reading.agg('sum').reset_index()['primary_use'],\n train_df.groupby('primary_use').meter_reading.agg('sum').reset_index()['meter_reading'])\nax.set(xlabel='Primary Usage', ylabel='# of records', title='Primary Usage vs. # of records')\nax.set_xticklabels(ax.get_xticklabels(), rotation=50, ha=\"right\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mode in Energy Meter Type"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby('primary_use').meter.agg(pd.Series.mode).reset_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Energy_Type Usage Count\n\nlet's see which energy type is used in most buildings"},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.barplot(['electricity', 'chiledwater', 'steam', 'hotwater'], train_df.meter.value_counts().to_frame()['meter'])\nax.set(xlabel='meter Types', ylabel='How much is used', title='meter Types vs. # of usage')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Energy_Type Count for each primary_use\n\ncheck type of energy each primary_use has for it's buildings."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=4, ncols=4)\nfig.set_figheight(12)\nfig.set_figwidth(15)\n# fig.tight_layout()\nfig.subplots_adjust(wspace=0.5)\nfig.subplots_adjust(hspace=0.5)\nprimary_uses = train_df.primary_use.unique()\n\nfor i, primary_use in enumerate(primary_uses):\n sns.countplot(x=\"meter\", data = train_df[train_df.primary_use == primary_use],ax=axes[i//4,i%4]).title.set_text(str(primary_use))\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# meter_reading of different energy sources for each primary use \n\nKWH energy usage for each energy type for each primay_use"},{"metadata":{"trusted":true},"cell_type":"code","source":"meter_types = { 0 : 'electricity', 1 : 'chilledwater', 2 : 'steam', 3 : 'hotwater' }\n\n\nfig, axes = plt.subplots(nrows=4, ncols=4)\nfig.set_figheight(12)\nfig.set_figwidth(15)\n# fig.tight_layout()\nfig.subplots_adjust(wspace=0.5)\nfig.subplots_adjust(hspace=0.5)\nprimary_uses = train_df.primary_use.unique()\n\nfor i, primary_use in enumerate(primary_uses):\n sns.barplot(train_df[train_df.primary_use == primary_use].groupby('meter').meter_reading.sum(),\n np.vectorize(meter_types.get)(train_df[train_df.primary_use == primary_use].groupby(['meter'])['meter_reading'].mean().keys()),\n ax=axes[i//4,i%4]).title.set_text(str(primary_use))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"STEAM. reasonable but was amazing to know steam is using this much energy."},{"metadata":{},"cell_type":"markdown","source":"# Mean Of square_feet"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby('primary_use').square_feet.agg(['mean'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mean Of year_built"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby('primary_use').year_built.agg(['mean'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Nothing for services. What a huge missing data."},{"metadata":{},"cell_type":"markdown","source":"# Mean Of meter_reading by primary_use"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.groupby('primary_use').meter_reading.agg(['mean'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Until here we have 2 suspect : EDUCATION AND STEAM\n"},{"metadata":{},"cell_type":"markdown","source":"# Noise Detection By hand \n\nWe have noises in data. To detect these noises, we need to see buildings and check.\n\nI think when we have sequential zeros for meter_reading ( for example 15 days. ) that is noise.\n\nI'll show just one sample. I'll create another notebook for noise detection using EDA\n\nShow Site ids"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(8,2,figsize=(14, 30), dpi=100)\n\nfor i in range(train_df['site_id'].nunique()):\n train_df[train_df.site_id == i][['timestamp', 'meter_reading']].set_index('timestamp').resample('H').mean()['meter_reading'].plot(ax = axes[i%8][i//8])\n axes[i%8][i//8].legend();\n axes[i%8][i//8].set_title('site_id {}'.format(i), fontsize=13);\n plt.subplots_adjust(hspace=0.45)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Show primary uses of one site id"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"fig, axes = plt.subplots(8,2,figsize=(14, 30), dpi=100)\nfor i, use in enumerate(train_df['primary_use'].value_counts().index.to_list()):\n try:\n train_df[(train_df['site_id'] == 13) & (train_df['primary_use'] == use)][['timestamp', 'meter_reading']].set_index('timestamp').resample('H').mean()['meter_reading'].plot(ax=axes[i%8][i//8], alpha=0.8, label='By hour', color='tab:blue').set_ylabel('Mean meter reading', fontsize=13);\n train_df[(train_df['site_id'] == 13) & (train_df['primary_use'] == use)][['timestamp', 'meter_reading']].set_index('timestamp').resample('D').mean()['meter_reading'].plot(ax=axes[i%8][i//8], alpha=1, label='By day', color='tab:orange').set_xlabel('');\n axes[i%8][i//8].legend();\n except TypeError:\n pass\n axes[i%8][i//8].set_title(use, fontsize=13);\n plt.subplots_adjust(hspace=0.45)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" Now we just choose, site_id 13.\n let's check "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[(train_df['site_id'] == 13) & (train_df['primary_use'] == 'Education')][['timestamp', 'meter_reading']].set_index('timestamp').resample('H').mean()['meter_reading'].plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"check this special site id and primary use energy type usage"},{"metadata":{"trusted":true},"cell_type":"code","source":"meter_types = { 0 : 'electricity', 1 : 'chilledwater', 2 : 'steam', 3 : 'hotwater' }\ncountry=train_df[(train_df.site_id == 13) & (train_df.primary_use == 'Education') ].groupby('meter')['meter_reading'].sum()\n\nfig = go.Figure(go.Treemap(\n labels=np.vectorize(meter_types.get)(country.keys()),\n parents = [\"Total Energy Usage Type \"]*len(country),\n values = country\n))\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let's see buildings in this site id and this special primary use"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfig, axes = plt.subplots(9,2,figsize=(14, 36), dpi=100)\nfor i, building in enumerate(train_df[(train_df['site_id'] == 13) & (train_df['primary_use'] == 'Education') & (train_df['meter'] == 2)]['building_id'].value_counts(dropna=False).index.to_list()):\n train_df[(train_df['site_id'] == 13) & (train_df['primary_use'] == 'Education') & (train_df['meter'] == 2) & (train_df['building_id'] == building)][['timestamp', 'meter_reading']].set_index('timestamp').resample('H').mean()['meter_reading'].plot(ax=axes[i%9][i//9], alpha=0.8, label='By hour', color='tab:blue').set_ylabel('Mean meter reading', fontsize=13);\n train_df[(train_df['site_id'] == 13) & (train_df['primary_use'] == 'Education') & (train_df['meter'] == 2) & (train_df['building_id'] == building)][['timestamp', 'meter_reading']].set_index('timestamp').resample('D').mean()['meter_reading'].plot(ax=axes[i%9][i//9], alpha=1, label='By day', color='tab:orange').set_xlabel('');\n axes[i%9][i//9].legend();\n axes[i%9][i//9].set_title('building_id: ' + str(building), fontsize=13);\n plt.subplots_adjust(hspace=0.45)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So. if you want to delete all the noises you must do these last parts for every site id and pirmary use. then remove those zeros.\n\nThat's it."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}