{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport gc\nimport os\n\nimport requests\nfrom bs4 import BeautifulSoup\nimport json\n\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# building_metadata.csv\nBUILDINGMETADATA_DTYPES = {'site_id': np.uint8, 'building_id': np.uint16, 'square_feet': np.int32, 'year_built': np.float32, 'floor_count': np.float32}\ndf_building_metadata = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv', dtype=BUILDINGMETADATA_DTYPES)\n\n# weather_train.csv and weather_test.csv\nWEATHER_DTYPES = {'site_id': np.uint8, 'air_temperature': np.float32, 'cloud_coverage': np.float32, 'dew_temperature': np.float32, \n                  'precip_depth_1_hr': np.float32, 'sea_level_pressure': np.float32, 'wind_direction': np.float32, 'wind_speed': np.float32}\ndf_weather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv', dtype=WEATHER_DTYPES)\ndf_weather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv', dtype=WEATHER_DTYPES)\ndf_weather = pd.concat([df_weather_train, df_weather_test], ignore_index=True)\n\n# train.csv\nTRAIN_DTYPES = {'building_id': np.uint16, 'meter': np.uint8, 'meter_reading': np.float32}\ndf_train = pd.read_csv('../input/ashrae-energy-prediction/train.csv', dtype=TRAIN_DTYPES)\n\n# test.csv\nTEST_DTYPES = {'building_id': np.uint16, 'meter': np.uint8}\ndf_test = pd.read_csv('../input/ashrae-energy-prediction/test.csv', dtype=TEST_DTYPES)\ndf_test.drop(columns=['row_id'], inplace=True)\n    \n# Keeping site 0\ndf_train = df_train[df_train['building_id'] < 105]\ndf_test = df_test[df_test['building_id'] < 105]\ndf_site0 = pd.concat([df_train, df_test], ignore_index=True, sort=False)\n\nfor df in [df_site0, df_weather]:\n    df['timestamp'] = pd.to_datetime(df['timestamp'], infer_datetime_format=True)\n\ndf_site0 = df_site0.merge(df_building_metadata, on='building_id', how='left')\ndf_site0 = df_site0.merge(df_weather, on=['site_id', 'timestamp'], how='left')\n\ndel df_train, df_test, df_weather_train, df_weather_test, df_weather\ngc.collect()\n\nprint('Site 0 Shape = {}'.format(df_site0.shape))\nprint('Site 0 Memory Usage = {:.2f} MB'.format(df_site0.memory_usage().sum() / 1024**2))\nprint('Site 0 Buildings with Electricity Meter = {}'.format(len(df_site0[df_site0['meter'] == 0]['building_id'].unique())))\nprint('Site 0 Buildings with Chilled Water Meter = {}'.format(len(df_site0[df_site0['meter'] == 1]['building_id'].unique())))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **1. UCF (University of Central Florida) Spider**\nBuilding metadata is scraped from https://www.oeis.ucf.edu/buildings, and there are two extra features in the page.\n  * `EUI (kBTU/sqft)`: Energy per square foot per year **Reference**: https://www.energystar.gov/buildings/facility-owners-and-managers/existing-buildings/use-portfolio-manager/understand-metrics/what-energy\n  * `LEED`: (**Leadership in Energy and Environmental Design**) is the most widely used green building rating system in the world"},{"metadata":{"trusted":true},"cell_type":"code","source":"SITE_0_START_URL = 'https://www.oeis.ucf.edu/buildings'\nSITE_0_AREAS = df_site0['square_feet'].unique()\n\nbuildings = BeautifulSoup(requests.get(SITE_0_START_URL).text, 'html.parser')\n\nbuilding_names = [link.text.strip() for link in buildings.select('table#buildings tr th a')]\nbuilding_links = [link.get('href') for link in buildings.select('table#buildings tr th a')]\nbuilding_types = [link.text.strip() for link in buildings.select('table#buildings tr td:nth-child(3)')]\nbuilding_areas = [link.text.strip() for link in buildings.select('table#buildings tr td:nth-child(4)')]\nbuilding_euis = [link.text.strip() for link in buildings.select('table#buildings tr td:nth-child(5)')]\nbuilding_leeds = [link.text.strip() for link in buildings.select('table#buildings tr td:nth-child(6)')]\n\nsite0_building_metadata = {k: v  for k, v in enumerate(zip(building_names, building_links, building_types, building_areas, building_euis, building_leeds))}\ndf_site0_building_metadata = pd.DataFrame(site0_building_metadata).T.replace('', np.nan)\ndf_site0_building_metadata.columns = ['building_name', 'building_link', 'building_type', 'square_feet', 'eui', 'leed']\ndf_site0_building_metadata['building_url_code'] = df_site0_building_metadata['building_link'].str.split('/', expand=True)[4] # Going to use this while sending AJAX requests\ndf_site0_building_metadata['square_feet'] = df_site0_building_metadata['square_feet'].astype(np.uint32)\ndf_site0_building_metadata['eui'] = df_site0_building_metadata['eui'].astype(np.float32)\ndf_site0_building_metadata = df_site0_building_metadata[df_site0_building_metadata['square_feet'].isin(SITE_0_AREAS)] # square_feet values don't exist in competition data are excluded\n\ndel building_names, building_links, building_types, building_areas, building_euis, building_leeds, site0_building_metadata, SITE_0_AREAS\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Building meter readings are scraped from https://www.oeis.ucf.edu/getData. API returns the records of meter readings from given time period in json format. The AJAX request requires a `building_url_code` for returning the records of a specific building. The `building_url_code` values in `df_site0_building_metadata` are iterated and used in the request, and the scraped meter readings concatenated into a DataFrame."},{"metadata":{"trusted":true},"cell_type":"code","source":"SITE_0_AJAX_URL = 'https://www.oeis.ucf.edu/getData'\nBUILDING_URL_CODES = df_site0_building_metadata['building_url_code'].unique().tolist()\nPARAMS = {\n    'building': None,\n    'start-date': '01/01/2016',\n    'end-date': '01/01/2019',\n    'resolution': 'hour',\n    'filetype': 'json'    \n}\n\ndf_site0_labels = pd.DataFrame(columns=['meter', 'meter_reading', 'timestamp', 'building_url_code'])\n\nfor building_url_code in tqdm(BUILDING_URL_CODES):\n    PARAMS['building'] = building_url_code\n    building_readings = json.loads(requests.post(url=SITE_0_AJAX_URL, params=PARAMS).text) \n\n    for meter_type in building_readings:\n        \n        if meter_type['key'] == 'Gas' or meter_type['key'] == 'Irrigation' or meter_type['key'] == 'Water':\n            continue\n        \n        timestamps = pd.Series([value['timestamp'] for value in meter_type['values']])\n        meter_readings = pd.Series([value['reading'] for value in meter_type['values']])\n        meter_types = pd.Series(np.tile(meter_type['key'], len(timestamps)))\n        building_url_codes = pd.Series(np.tile(building_url_code, len(timestamps)))\n\n        df_meter_reading = pd.DataFrame(columns=['meter', 'meter_reading', 'timestamp', 'building_url_code'])\n        df_meter_reading['timestamp'] = timestamps\n        df_meter_reading['meter_reading'] = meter_readings\n        df_meter_reading['meter'] = meter_types\n        df_meter_reading['building_url_code'] = building_url_codes\n\n        df_site0_labels = pd.concat([df_site0_labels, df_meter_reading], ignore_index=True)\n        \ndf_site0_labels = df_site0_labels[df_site0_labels['timestamp'] < '2019-01-01 00:00:00']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **2. Data Cleaning**\n\n* Unique `square_feet` values are directly matched with their correct `building_id`\n* Not unique `square_feet` values are manually matched with their correct `building_id` by checking the training set labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Unique square_feet values in site 0\nsite0_unique_areas = df_site0_building_metadata['square_feet'].value_counts()[df_site0_building_metadata['square_feet'].value_counts() < 2].index.tolist()\ndf_site0_unique_areas = df_building_metadata[df_building_metadata['square_feet'].isin(site0_unique_areas) & (df_building_metadata['site_id'] == 0)]\narea_building_id_mapping = df_site0_unique_areas.set_index('square_feet')['building_id'].to_dict()\ndf_site0_building_metadata['building_id'] = df_site0_building_metadata['square_feet'].map(area_building_id_mapping)\n\n# Not Unique square_feet values in site 0\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"86\"').index, 'building_id'] = '27'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"149\"').index, 'building_id'] = '90'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"12\"').index, 'building_id'] = '33'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"92\"').index, 'building_id'] = '61'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"68\"').index, 'building_id'] = '49'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"142\"').index, 'building_id'] = '67'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"131\"').index, 'building_id'] = '77'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"7\"').index, 'building_id'] = '100'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"28\"').index, 'building_id'] = '34'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"79\"').index, 'building_id'] = '62'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"44\"').index, 'building_id'] = '51'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"125\"').index, 'building_id'] = '69'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"140\"').index, 'building_id'] = '70'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"52\"').index, 'building_id'] = '71'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"100\"').index, 'building_id'] = '72'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"74\"').index, 'building_id'] = '73'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"130\"').index, 'building_id'] = '74'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"10\"').index, 'building_id'] = '35'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"134\"').index, 'building_id'] = '63'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"105\"').index, 'building_id'] = '36'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"69\"').index, 'building_id'] = '37'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"138\"').index, 'building_id'] = '64'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"24\"').index, 'building_id'] = '65'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"6\"').index, 'building_id'] = '66'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"71\"').index, 'building_id'] = '85'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"146\"').index, 'building_id'] = '95'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"98\"').index, 'building_id'] = '96'\ndf_site0_building_metadata.loc[df_site0_building_metadata.query('building_url_code == \"30\"').index, 'building_id'] = '98'\n\n# building_id mapped to the scraped labels\ndf_site0_building_metadata['building_id'] = df_site0_building_metadata['building_id'].astype(np.uint16)\nurl_code_building_id_mapping = df_site0_building_metadata.set_index('building_url_code')['building_id'].to_dict()\ndf_site0_labels['building_id'] = df_site0_labels['building_url_code'].map(url_code_building_id_mapping)\n\n# Removing unnecessary columns\ndf_site0_building_metadata.drop(columns=['building_name', 'building_link', 'building_type', 'building_url_code'], inplace=True)\ndf_site0_labels.drop(columns=['building_url_code'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Scraped data are standardized to the competition format\n* Scraped null readings are dropped\n* `meter` is label encoded\n* Chilled water readings are multiplied with **3.51684**  (1 ton of refrigeration) **Reference**: https://www.quora.com/What-is-the-1-tonnes-of-refrigeration-effect\n\nFinally, standardized data merged to the competition data set."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_site0_labels.drop(df_site0_labels.query('meter_reading.isnull()', engine='python').index, inplace=True)\n\ndf_site0_labels['meter'] = df_site0_labels['meter'].map({'Electric': 0, 'Chilled Water': 1})\ndf_site0_labels['timestamp'] = pd.to_datetime(df_site0_labels['timestamp'], infer_datetime_format=True)\ndf_site0_labels.loc[df_site0_labels.query('meter == 1').index, 'meter_reading'] = df_site0_labels.loc[df_site0_labels.query('meter == 1').index, 'meter_reading'] * 3.51684\n\ndf_site0_labels.sort_values(by=['timestamp', 'building_id', 'meter'], inplace=True)\n\ndf_site0 = df_site0.merge(df_site0_labels, on=['timestamp', 'building_id', 'meter'], how='left')    \ndf_building_metadata_external = df_building_metadata.merge(df_site0_building_metadata, on=['building_id', 'square_feet'], how='left')\ndf_site0.rename(columns={'meter_reading_x': 'meter_reading_original', 'meter_reading_y':'meter_reading_scraped'}, inplace=True)\n\ndel df_site0_labels\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_site0[['building_id', 'meter', 'timestamp', 'meter_reading_original', 'meter_reading_scraped']].sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_building_metadata_external","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndf_site0[['building_id', 'meter', 'timestamp', 'meter_reading_original', 'meter_reading_scraped']].to_csv('site0.csv.gz', index=False)\ndf_site0[['building_id', 'meter', 'timestamp', 'meter_reading_original', 'meter_reading_scraped']].to_pickle('site0.pkl')\n\ndf_building_metadata_external.to_csv('building_metadata_external.csv', index=False)\ndf_building_metadata_external.to_pickle('building_metadata_external.pkl')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **3. Scraped Yearly Readings**\nThe differences and patterns learned from yearly readings of UCF can be useful, and could be translated to meter readings of other sites. In order to analyze patterns and missing readings, `df_site0` has to be assigned with 3 years of full timeframe index. Data analysis would be more accurate that way."},{"metadata":{"trusted":true},"cell_type":"code","source":"site0_meter0_buildings = sorted(df_site0[df_site0['meter'] == 0]['building_id'].unique())\nsite0_meter1_buildings = sorted(df_site0[df_site0['meter'] == 1]['building_id'].unique())\n\nbuilding_ids = sorted(np.unique(df_site0['building_id']))\nmeters = sorted(np.unique(df_site0['meter']))\ndf_site0 = df_site0.set_index(['building_id', 'timestamp', 'meter'], drop=False).sort_index()\n\nsite0_full_index = pd.MultiIndex.from_product([building_ids, pd.date_range(start='2016-01-01 00:00:00', end='2018-12-31 23:00:00', freq='H'), meters])\n\ndf_site0 = df_site0.reindex(site0_full_index)\ndf_site0['building_id'] = df_site0.index.get_level_values(0)\ndf_site0['timestamp'] = df_site0.index.get_level_values(1)\ndf_site0['meter'] = df_site0.index.get_level_values(2)\n\ndef plot_buildings(building_ids, meter_type):\n    \n    QUERY_2016 = 'building_id == {} and meter == {} and (\"2016-01-01 00:00:00\" <= timestamp < \"2017-01-01 00:00:00\")'\n    QUERY_2017 = 'building_id == {} and meter == {} and (\"2017-01-01 00:00:00\" <= timestamp < \"2018-01-01 00:00:00\")'\n    QUERY_2018 = 'building_id == {} and meter == {} and (\"2018-01-01 00:00:00\" <= timestamp < \"2019-01-01 00:00:00\")'\n    \n    fig, axes = plt.subplots(len(building_ids), figsize=(20, 80), dpi=100)    \n\n    for i, building_id in enumerate(building_ids):\n        df_site0.query(QUERY_2016.format(building_id, meter_type)).reset_index().set_index('level_1')['meter_reading_scraped'].plot(label='Training 2016', ax=axes[i])\n        df_site0.query(QUERY_2017.format(building_id, meter_type)).reset_index().set_index('level_1')['meter_reading_scraped'].plot(label='Test 2017', ax=axes[i])\n        df_site0.query(QUERY_2018.format(building_id, meter_type)).reset_index().set_index('level_1')['meter_reading_scraped'].plot(label='Test 2018', ax=axes[i])\n\n        axes[i].legend()\n        axes[i].set_title('building_id {} meter {}'.format(building_id, meter_type), fontsize=13);\n\n        plt.subplots_adjust(hspace=0.45)\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **3.1 Electricity Readings**\nBuildings consume electricity throughout the year with very few interruptions. Electricity readings looks more stable than the Chilled Water readings in this site. There are **14** timestamps at which the entire site has missing Electricity readings. Those **14** timestamps belong to **7** different days and none of those days are in the training set;\n  * **2017-02-14** (Valentine's Day)\n  * **2017-06-19**\n  * **2017-06-20**\n  * **2018-03-11**\n  * **2018-03-12**\n  * **2018-06-26**\n  * **2018-06-27**\n  \nThose dates could be related to planned maintenance days for the entire site or they could be related to meter errors. The other dates of missing electricity readings have fewer buildings. They are more likely to be random power outages, and they are more common in training set."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(18, 10))\n                 \ndf_site0.query('meter == 0 and meter_reading_scraped.isnull() and (\"2016-01-01 00:00:00\" <= timestamp < \"2017-01-01 00:00:00\")', engine='python')['timestamp'].value_counts().plot(label='Training 2016')\ndf_site0.query('meter == 0 and meter_reading_scraped.isnull() and (\"2017-01-01 00:00:00\" <= timestamp < \"2018-01-01 00:00:00\")', engine='python')['timestamp'].value_counts().plot(label='Test 2017')\ndf_site0.query('meter == 0 and meter_reading_scraped.isnull() and (\"2018-01-01 00:00:00\" <= timestamp < \"2019-01-01 00:00:00\")', engine='python')['timestamp'].value_counts().plot(label='Test 2018')\n\nplt.title('Site 0 Building Count with Missing Electricity Readings in 2016, 2017 and 2018 ')\nplt.ylabel('Building Count')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* There are excessive **0** readings and some huge outliers in the beginning of 2016. Those **0** readings are between **2016-01-01 00:00:00** - **2016-05-20 17:00:00** for most of the buildings. This phenomenon can be seen even in June and July for some buildings, and they don't represent the meter readings of next years.\n* There are huge negative outliers in some buildings. They could be related to random power outages because they don't have a date or time pattern. They occur right before a huge positive outlier. Buildings with negative outliers: **(0, 25, 38, 41, 77, 78, 86)**"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plot_buildings(range(0, 12), 0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(range(12, 24), 0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(range(24, 36), 0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(range(36, 48), 0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(range(48, 60), 0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(range(60, 72), 0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(range(72, 84), 0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(range(84, 96), 0)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(range(96, 105), 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **3.2 Chilled Water Readings**\nChilled water readings are less stable than electricity readings in a 3 year timeframe. There are only **24** buildings with chilled water meter in site 0, so it is even harder to find patterns in those buildings. Between some periods, all of the buildings have missing chilled water readings, and a time period without missing chilled water reading is very rare in 3 years. The dates of missing chilled water readings look like totally random except the first couple months of 2016. It was the time when electricity meters were displaying **0** readings."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(18, 10))\n                 \n(df_site0.query('meter == 1 and meter_reading_scraped.isnull() and (\"2016-01-01 00:00:00\" <= timestamp < \"2017-01-01 00:00:00\")', engine='python')['timestamp'].value_counts() - 81).plot(label='Training 2016')\n(df_site0.query('meter == 1 and meter_reading_scraped.isnull() and (\"2017-01-01 00:00:00\" <= timestamp < \"2018-01-01 00:00:00\")', engine='python')['timestamp'].value_counts() - 81).plot(label='Test 2017')\n(df_site0.query('meter == 1 and meter_reading_scraped.isnull() and (\"2018-01-01 00:00:00\" <= timestamp < \"2019-01-01 00:00:00\")', engine='python')['timestamp'].value_counts() - 81).plot(label='Test 2018')\n\nplt.title('Site 0 Building Count with Missing Chilled Water Readings in 2016, 2017 and 2018 ')\nplt.ylabel('Building Count')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Chilled water readings are recorded on a different scale on UCF's website. They are multiplied with **3.51684** (A ton of refrigeration effect) in the competition data. **A ton of refrigeration** is defined as the amount of refrigeration effect produced by uniform melting of **1 ton** of ice from and at **0º** C in **24** hours.\n* Missing chilled water readings are more common and unpredictable and the periods are longer compared to electricity readings.\n* Positive outliers are also more common in chilled water readings, and unlike electricity readings, there are no negative values."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(site0_meter1_buildings[:12], 1)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_buildings(site0_meter1_buildings[12:], 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **4. Scraped Building Metadata**\n\nThere were two new features (`eui` and `leed`) in the scraped building metadata for site 0. Both of those features are dependent to yearly `meter_reading` and `square_feet`.\n* `eui` is not strongly correlated with `square_feet` because there is no causality in their relationship. Apparently, some buildings can be very large and use energy very efficiently at the same time. However `eui` of 2016 can be a predictor of next years.\n* `leed` gives information about `eui`, but most of the buildings don't have certification. Gold certification buildings have higher `eui` than silver certification buildings, but there are other buildings that are using energy more efficiently without any certification. This feature is not reliable because of this reason."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"df_site0_building_metadata = df_building_metadata_external[df_building_metadata_external['site_id'] == 0]\n\nfig = plt.figure(figsize=(16, 8))\n\nsns.scatterplot(x='square_feet', y='eui', hue='primary_use', data=df_site0_building_metadata)\nplt.title('eui vs square_feet')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 8))\n\nsns.boxplot(x='leed', y='eui', data=df_site0_building_metadata.fillna('No Certification'))\nplt.title('eui vs leed')\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}