{"cells":[{"metadata":{},"cell_type":"markdown","source":"# A late night ensemble of the best public kernels\n* You can also see a sample meter below, using the [clean weather data](https://www.kaggle.com/hmendonca/clean-weather-data-eda)\n\nKernels:\n* https://www.kaggle.com/rohanrao/ashrae-half-and-half\n* https://www.kaggle.com/kimtaegwan/what-s-your-cv-method\n* https://www.kaggle.com/nz0722/aligned-timestamp-lgbm-by-meter-type\n* https://www.kaggle.com/isaienkov/keras-nn-with-embeddings-for-cat-features-1-15"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom pandas.plotting import register_matplotlib_converters\nregister_matplotlib_converters()\n\nsub = None\nfor dirname, _, filenames in os.walk('/kaggle/input/subs20191106/'):\n    for filename in filenames:\n        filename = os.path.join(dirname, filename)\n        print(filename)\n        if sub is None:\n            sub = pd.read_csv(filename)\n        else:\n            sub.meter_reading += pd.read_csv(filename, usecols=['meter_reading']).meter_reading\n    sub.meter_reading = sub.meter_reading.clip(lower=0) / len(filenames)\n\nsub.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# prediction's log-log histogram:\nax = np.log1p(sub.meter_reading).hist(bins=100)\nax.set_yscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"path = '../input/clean-weather-data-eda'\nbuilding = pd.read_csv(f'{path}/building_metadata.csv.gz', dtype={'building_id':np.uint16, 'site_id':np.uint8})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"train = pd.read_csv(f'{path}/train.csv.gz', dtype={'building_id':np.uint16, 'meter':np.uint8}, parse_dates=['timestamp'])\ntrain = train.merge(building, on='building_id', how='left')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# target's log-log histogram:\nax = np.log1p(train.meter_reading).hist(bins=100)\nax.set_yscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"test = pd.read_csv(f'{path}/test.csv.gz', dtype={'building_id':np.uint16, 'meter':np.uint8}, parse_dates=['timestamp'])\ntest['meter_reading'] = sub.meter_reading\ntest = test.merge(building, on='building_id', how='left')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"weather_trn = pd.read_csv(f'{path}/weather_train.csv.gz', parse_dates=['timestamp'],\n                                                       dtype={'site_id':np.uint8, 'air_temperature':np.float16},\n                                                       usecols=['site_id', 'timestamp', 'air_temperature'])\nweather_tst = pd.read_csv(f'{path}/weather_test.csv.gz', parse_dates=['timestamp'],\n                                                       dtype={'site_id':np.uint8, 'air_temperature':np.float16},\n                                                       usecols=['site_id', 'timestamp', 'air_temperature'])\ntrain = train.merge(weather_trn, on=['site_id', 'timestamp'], how='left')\ntest = test.merge(weather_tst, on=['site_id', 'timestamp'], how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"building_id = 1258\nmeter = 1 # pick a meter\n\ntrain_sample = train[(train['building_id'] == building_id) & (train['meter'] == meter)]  # same train sample as above\ntest_sample = test[(test['building_id'] == building_id) & (test['meter'] == meter)]  # and the same meter in the test set\n\nfig, ax = plt.subplots(figsize=(16,4))\nplt.title(f'Building {building_id} Meter {meter}')\nax.xaxis.set_tick_params(rotation=30, labelsize=10)\nax2 = ax.twinx()\n\n# plot training sample\ndates = matplotlib.dates.date2num(train_sample['timestamp'])\nax2.plot_date(dates, train_sample['meter_reading'], '-', label='train', alpha=0.8)\nax.plot_date(dates, train_sample['air_temperature'], '.', color='tab:cyan', label='air_temperature')\n\n# plot test sample\ndates = matplotlib.dates.date2num(test_sample['timestamp'])\nax2.plot_date(dates, test_sample['meter_reading'], '*', label='test', alpha=0.8)\nax.plot_date(dates, test_sample['air_temperature'], '.', color='tab:cyan', label='air_temperature')\n\nax.set_ylabel('air_temperature'); ax2.set_ylabel('meter_reading')\nax.legend(loc='upper left'); ax2.legend(loc='upper right')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv(f'submission.csv', index=False, float_format='%g')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Conclusions?\nIt looks like the models fail to predict the extremes, as both very low and very high values don't appear in the histograms"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}