{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\n\nfrom IPython.display import display\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom scipy.stats import norm, probplot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_PATH = Path('/kaggle', 'input', 'ashrae-energy-prediction')\nTRAIN_PATH = INPUT_PATH / 'train.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_PATH, parse_dates=['timestamp'])\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['timestamp'].min(), df_train['timestamp'].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(df_train.describe().T)\ndisplay(pd.DataFrame([df_train[col].nunique() for col in df_train.columns],\n             index=df_train.columns, columns=['nunique']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['timestamp'].min(), df_train['timestamp'].max()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The training set covers 1 year of data (test set might covers the 2 other years), from 01/01/2016 to 12/31/2016."},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.distplot(df_train['building_id'].value_counts(),\n                  kde=False, hist_kws=dict(density=True));\nax.set_title('Histogram of the frequencies of building_id');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Seems there is a pattern in the number of occurences of `building_ids`. We see 3 modes, around 10000, 18000 and 27000. That means, the building_ids have not been sampled randomly."},{"metadata":{"trusted":true},"cell_type":"code","source":"METER_MAPPER = {0: 'electricity', 1: 'chilledwater', 2: 'steam', 3: 'hotwater'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, ax = plt.subplots()\n\ndf_train['meter'].value_counts().plot.bar(ax=ax)\nax.set_xlabel(\"Meter\")\nax.set_ylabel('Count')\nax.set_xticklabels(METER_MAPPER.values());","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Electricity is the most common energy type, by far."},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.countplot(df_train.groupby('building_id')['meter'].nunique())\nax.set_title(\"Frequencies of count of meter per building_id\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most `building_id` have only one energy source, some have 2 or 3 while very few have 4.  \nIs there some temporality, e.g. buildings have changed their energy source over the years?"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(df_train['meter_reading']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['meter_reading'].describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is looking like a power-law, let's see the impact of a log transform."},{"metadata":{"trusted":true},"cell_type":"code","source":"_, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 4))\nsns.distplot(np.log1p(df_train['meter_reading']), fit=norm, ax=ax1)\n_ = probplot(np.log1p(df_train['meter_reading']), plot=ax2);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Far from lognormal, we have a spike around 0 (e.g log1p(0)), that means we have a lot of reading_meters close or equals to 0."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.loc[df_train['meter_reading'] == 0].shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Indeed, close to 2 millions of meter readings are zero-valued, that's almost 10%."},{"metadata":{"trusted":true},"cell_type":"code","source":"non_zero_meter_readings = df_train.loc[df_train['meter_reading'] != 0, 'meter_reading']\n\n_, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 4))\nsns.distplot(np.log1p(non_zero_meter_readings), fit=norm, ax=ax1)\n_ = probplot(np.log1p(non_zero_meter_readings), plot=ax2);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looking a bit better but still not close to a normal probplot, cutting the purely zero-valued meter is not enough, there seems to be a huge amount of non-zero small values."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['log1p_meter_reading'] = np.log1p(df_train['meter_reading'])\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, axes = plt.subplots(1, 2, figsize=(14, 4))\n\nsns.barplot(x='meter', y='log1p_meter_reading', data=df_train, ax=axes[0])\nsns.violinplot(x='meter', y='log1p_meter_reading', data=df_train, ax=axes[1])\n\nfor ax in axes:\n    ax.set_xticklabels(METER_MAPPER.values());","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Variations over time"},{"metadata":{"trusted":true},"cell_type":"code","source":"frequencies_grouper = [\n    ('H', 'hour'),\n    ('D', 'day'),\n    ('D', 'dayofweek'),\n    ('W', 'week'),\n    ('MS', 'month'),\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for _, g in frequencies_grouper:\n    df_train[g] = getattr(df_train['timestamp'].dt, g)\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(18, 8), tight_layout=True)\n\ngs = fig.add_gridspec(2, 6)\n\naxes = np.array([\n    fig.add_subplot(gs[0, :3]),\n    fig.add_subplot(gs[0, 3:]),\n    fig.add_subplot(gs[1, :2]),\n    fig.add_subplot(gs[1, 2:4]),\n    fig.add_subplot(gs[1, 4:]),\n])\n\nfor (f, g), ax in zip(frequencies_grouper, axes):\n    grouper = (df_train.groupby(pd.Grouper(freq=f, key='timestamp'))\\\n               .agg(average=('meter_reading', 'mean')))\n    grouper[g] = getattr(grouper.index, g)\n    sns.barplot(x=g, y='average', data=grouper, ax=ax)\n    ax.set_xlabel(g)\n    ax.set_ylabel(f\"Average meter reading\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Hard to see small periods trends like this, but we can see that for the year 2016, average meter readings for months 3, 4, 5 and 6 were particularly high."},{"metadata":{},"cell_type":"markdown","source":"### To be continued..."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}