{"cells":[{"metadata":{},"cell_type":"markdown","source":"**ASHRAE - Great Energy Predictor III**\n# SUBMISSION VISUALIZATION\n\nThis notebook is a tool to graphically compare traininig and submissions data. Just load 1, 2 or more submissions "},{"metadata":{},"cell_type":"markdown","source":"# imports and loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"KAGGLE_MODE = True  # drives file loading","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \n\nfrom tqdm import tqdm, tqdm_notebook\nimport gc\nimport zipfile\nimport os\nimport datetime\n\nimport matplotlib.pyplot as plt\n%matplotlib inline  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { width:100% !important; }</style>\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# load files for test\nif KAGGLE_MODE:\n    building_df = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\n#     weather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\n#     weather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\n    train = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")\n    test =  pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")\nelse:\n    zf = zipfile.ZipFile('./ashrae-energy-prediction.zip') \n    building_df = pd.read_csv(zf.open('building_metadata.csv'))\n    train = pd.read_csv(zf.open('train.csv'))\n#     weather_train = pd.read_csv(zf.open('weather_train.csv'))\n#     weather_test = pd.read_csv(zf.open('weather_test.csv'))\n    test =  pd.read_csv(zf.open('test.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_df(df):\n    # adding timestamp, building_id and meter to submission data\n    if not 'timestamp' in df.columns:\n        df = pd.merge (df, test, on='row_id')\n        df = df.drop(columns=['row_id'])\n\n    # transforming timestamp\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n    df[\"date\"] = df[\"timestamp\"].dt.date\n    df[\"hour\"] = df[\"timestamp\"].dt.hour.astype(np.uint8)\n    \n    # aggregating data EDIT THIS PART TO GENERATE MORE STATISTICS\n    df_daily = df.groupby(['building_id', 'date', 'meter']).agg({'meter_reading':['std','mean','count']}).reset_index()\n    df_daily[('meter_reading','mean')] = np.log1p(df_daily[('meter_reading','mean')])\n    df_daily[('meter_reading','std')] = np.log1p(df_daily[('meter_reading','std')])\n    return df_daily","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading and processing train data\ndfs = [process_df(train)]\ndel train\ngc.collect()\ndfs[0].shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load and augment submission"},{"metadata":{},"cell_type":"markdown","source":"Insert your submission names here. Any number of them, but above 3 will be crowded ...\n\nFor this example I used output from 2 of the popular notebooks:\n- baseline from Konstantin Yakovlev (https://www.kaggle.com/kyakovlev/ashrae-baseline-lgbm)\n- higher-scoring Half and Half one from Vopani (https://www.kaggle.com/rohanrao/ashrae-half-and-half)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ls ../input/ -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Manually edit path, filenames and names. Names should correspond to filenames.\nsub_path = '../input/'\nsub_filenames = [\"ashrae-baseline-lgbm/submission.csv\",\"ashrae-half-and-half/submission.csv\" ]\nsub_names = [\"baseline\",\"half-and-half\" ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for sub_filename in sub_filenames:\n    print(f'adding submission {sub_path + sub_filename}: ', end='')\n    print('loading...', end='')\n    sub = pd.read_csv(sub_path + sub_filename)\n    print(', processing...')\n    dfs.append(process_df(sub))\n    del sub\n    gc.collect()\nprint('done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function to generate chart data and build charts\ndef chart_submissions (building_id, meter=0):\n    titles = ['2016 train data', *sub_names]\n    tmp_df = [df[(df.building_id == building_id) & (df.meter == meter)] \\\n                [['date', 'meter_reading']].set_index('date') for df in dfs]\n    if tmp_df[0].shape[0]:\n\n        fig, axes = plt.subplots(nrows=len(tmp_df), figsize=(18, 2+len(tmp_df)*2),)\n        fig.suptitle(f'Building {building_id}, meter {meter}', fontsize=18, y = 0.94)\n        max_y = np.concatenate([df[('meter_reading','mean')].values for df in tmp_df]).max() *  1.05\n\n        for i in range (3):\n#             tmp_df[i][('meter_reading', 'std')].plot(ax=axes[i], label='log_std')\n            tmp_df[i][('meter_reading', 'mean')].plot(ax=axes[i], label='log mean')\n#             tmp_df[i][('meter_reading', 'count')].plot(ax=axes[i], label='count')\n            axes[i].axvline(x=datetime.date(2018, 1, 1),  color='k', linestyle='--')\n\n            axes[i].legend()\n            axes[i].set_title(titles[i], fontsize=16, y = 0.8)\n            axes[i].set_ylim(0,max_y)\n        building_df[building_df.building_id==building_id]\n    else:\n        print (f\"Building_id={building_id}, Meter={meter} combination not present...\" )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Set desired building ID and meter ID in the cell below and see the resulting charts. Repeat as needed."},{"metadata":{"trusted":true},"cell_type":"code","source":"chart_submissions (building_id=0, meter=0)\n# some interesting ones: 107, 869, 1001, 0, 888","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"The example above shows how the missing data in building 0 was affecting forecast in baseline (score 1.25) and in a better developed kernel (score 1.10)."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}