{"cells":[{"metadata":{},"cell_type":"markdown","source":"The main purpose of this kernel is **to build a simple benchmark prediction without any  ML algorithm**.\nTo do this we will identify a couple of variables and apply the mean of the target on the test sample.\n\nA few important points:\n- we will model log(meter_reading+1);\n- we will model separately each of meter types;\n- we will use functions to run the same functionality on the training and testing dataset;"},{"metadata":{},"cell_type":"markdown","source":"# Import packages"},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T16:56:19.525694Z","start_time":"2019-10-16T16:56:15.221509Z"},"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\npd.set_option('max_colwidth', 500)\npd.set_option('max_columns', 500)\npd.set_option('max_rows', 100)\n\nimport gc\nfrom sklearn.metrics import mean_squared_error\n\nDATA = '../input/ashrae-energy-prediction/'\n\nprint(os.listdir(DATA))","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T16:56:20.518583Z","start_time":"2019-10-16T16:56:20.309016Z"},"trusted":true},"cell_type":"code","source":"%load_ext autoreload\n%autoreload 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper function to reduce memory footprint"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, force_obj_in_category=True, debug=True):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage. \n        This function originates from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n    \"\"\"\n    if debug:\n        start_mem = df.memory_usage(deep=True).sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object and df[col].dtype.name != 'category' and 'datetime' not in col_type.name:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                for i_type in [np.int8, np.int16, np.int32, np.int64]:\n                    if c_min > np.iinfo(i_type).min and c_max < np.iinfo(i_type).max:\n                        df[col] = df[col].astype(i_type)\n                        break\n            elif str(col_type)[:4] == 'uint':\n                for i_type in [np.uint8, np.uint16, np.uint32, np.uint64]:\n                    if c_max < np.iinfo(i_type).max:\n                        df[col] = df[col].astype(i_type)\n                        break\n            elif col_type == bool:\n                df[col] = df[col].astype(np.uint8)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        elif force_obj_in_category and 'datetime' not in col_type.name:\n            df[col] = df[col].astype('category')\n\n    if debug:\n        end_mem = df.memory_usage(deep=True).sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read the data"},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T16:56:20.655616Z","start_time":"2019-10-16T16:56:20.521743Z"},"trusted":true},"cell_type":"code","source":"def get_data(v):\n    '''\n    Read input data and merge with building and weather information.\n    \n    Parameters\n    -----------\n    v: string [train/test]\n        Read either tarining or test dataset\n    '''\n    df = pd.read_csv(f'{DATA}{v}.csv')#, parse_dates=['timestamp'])\n    df = reduce_mem_usage(df)\n    \n#     df_buildings = pd.read_csv(f'{DATA}building_metadata.csv')\n#     # drop arbitrary set of potentially irrelevant features\n#     df_buildings.drop(['primary_use', 'floor_count'],\n#                       axis=1, inplace=True)\n#     df_buildings = reduce_mem_usage(df_buildings, debug=False)\n#     df = df.merge(df_buildings, how='left', on='building_id')\n    \n#     df_weather = pd.read_csv(f'{DATA}weather_{v}.csv')\n#     # drop arbitrary set of potentially irrelevant features\n#     df_weather.drop(['cloud_coverage','dew_temperature','precip_depth_1_hr','sea_level_pressure','wind_direction','wind_speed'],\n#                     axis=1, inplace=True)\n#     df_weather = reduce_mem_usage(df_weather, debug=False)\n#     df = df.merge(df_weather, how='left', on=['site_id', 'timestamp'])\n    \n    if 'meter_reading' in df.columns:\n        df['target'] = np.log1p(df['meter_reading']).astype(np.float32)\n        del df['meter_reading']\n    \n    print(f'Final memory size of {v} dataset = {df.memory_usage(deep=True).sum()/2**20:.2f} MB')\n        \n    return df","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T17:45:01.466807Z","start_time":"2019-10-16T16:56:20.658640Z"},"trusted":true},"cell_type":"code","source":"df_trn = get_data('train')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Extract datetime features"},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T17:46:30.791062Z","start_time":"2019-10-16T17:46:30.709574Z"},"trusted":true},"cell_type":"code","source":"def get_ts_features(df):\n    df['timestamp'] = pd.to_datetime(df['timestamp'])\n    df['ts_day_of_month'] = df['timestamp'].dt.day.astype(np.uint8)\n    df['ts_month'] = df['timestamp'].dt.month.astype(np.uint8)\n    df['ts_hour'] = df['timestamp'].dt.hour.astype(np.uint8)\n    df['ts_day_of_week'] = (df['timestamp'].dt.weekday<=4).astype(np.uint8)\n    del df['timestamp']\n    return df","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T17:46:38.069716Z","start_time":"2019-10-16T17:46:33.129213Z"},"trusted":true},"cell_type":"code","source":"df_trn = get_ts_features(df_trn)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T17:47:44.909883Z","start_time":"2019-10-16T17:47:44.814370Z"},"trusted":true},"cell_type":"code","source":"df_trn.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Naive average model\n### Check average target as a function of several variables"},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T17:46:43.232896Z","start_time":"2019-10-16T17:46:40.054127Z"},"scrolled":false,"trusted":true},"cell_type":"code","source":"df_ts = {'d':[], 'm':[], 'h':[], 'w':[]}\n\nfor i in range(4):\n    x = df_trn.query('meter == @i')\n    \n    df_ts['d'].append(x.groupby('ts_day_of_month')['target'].mean())\n    df_ts['m'].append(x.groupby('ts_month')['target'].mean())\n    df_ts['h'].append(x.groupby('ts_hour')['target'].mean())\n    df_ts['w'].append(x.groupby('ts_day_of_week')['target'].mean())\n\n    \nfor k,d in df_ts.items():\n    for i,df in enumerate(d):\n        df.plot(label=i)\n    plt.title(f'{k}')\n    plt.legend()\n    plt.show()\n    \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see that different meter types have very different dependence on time"},{"metadata":{},"cell_type":"markdown","source":"## Read TEST"},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T18:20:59.710634Z","start_time":"2019-10-16T18:20:19.872480Z"},"trusted":true},"cell_type":"code","source":"df_tst = get_data('test')\ndf_tst = get_ts_features(df_tst)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T18:34:59.123034Z","start_time":"2019-10-16T18:34:53.057385Z"},"trusted":true},"cell_type":"code","source":"df_sub = pd.read_csv(f'{DATA}sample_submission.csv', index_col='row_id')\n_ = reduce_mem_usage(df_sub)","execution_count":null,"outputs":[]},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T18:27:02.892555Z","start_time":"2019-10-16T18:27:02.803513Z"},"trusted":true},"cell_type":"code","source":"df_tst.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build naive \"model\""},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T18:35:38.134332Z","start_time":"2019-10-16T18:35:20.439019Z"},"trusted":true},"cell_type":"code","source":"# iterate over meter types:\nfor j in range(4):\n    is_trn_j = (df_trn['meter'] == j)\n    is_tst_j = (df_tst['meter'] == j)\n\n    gb_cols = ['building_id', 'ts_month', 'ts_hour', 'ts_day_of_week']\n    # Naive model: get mean meter counts on the \"training\" subset\n    df_naive = df_trn[is_trn_j].groupby(gb_cols)['target'].mean()\n\n    # Apply naive model on the validation subset\n    x = df_tst[is_tst_j].merge(df_naive.rename('preds').to_frame(), right_index=True, left_on=gb_cols, how='left')\n\n    # fill missing values with the average in the training data\n    x['preds'].fillna(df_trn[is_trn_j]['target'].mean(), inplace=True)\n\n    # fill in the submission\n    df_sub.loc[x['row_id']] = x['preds'].values.reshape(-1,1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Transform predicted target back to the original scale"},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T18:35:45.803899Z","start_time":"2019-10-16T18:35:45.041499Z"},"trusted":true},"cell_type":"code","source":"df_sub = np.expm1(df_sub)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Write out predictions"},{"metadata":{"ExecuteTime":{"end_time":"2019-10-16T18:58:07.681024Z","start_time":"2019-10-16T18:56:04.927741Z"},"trusted":true},"cell_type":"code","source":"df_sub.to_csv('sub.csv', float_format='%.2f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false}},"nbformat":4,"nbformat_minor":1}