{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Simple Benchmark using Load Profile\n\nIn this notebook I created a simple benchmark by taking the median (or average) of each meter's building per hour and day of the week. In short, the model is the following group by function:\n```\nmodel = X_train[['building_id', 'meter', 'weekday','hour','y_train']].groupby(['building_id', 'meter', 'weekday', 'hour']).median()\n```\n\nIn Electrical Engineering, the previous model is known as [Load Profile](https://en.wikipedia.org/wiki/Load_profile) and it is very common to be used as the representative energy model of a load (building+meter in our case). \n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport random\n\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nfrom matplotlib import pyplot as plt\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\n\npath_data = \"/kaggle/input/ashrae-energy-prediction/\"\npath_train = path_data + \"train.csv\"\npath_test = path_data + \"test.csv\"\npath_building = path_data + \"building_metadata.csv\"\npath_weather_train = path_data + \"weather_train.csv\"\npath_weather_test = path_data + \"weather_test.csv\"\n\nplt.style.use(\"seaborn\")\nsns.set(font_scale=1)\n\nmyfavouritenumber = 0\nseed = myfavouritenumber\nrandom.seed(seed)\n\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\ndef prepare_data(X, building_data, weather_data, test=False):\n    \"\"\"\n    Preparing final dataset with all features.\n    \"\"\"\n    \n    X = X.merge(building_data, on=\"building_id\", how=\"left\")\n    X = X.merge(weather_data, on=[\"site_id\", \"timestamp\"], how=\"left\")\n    \n    X.timestamp = pd.to_datetime(X.timestamp, format=\"%Y-%m-%d %H:%M:%S\")\n    X.square_feet = np.log1p(X.square_feet)\n    \n    if not test:\n        X.sort_values(\"timestamp\", inplace=True)\n        X.reset_index(drop=True, inplace=True)\n    \n    gc.collect()\n    \n    holidays = [\"2016-01-01\", \"2016-01-18\", \"2016-02-15\", \"2016-05-30\", \"2016-07-04\",\n                \"2016-09-05\", \"2016-10-10\", \"2016-11-11\", \"2016-11-24\", \"2016-12-26\",\n                \"2017-01-01\", \"2017-01-16\", \"2017-02-20\", \"2017-05-29\", \"2017-07-04\",\n                \"2017-09-04\", \"2017-10-09\", \"2017-11-10\", \"2017-11-23\", \"2017-12-25\",\n                \"2018-01-01\", \"2018-01-15\", \"2018-02-19\", \"2018-05-28\", \"2018-07-04\",\n                \"2018-09-03\", \"2018-10-08\", \"2018-11-12\", \"2018-11-22\", \"2018-12-25\",\n                \"2019-01-01\"]\n    \n    X[\"hour\"] = X.timestamp.dt.hour\n    X[\"weekday\"] = X.timestamp.dt.weekday\n    X[\"is_holiday\"] = (X.timestamp.dt.date.astype(\"str\").isin(holidays)).astype(int)\n    \n    drop_features = [\"timestamp\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\"]\n\n    X.drop(drop_features, axis=1, inplace=True)\n\n    if test:\n        row_ids = X.row_id\n        X.drop(\"row_id\", axis=1, inplace=True)\n        return X, row_ids\n    else:\n        y = np.log1p(X.meter_reading)\n        X.drop(\"meter_reading\", axis=1, inplace=True)\n        return X, y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"## 1. Reading CSV Files\nprint('## 1. Reading CSV files')\ndf_train = pd.read_csv(path_train)\n\nbuilding = pd.read_csv(path_building)\nle = LabelEncoder()\nbuilding.primary_use = le.fit_transform(building.primary_use)\n\nweather_train = pd.read_csv(path_weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test=pd.read_feather('/kaggle/input/ashrae-feather-format-for-fast-loading/test.feather')\nsubmission=pd.read_feather('/kaggle/input/ashrae-feather-format-for-fast-loading/sample_submission.feather')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"## 1.1 Reducing filesize\nprint('## 1.1 Reducing filesize')\ndf_train = reduce_mem_usage(df_train, use_float16=True)\nbuilding = reduce_mem_usage(building, use_float16=True)\nweather_train = reduce_mem_usage(weather_train, use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## 2. Preprocessing Data\nprint('## 2. Preprocessing Data')\nX_train, y_train = prepare_data(df_train, building, weather_train)\n\ndel df_train, weather_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('## 3. Fitting Model')\n\nX_train['meter_reading'] = y_train\nmodel = X_train[['building_id', 'meter', 'weekday','hour','meter_reading']].groupby(['building_id', 'meter', 'weekday', 'hour']).median()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['hour'] = test['timestamp'].dt.hour\ntest['weekday'] = test['timestamp'].dt.weekday\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model['meter_reading'].to_dict()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('## 4. Predicting')\n\nfrom tqdm import tqdm\n\npreds = []\n\nfor b, m, w, h in tqdm(test[['building_id', 'meter', 'weekday', 'hour']].values):\n    preds.append(np.expm1(model[b, m, w, h]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['meter_reading']=preds\nprint('converting to csv')\nprint(submission.head())\nsubmission.to_csv('submission.csv', index=False, float_format='%.4f')\nprint('Done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}