{"cells":[{"metadata":{"trusted":false},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport sklearn\nimport matplotlib.pyplot as plt\nimport gc\nfrom sklearn.preprocessing import StandardScaler\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# taken from https://www.kaggle.com/aitude/ashrae-missing-weather-data-handling\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Reading the Data"},{"metadata":{"trusted":false},"cell_type":"code","source":"path = \"/kaggle/input/ashrae-energy-prediction/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"data_dtype = {'meter':\"uint8\",'building_id':'uint16'}\n\ntrain = pd.read_csv(path + \"train.csv\", delimiter=\",\", dtype=data_dtype)\ntest = pd.read_csv(path + \"test.csv\", delimiter=\",\", dtype=data_dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"metadata_dtype = {'site_id':\"uint8\",'building_id':'uint16','square_feet':'float32','year_built':'float32','floor_count':\"float16\"}\nweather_dtype = {\"site_id\":\"uint8\"}\n\nweather_train = pd.read_csv(path + \"weather_train.csv\", delimiter=\",\", dtype=weather_dtype)\nweather_test = pd.read_csv(path + \"weather_test.csv\", delimiter=\",\", dtype=weather_dtype)\n\nmetadata = pd.read_csv(path + \"building_metadata.csv\", delimiter=\",\", dtype=metadata_dtype)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Encoding primary_use"},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\nmetadata.primary_use = le.fit_transform(metadata.primary_use)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Filling missing data\nBased on https://www.kaggle.com/aitude/ashrae-missing-weather-data-handling. I made some changes in code to simplify it."},{"metadata":{"trusted":false},"cell_type":"code","source":"import datetime\ndef fill_weather_missing_rows(weather_data):\n    time_format = \"%Y-%m-%d %H:%M:%S\"\n    start_date = datetime.datetime.strptime(weather_data['timestamp'].min(),time_format)\n    end_date = datetime.datetime.strptime(weather_data['timestamp'].max(),time_format)\n    total_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)\n    hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format) for x in range(total_hours)]\n    missing_hours = []\n    for site_id in range(16):\n        \n        site_hours = np.array(weather_data[weather_data['site_id'] == site_id]['timestamp'])\n        new_rows = pd.DataFrame(np.setdiff1d(hours_list, site_hours),columns=['timestamp'])\n        new_rows['site_id'] = site_id\n        weather_data = pd.concat([weather_data, new_rows])\n    return weather_data.reset_index(drop=True) \n\ndef extend_timestamp_weather(df):\n    df[\"datetime\"] = pd.to_datetime(df[\"timestamp\"])\n    df[\"day\"] = df[\"datetime\"].dt.day\n    df[\"week\"] = df[\"datetime\"].dt.week\n    df[\"month\"] = df[\"datetime\"].dt.month\n    return df.set_index(['site_id','day','month'])\n\ndef fill_weather_missing_values(df):    \n    missing_hours = [\"air_temperature\",\"dew_temperature\",\"wind_direction\", \"wind_speed\"]\n    missing_days = [\"cloud_coverage\", \"sea_level_pressure\", \"precip_depth_1_hr\"]\n    \n    for col in missing_hours:\n        filler = pd.DataFrame(df.groupby(['site_id','day','month'])[col].mean(),columns=[col])\n        df.update(filler,overwrite=False) \n        \n    for col in missing_days:\n        filler = df.groupby(['site_id','day','month'])[col].mean()\n        filler = pd.DataFrame(filler.fillna(method='ffill'),columns=[col])\n        df.update(filler,overwrite=False)\n        #'datetime','day','week','month'\n    df = df.reset_index().drop(['datetime','day','week','month'],axis=1)   \n\n    return df\n\ndef fill_weather(df):\n    return fill_weather_missing_values(extend_timestamp_weather(fill_weather_missing_rows(df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def merge_sets(data, metadata, weather):\n    data = pd.merge(data, metadata, on='building_id',how='left')\n    gc.collect()\n    data = pd.merge(data, weather,how='left',on=['site_id','timestamp'])\n    gc.collect()\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def engineer_metadata(data=metadata):\n    data.drop(\"floor_count\", axis=1, inplace=True)\n    data['year_built'].fillna(-999, inplace=True)\n    data['year_built'] = data['year_built'] - 1900\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train = fill_weather(weather_train)\nmetadata = engineer_metadata()\ntrain = merge_sets(train, metadata, weather_train)\ndel weather_train\ngc.collect()\n\nweather_test = fill_weather(weather_test)\ntest = merge_sets(test, metadata, weather_test)\ndel weather_test, metadata\nreduce_mem_usage(test)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Further Feature Engineering"},{"metadata":{"trusted":false},"cell_type":"code","source":"def feature_engineering(data=train, cat_cols=None, drop_meter_readings=False, drop_outliers=False, drop_zero_el=False, cols_to_drop=None, log_cols=[], std_cols=None, shuffle_train=False):   \n    # https://www.kaggle.com/c/ashrae-energy-prediction/discussion/117083\n    if drop_zero_el:\n        idx_to_drop = list(data[(data['meter'] == \"Electricity\") & (data['meter_reading'] == 0)].index)\n        data.drop(idx_to_drop,axis='rows',inplace=True)\n\n    if drop_meter_readings:\n        idx_to_drop = list((train[(data['site_id'] == 0) & (data['timestamp'] < \"2016-05-21 00:00:00\")]).index)\n        data.drop(idx_to_drop,axis='rows',inplace=True)\n      \n    # got the idea to include holidays from here https://www.kaggle.com/rohanrao/ashrae-half-and-half\n    holidays = [\"2016-01-01\", \"2016-01-18\", \"2016-02-15\", \"2016-05-30\", \"2016-07-04\",\n            \"2016-09-05\", \"2016-10-10\", \"2016-11-11\", \"2016-11-24\", \"2016-12-26\",\n            \"2017-01-01\", \"2017-01-16\", \"2017-02-20\", \"2017-05-29\", \"2017-07-04\",\n            \"2017-09-04\", \"2017-10-09\", \"2017-11-10\", \"2017-11-23\", \"2017-12-25\",\n            \"2018-01-01\", \"2018-01-15\", \"2018-02-19\", \"2018-05-28\", \"2018-07-04\",\n            \"2018-09-03\", \"2018-10-08\", \"2018-11-12\", \"2018-11-22\", \"2018-12-25\",\n            \"2019-01-01\"]\n    \n    data[\"timestamp\"] = pd.to_datetime(data[\"timestamp\"])\n    data[\"is_holiday\"] = (data.timestamp.dt.date.astype(\"str\").isin(holidays)).astype(int)\n    data['month'] = data['timestamp'].dt.month.astype(\"uint8\")\n    data['day'] = data['timestamp'].dt.day.astype(\"uint8\")\n    data['weekday'] = data['timestamp'].dt.dayofweek.astype(\"uint8\")\n    data['hour'] = data['timestamp'].dt.hour.astype(\"uint8\")\n    gc.collect()\n    \n    # IIRC I got the idea for that in here https://www.kaggle.com/starl1ght/ashrae-stacked-regression-lasso-ridge-lgbm\n    if drop_outliers:\n        data = data[data['building_id'] != 1099 ]\n        data = data.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')\n    gc.collect()\n    \n    if cols_to_drop is not None:\n        data.drop(cols_to_drop, inplace=True, axis=1)\n    \n    for col in log_cols:\n        data[col] = np.log1p(data[col])\n        \n    # ended up not using this\n    if std_cols is not None:\n        scaler = StandardScaler()\n        scaler.fit(train[std_cols])\n        data[std_cols] = scaler.transform(data[std_cols])\n    \n    # ended up not using this\n    if cat_cols is not None:\n        data = pd.get_dummies(data, columns=categorical_cols, sparse=True)\n    \n\n    gc.collect()\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def splitXy(data):\n    return data.drop(\"meter_reading\",axis=1), data.meter_reading","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"categorical_cols = ['building_id','month','meter','hour','primary_use','weekday','day']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"X_tr, y_tr = splitXy(feature_engineering(\n    drop_meter_readings=True,\n    drop_zero_el=True, \n    drop_outliers=True,\n    cols_to_drop=[\"timestamp\",\"sea_level_pressure\", \"wind_direction\", \"wind_speed\",\"year_built\"], \n    log_cols=['square_feet', \"meter_reading\"],\n    std_cols=None\n))\ndel train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"row_ids = test.row_id\nX_te = feature_engineering(\n    data=test,\n    drop_meter_readings=False,\n    drop_zero_el=False, \n    drop_outliers=False,\n    cols_to_drop=[\"timestamp\",\"sea_level_pressure\", \"wind_direction\", \"wind_speed\",\"year_built\"], \n    log_cols=['square_feet'],\n    std_cols=None\n)\ndel test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Training\nInitial model is based on https://www.kaggle.com/morituri/lgbm-baseline and https://www.kaggle.com/aitude/ashrae-kfold-lightgbm-without-leak-1-08. Here I tried to train a model for each site but it ended up being worse than just K-Fold LGBM."},{"metadata":{"trusted":false},"cell_type":"code","source":"import lightgbm as lgb\nfrom sklearn.model_selection import KFold\n\nparams = {\"objective\": \"regression\",\n                  \"num_leaves\": 40,\n                  \"learning_rate\": 0.05,\n                  \"boosting\": \"gbdt\",\n                  \"bagging_freq\": 5,\n                  \"feature_fraction\": 0.85,\n                  \"bagging_fraction\": 0.51,\n                  \"metric\": \"rmse\"\n                  }\n\nfor i in range(16):\n    print(\"----------------\")\n    print(\"Model\", i)\n    idx = (X_tr.site_id == i)\n    X = X_tr.loc[idx].drop(\"site_id\", axis=1)\n    y = y_tr.loc[idx]\n    \n    del idx\n    gc.collect() \n    \n    kf = KFold(n_splits=3)\n    models = []\n    for train_index,test_index in kf.split(X):\n        train_features = X.iloc[train_index]\n        train_target = y.iloc[train_index]\n\n        test_features = X.iloc[test_index]\n        test_target = y.iloc[test_index]\n\n        d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_cols, free_raw_data=False)\n        d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_cols, free_raw_data=False)\n\n        model = lgb.train(params, train_set=d_training, num_boost_round=1000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)\n        models.append(model)\n        del train_features, train_target, test_features, test_target, d_training, d_test\n        gc.collect()\n        \n    del X, y\n    gc.collect() \n    \n    print(\"Predicting...\")\n    idx = (X_te.site_id == i)\n    test_features = X_te.loc[idx].drop([\"site_id\", \"row_id\"], axis=1)\n    \n    results = []\n    for model in models:\n        if  results == []:\n            results = np.expm1(model.predict(test_features, num_iteration=model.best_iteration)) / len(models)\n        else:\n            results += np.expm1(model.predict(test_features, num_iteration=model.best_iteration)) / len(models)\n        del model\n        gc.collect()\n    del models  \n    gc.collect()\n    \n    results = pd.DataFrame({\"row_id\": X_te.row_id[idx], \"meter_reading\": np.clip(results, 0, a_max=None)})\n    results.to_csv('results.csv', mode='a', header=False)\n    del results, test_features, idx\n    gc.collect() \n    \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Saving results could be done more simply, but I was too lazy to change it and start the calculation again."},{"metadata":{"trusted":false},"cell_type":"code","source":"del X_tr, y_tr, X_te\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df = pd.read_csv(\"results.csv\", names=['row_id', 'meter_reading'], header=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}