{"cells":[{"metadata":{},"cell_type":"markdown","source":" # **ASHRAE Energy Prediction**"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# Import Statements\nimport gc\nimport datetime\nimport pandas as pd\nimport numpy as np\n# import pickle\nimport lightgbm as lgb\n# from lightgbm import LGBMRegressor, plot_importance\n# from sklearn.metrics import mean_squared_log_error as msle, mean_squared_error as mse\n# from sklearn.model_selection import KFold, train_test_split\n# from sklearn.preprocessing import OneHotEncoder\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code from https://www.kaggle.com/caesarlupum/ashrae-start-here-a-gentle-introduction \n# Function to reduce the DF size\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\n# function to calculate evaluation metric\ndef rmsle(y_true: pd.Series, y_predict: pd.Series) -> float:\n    \"\"\"\n    Evaluate root mean squared log error\n    :param y_true:\n    :param y_predict:\n    :return:\n    \"\"\"\n    return np.sqrt(msle(y_true, y_predict))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Import data\nINPUT = \"../input/ashrae-energy-prediction/\"\n\ndf_train = pd.read_csv(f\"{INPUT}train.csv\")\n# df_test = pd.read_csv(f\"{INPUT}test.csv\")\nbldg_metadata = pd.read_csv(f\"{INPUT}building_metadata.csv\")\nweather_train = pd.read_csv(f\"{INPUT}weather_train.csv\")\n# weather_test = pd.read_csv(f\"{INPUT}weather_test.csv\")\nsample = pd.read_csv(f\"{INPUT}sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_lag_feature(weather_df, window=3):\n    group_df = weather_df.groupby('site_id')\n    cols = ['air_temperature'] \n#             'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']\n    rolled = group_df[cols].rolling(window=window, min_periods=0)\n    lag_mean = rolled.mean().reset_index().astype(np.float16)\n    lag_max = rolled.max().reset_index().astype(np.float16)\n    lag_min = rolled.min().reset_index().astype(np.float16)\n    lag_std = rolled.std().reset_index().astype(np.float16)\n    for col in cols:\n        weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]\n        weather_df[f'{col}_max_lag{window}'] = lag_max[col]\n        weather_df[f'{col}_min_lag{window}'] = lag_min[col]\n        weather_df[f'{col}_std_lag{window}'] = lag_std[col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_data(X,metadata,weather,test=False,new_df=None):\n    if test :\n        X = X.drop(columns=['row_id'])\n        \n    add_lag_feature(weather, window=72)\n    X = X.merge(metadata, on='building_id',how='left')  \n    X = X.merge(weather, on=['site_id', 'timestamp'], how='left')\n    X['timestamp'] = pd.to_datetime(arg=X['timestamp'])\n        \n    X['year'] = X['timestamp'].dt.year\n    X['month'] = X['timestamp'].dt.month\n    X['day'] = X['timestamp'].dt.day\n    X['hour'] = X['timestamp'].dt.hour\n    X['weekday'] = X['timestamp'].dt.weekday\n    \n    X = reduce_mem_usage(X)\n    \n    beaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8), (6, 10.8, 13.9), \n          (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5), (11, 28.5, 33), (12, 33, 200)]\n\n    for item in beaufort:\n        X.loc[(X['wind_speed']>=item[1]) & (X['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n\n    \n    X['age'] = X['year'] - X['year_built']\n    if new_df is None:\n        new_df = X.groupby(by=['building_id'], as_index=False)['timestamp'].min()\n        new_df = new_df.rename(columns = {'timestamp': 'start_ts'})\n    X = X.merge(new_df, on = 'building_id', how='left')\n    X['hours_passed'] = (X['timestamp'] - X['start_ts']).dt.total_seconds()/3600\n    X = reduce_mem_usage(X)\n    if not test:\n        X = X.query('not(site_id==0 & timestamp<\"2016-05-21 00:00:00\")')\n        \n    cols = ['floor_count', 'air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', \n        'wind_direction', 'wind_speed']\n    X.loc[:, cols] = X.loc[:, cols].interpolate(axis=0)\n    cat_cols = ['meter', 'primary_use', 'site_id', 'building_id', 'year', 'month', 'day', 'hour']\n    for col in cat_cols:\n        X[col] = X[col].astype('category')\n    X = reduce_mem_usage(X)   \n    if not test:\n        return X,new_df\n    else:\n        return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_test = df_test.drop(columns=['row_id'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = reduce_mem_usage(df=df_train)\n# df_test = reduce_mem_usage(df=df_test)\nweather_train = reduce_mem_usage(df=weather_train)\n# weather_test = reduce_mem_usage(df=weather_test)\nbldg_metadata = reduce_mem_usage(df=bldg_metadata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train,new_df = prepare_data(df_train,bldg_metadata,weather_train)\ndel weather_train;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = reduce_mem_usage(df_train)\ndf_train['group'] = df_train['month']\ndf_train['group'].replace((1,2,3,4,5,6), 1,inplace=True)\ndf_train['group'].replace((7,8,9,10,11,12), 2, inplace=True)\ndf_train['group'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numericals = [\"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\",\n              \"dew_temperature\", 'precip_depth_1_hr', 'floor_count', 'beaufort_scale','air_temperature_mean_lag72', 'air_temperature_max_lag72',\n       'air_temperature_min_lag72', 'air_temperature_std_lag72']\n#        'cloud_coverage_mean_lag72', 'cloud_coverage_max_lag72',\n#        'cloud_coverage_min_lag72', 'cloud_coverage_std_lag72',\n#        'dew_temperature_mean_lag72', 'dew_temperature_max_lag72',\n#        'dew_temperature_min_lag72', 'dew_temperature_std_lag72',\n#        'precip_depth_1_hr_mean_lag72', 'precip_depth_1_hr_max_lag72',\n#        'precip_depth_1_hr_min_lag72', 'precip_depth_1_hr_std_lag72',\n#        'sea_level_pressure_mean_lag72', 'sea_level_pressure_max_lag72',\n#        'sea_level_pressure_min_lag72', 'sea_level_pressure_std_lag72',\n#        'wind_direction_mean_lag72', 'wind_direction_max_lag72',\n#        'wind_direction_min_lag72', 'wind_direction_std_lag72',\n#        'wind_speed_mean_lag72', 'wind_speed_max_lag72', 'wind_speed_min_lag72',\n#        'wind_speed_std_lag72']\ncategoricals = [\"site_id\", \"building_id\", \"primary_use\", \"hour\", \"weekday\", \"meter\",  \"wind_direction\"]\ntarget = 'meter_reading'\nfeat_cols = categoricals + numericals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[target] = np.log1p(df_train[target])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### by month 6 vs 6"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_half_1 = df_train.loc[df_train.group==1][feat_cols]\nX_half_2 = df_train.loc[df_train.group==2][feat_cols]\ny_half_1 = df_train.loc[df_train.group==1][target]\ny_half_2 = df_train.loc[df_train.group==2][target]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categoricals, free_raw_data=False)\nd_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categoricals, free_raw_data=False)\nwatchlist_1 = [d_half_2, d_half_1]\nwatchlist_2 = [d_half_1, d_half_2]\nparams = {\n    \"objective\": \"regression\",\n    \"boosting\": \"gbdt\",#dart,gbdt\n    \"num_leaves\": 45,\n    \"learning_rate\": 0.02,\n    \"feature_fraction\": 0.9,\n    \"reg_lambda\": 2,\n    \"metric\": \"rmse\"\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Building model with first half and validating on second half:\")\nmodel_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=10, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)\n\nprint(\"Building model with second half and validating on first half:\")\nmodel_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=10, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [model_half_1,model_half_2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_half_1,X_half_2,d_half_1,d_half_2,df_train, watchlist_1,watchlist_2,y_half_1,y_half_2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### try devide by build_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from sklearn.metrics import mean_squared_error\n# import lightgbm as lgb\n# from sklearn.model_selection import KFold, StratifiedKFold\n# from tqdm import tqdm\n# y_train = df_train['meter_reading']\n# y_train = np.log1p(y_train)\n# X_train = df_train.drop(columns=['meter_reading','start_ts','timestamp'])\n# del df_train;gc.collect()\n\n# params = {\n#             'boosting_type': 'gbdt',\n#             'objective': 'regression',\n#             'metric': {'rmse'},\n#             'subsample': 0.25,\n#             'subsample_freq': 1,\n#             'learning_rate': 0.4,\n#             'num_leaves': 20,\n#             'feature_fraction': 0.9,\n#             'lambda_l1': 1,  \n#             'lambda_l2': 1\n#             }\n\n# folds = 4\n# seed = 666\n\n# kf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)\n\n# models = []\n# for train_index, val_index in kf.split(X_train, X_train['building_id']):\n#     train_X = X_train[feat_cols].iloc[train_index]\n#     val_X = X_train[feat_cols].iloc[val_index]\n#     train_y = y_train.iloc[train_index]\n#     val_y = y_train.iloc[val_index]\n#     lgb_train = lgb.Dataset(train_X, train_y, categorical_feature=categoricals)\n#     lgb_eval = lgb.Dataset(val_X, val_y, categorical_feature=categoricals)\n#     gbm = lgb.train(params,\n#                 lgb_train,\n#                 num_boost_round=1000,\n#                 valid_sets=(lgb_train, lgb_eval),\n#                 early_stopping_rounds=100,\n#                 verbose_eval = 100)\n#     models.append(gbm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import gc\n# del train_X, val_X, lgb_train, lgb_eval, train_y, val_y\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(f\"{INPUT}test.csv\")\nweather_test = pd.read_csv(f\"{INPUT}weather_test.csv\")\ndf_test = reduce_mem_usage(df=df_test)\nweather_test = reduce_mem_usage(df=weather_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = prepare_data(df_test,bldg_metadata,weather_test,test=True,new_df=new_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = reduce_mem_usage(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.drop(columns=['start_ts','timestamp'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del bldg_metadata,weather_test,new_df;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = df_test[feat_cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\ni=0\nres=[]\nstep_size = 50000\nfor j in tqdm(range(int(np.ceil(test.shape[0]/50000)))):\n    res.append(sum(np.expm1([model.predict(test.iloc[i:i+step_size]) for model in models])/len(models)))\n    i+=step_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = np.concatenate(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')\nsubmission['meter_reading'] = res\nsubmission.loc[submission['meter_reading']<0, 'meter_reading'] = 0\nsubmission.to_csv('submission.csv', index=False, float_format='%.4f')\n# submission.to_csv('submission.csv', index=False)\n# submission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"this is 1.1"},{"metadata":{},"cell_type":"markdown","source":"### replace to UCL data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# site 0\nfrom sklearn.metrics import mean_squared_error\nimport tqdm\nleak_score0 = 0\n\nleak_df = pd.read_pickle('/kaggle/input/ashrae-ucf-spider-and-eda-full-test-labels/site0.pkl') \nleak_df['meter_reading'] = leak_df.meter_reading_scraped\nleak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)\nleak_df.fillna(0, inplace=True)\nleak_df = leak_df[leak_df.timestamp.dt.year > 2016]\nleak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0 # remove large negative values\n\nsubmission.loc[submission.meter_reading < 0, 'meter_reading'] = 0\n\nfor bid in leak_df.building_id.unique():\n    temp_df = leak_df[(leak_df.building_id == bid)]\n    for m in temp_df.meter.unique():\n        v0 = submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'].values\n        v1 = temp_df[temp_df.meter==m].meter_reading.values\n        \n        leak_score0 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)\n        \n        submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values\n        \nleak_score0 /= len(leak_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# site 1\n\nleak_score1 = 0\n\nleak_df = pd.read_pickle('/kaggle/usr/lib/ucl_data_leakage_episode_2/site1.pkl') \nleak_df['meter_reading'] = leak_df.meter_reading_scraped\nleak_df.drop(['meter_reading_scraped'], axis=1, inplace=True)\nleak_df.fillna(0, inplace=True)\nleak_df = leak_df[leak_df.timestamp.dt.year > 2016]\nleak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0 # remove large negative values\n\n#sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0\n\nfor bid in leak_df.building_id.unique():\n    temp_df = leak_df[(leak_df.building_id == bid)]\n    for m in temp_df.meter.unique():\n        v0 = submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'].values\n        v1 = temp_df[temp_df.meter==m].meter_reading.values\n        \n        leak_score1 += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)\n        \n        submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values\n\nleak_score1 /= len(leak_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"this is 1.06"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":1}