{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import gc\nimport os\nimport numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"path_external_data = \"/kaggle/input/ashrae-ucf-spider-and-eda-full-test-labels\"\npath_data = \"/kaggle/input/ashrae-energy-prediction/\"\npath_train = path_data + \"train.csv\"\npath_building = path_data + \"building_metadata.csv\"\npath_weather_train = path_data + \"weather_train.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import holidays\nen_holidays = holidays.England()\nir_holidays = holidays.Ireland()\nca_holidays = holidays.Canada()\nus_holidays = holidays.UnitedStates()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_train = pd.read_csv(path_train)\nbuilding = pd.read_csv(path_building)\nweather_train = pd.read_csv(path_weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train = weather_train.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\nbuilding.primary_use = le.fit_transform(building.primary_use)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#https://www.kaggle.com/isaienkov/lightgbm-fe-1-19\n\nbeaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8)\n            , (6, 10.8, 13.9), (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5)\n            , (11, 28.5, 33), (12, 33, 200)]\n\ndef average_imputation(df, column_name):\n    imputation = df.groupby(['timestamp'])[column_name].mean()\n    \n    df.loc[df[column_name].isnull(), column_name] = df[df[column_name].isnull()][[column_name]].apply(lambda x: imputation[df['timestamp'][x.index]].values)\n    del imputation\n    return df\n\ndef degToCompass(num):\n    val=int((num/22.5)+.5)\n    arr=[i for i in range(0,16)]\n    return arr[(val % 16)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#https://www.kaggle.com/c/ashrae-energy-prediction/discussion/114161#latest-658796\ndef relative_humidity(Tc,Tdc):\n    E = 6.11*10.0**(7.5*Tdc/(237.7+Tdc))\n    Es = 6.11*10.0**(7.5*Tc/(237.7+Tc))    \n    RH = (E/Es)*100\n    return RH","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"df_train['meter_reading'] = np.log1p(df_train['meter_reading'])\nbuilding_median = df_train.groupby('building_id')['meter_reading'].median().astype(np.float16)\nbuilding_mean = df_train.groupby('building_id')['meter_reading'].mean().astype(np.float16)\nbuilding_min = df_train.groupby('building_id')['meter_reading'].min().astype(np.float16)\nbuilding_max = df_train.groupby('building_id')['meter_reading'].max().astype(np.float16)\nbuilding_std = df_train.groupby('building_id')['meter_reading'].std().astype(np.float16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def prepare_data(X, building_data, weather_data, test=False):\n    \"\"\"\n    Preparing final dataset with all features.\n    \"\"\"\n    \n    X = X.merge(building_data, on=\"building_id\", how=\"left\")\n    X = X.merge(weather_data, on=[\"site_id\", \"timestamp\"], how=\"left\")\n    X['building_median'] = X['building_id'].map(building_median)\n    X['building_mean'] = X['building_id'].map(building_median)\n    X['building_min'] = X['building_id'].map(building_median)\n    X['building_max'] = X['building_id'].map(building_median)\n    X['building_std'] = X['building_id'].map(building_median)\n    #--------------------------------------\n    \n    X.timestamp = pd.to_datetime(X.timestamp, format=\"%Y-%m-%d %H:%M:%S\")\n    \n    # site_id = 0 has some building where meter readings before May 21, 2016 are not reliable so dropping those records \n    X = X.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')\n    \n    #--------------------------------------\n    #https://www.kaggle.com/isaienkov/lightgbm-fe-1-19\n    X = average_imputation(X, 'wind_speed')\n    X = average_imputation(X, 'wind_direction')\n     \n    for item in beaufort:\n        X.loc[(X['wind_speed']>=item[1]) & (X['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n        \n    X['wind_direction'] = X['wind_direction'].apply(degToCompass)\n    X['beaufort_scale'] = X['beaufort_scale'].astype(np.uint8)\n    X[\"wind_direction\"] = X['wind_direction'].astype(np.uint8)\n    \n    #--------------------------------------\n    \n    X.square_feet = np.log1p(X.square_feet)\n    \n    if not test:\n        X.sort_values(\"timestamp\", inplace=True)\n        X.reset_index(drop=True, inplace=True)\n    \n    gc.collect()\n    \n    #-------------------------------------\n    \n    X[\"hour\"] = X.timestamp.dt.hour\n    X[\"weekday\"] = X.timestamp.dt.weekday\n    \n    X[\"year\"] = X.timestamp.dt.year\n    X['age'] = X['year'] - X['year_built'] \n    \n    \n    #Jump from 429 to 408 (21 person)\n    # https://www.kaggle.com/c/ashrae-energy-prediction/discussion/115256#latest-669944\n    en_idx = X.query('site_id == 1 or site_id == 5').index\n    ir_idx = X.query('site_id == 12').index\n    ca_idx = X.query('site_id == 7 or site_id == 11').index\n    us_idx = X.query('site_id == 0 or site_id == 2 or site_id == 3 or site_id == 4 or site_id == 6 or site_id == 8 or site_id == 9 or site_id == 10 or site_id == 13 or site_id == 14 or site_id == 15').index\n    \n    X['is_holiday'] = 0\n    X.loc[en_idx, 'is_holiday'] = X.loc[en_idx, 'timestamp'].apply(lambda x: en_holidays.get(x, default=0))\n    X.loc[ir_idx, 'is_holiday'] = X.loc[ir_idx, 'timestamp'].apply(lambda x: ir_holidays.get(x, default=0))\n    X.loc[ca_idx, 'is_holiday'] = X.loc[ca_idx, 'timestamp'].apply(lambda x: ca_holidays.get(x, default=0))\n    X.loc[us_idx, 'is_holiday'] = X.loc[us_idx, 'timestamp'].apply(lambda x: us_holidays.get(x, default=0))\n    \n    holiday_idx = X['is_holiday'] != 0\n    X.loc[holiday_idx, 'is_holiday'] = 1\n    X['is_holiday'] = X['is_holiday'].astype(np.uint8)\n    \n    #weekends\n    X.loc[(X['weekday'] == 5) | (X['weekday'] == 6) , 'is_holiday'] = 1\n    \n    #-------------------------------------\n    #https://www.kaggle.com/c/ashrae-energy-prediction/discussion/114161#latest-658796\n    X['humidity'] = relative_humidity(X.air_temperature, X.dew_temperature).astype(np.float16)\n    #-------------------------------------\n    \n    drop_features = [\"timestamp\", \"sea_level_pressure\", \"year\", \"year_built\"]\n\n    X.drop(drop_features, axis=1, inplace=True)\n\n    if test:\n        row_ids = X.row_id\n        X.drop(\"row_id\", axis=1, inplace=True)\n        return X, row_ids\n    else:\n        y = X.meter_reading\n        X.drop(\"meter_reading\", axis=1, inplace=True)\n        return X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"X_train, y_train = prepare_data(df_train, building, weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#del df_train, weather_train\n#gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"X_half_1 = X_train[:int(X_train.shape[0] / 2)]\nX_half_2 = X_train[int(X_train.shape[0] / 2):]\ndel X_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_half_1 = y_train[:int(y_train.shape[0] / 2)]\ny_half_2 = y_train[int(y_train.shape[0] / 2):]\ndel y_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"categorical_features = [\"building_id\", \"site_id\", \"meter\", \"primary_use\", \"hour\", \"weekday\"]\n\nn_estimators=40000 \nlearning_rate=0.04\nbagging_fraction=0.7\nfeature_fraction=0.8\nlambda_l2=2\nmetric=\"rmse\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import lightgbm as lgb\nmodel_half_1 = lgb.LGBMRegressor(n_estimators=n_estimators, \n                                 learning_rate=learning_rate,\n                                 bagging_fraction=bagging_fraction,\n                                 feature_fraction=feature_fraction, \n                                 lambda_l2=lambda_l2,\n                                 metric=metric)\nmodel_half_2 = lgb.LGBMRegressor(n_estimators=n_estimators, \n                                 learning_rate=learning_rate,\n                                 bagging_fraction=bagging_fraction,\n                                 feature_fraction=feature_fraction, \n                                 lambda_l2=lambda_l2,\n                                 metric=metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#from catboost import CatBoostRegressor\n#model_half_1 = CatBoostRegressor(iterations=n_estimators, \n#                                 learning_rate=learning_rate,\n#                                 bagging_temperature=bagging_fraction,\n#                                 l2_leaf_reg=lambda_l2,\n#                                 eval_metric=metric.upper())\n#model_half_2 = CatBoostRegressor(iterations=n_estimators, \n#                                 learning_rate=learning_rate,\n#                                 bagging_temperature=bagging_fraction,\n#                                 l2_leaf_reg=lambda_l2,\n#                                 eval_metric=metric.upper())","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"print(\"Building model with first half and validating on second half:\")\nmodel_half_1.fit(X_half_1,\n                 y_half_1,\n                 eval_set=[(X_half_1,y_half_1),(X_half_2,y_half_2)],\n                 categorical_feature=categorical_features, \n                 #cat_features=categorical_features,\n                 early_stopping_rounds=200,\n                 verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"print(\"Building model with second half and validating on first half:\")\nmodel_half_2.fit(X_half_2,\n                 y_half_2,\n                 eval_set=[(X_half_2,y_half_2),(X_half_1,y_half_1)],\n                 categorical_feature=categorical_features,\n                 early_stopping_rounds=200,\n                 verbose=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del X_half_1, X_half_2, y_half_1, y_half_2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Test**"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_test = pd.read_csv(path_data + \"test.csv\")\nweather_test = pd.read_csv(path_data + \"weather_test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_test = weather_test.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"X_test, row_ids = prepare_data(df_test, building, weather_test, test=True)\npred = np.expm1(model_half_1.predict(X_test, num_iteration=model_half_1.best_iteration_)) / 2\ndel model_half_1\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pred += np.expm1(model_half_2.predict(X_test, num_iteration=model_half_2.best_iteration_)) / 2\ndel model_half_2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission = pd.DataFrame({\"row_id\": row_ids, \"meter_reading\": np.clip(pred, 0, a_max=None)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"leak_df = pd.read_pickle(path_external_data+'site0.pkl') \nleak_df['meter_reading'] = leak_df.meter_reading_scraped\nleak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)\nleak_df.fillna(0, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for bid in leak_df.building_id.unique():\n    if bid % 25 == 0:\n        print(bid)\n    temp_df = leak_df[(leak_df.building_id == bid) & (leak_df.timestamp.dt.year > 2016)]\n    for m in temp_df.meter.unique():\n        submission.loc[(df_test.building_id == bid)&(df_test.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.3"}},"nbformat":4,"nbformat_minor":1}