{"cells":[{"metadata":{},"cell_type":"markdown","source":"One of the key base models in my top 1% (25th place) solution.\nGreatly influenced by/borrowed from the following great kernels - please go and upvote the original work of the authors:\n\nhttps://www.kaggle.com/rohanrao/ashrae-divide-and-conquer  \nhttps://www.kaggle.com/purist1024/ashrae-simple-data-cleanup-lb-1-08-no-leaks  \nhttps://www.kaggle.com/roydatascience/ashrae-energy-prediction-using-stratified-kfold  "},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport sklearn\nsklearn.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install meteocalc\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.experimental import enable_hist_gradient_boosting\nfrom sklearn.ensemble import HistGradientBoostingRegressor\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, StratifiedKFold, GroupKFold\nfrom tqdm import tqdm_notebook as tqdm\nimport datetime\nfrom meteocalc import feels_like, Temp\nfrom sklearn import metrics\nimport gc\nimport os\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom tqdm.notebook import tqdm\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv')\nbuilding_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\nweather_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Original code from https://www.kaggle.com/aitude/ashrae-missing-weather-data-handling by @aitude\ndef fill_weather_dataset(weather_df):\n    \n    # Find Missing Dates\n    time_format = \"%Y-%m-%d %H:%M:%S\"\n    start_date = datetime.datetime.strptime(weather_df['timestamp'].min(),time_format)\n    end_date = datetime.datetime.strptime(weather_df['timestamp'].max(),time_format)\n    total_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)\n    hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format) for x in range(total_hours)]\n\n    missing_hours = []\n    for site_id in range(16):\n        site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])\n        new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])\n        new_rows['site_id'] = site_id\n        weather_df = pd.concat([weather_df,new_rows])\n\n        weather_df = weather_df.reset_index(drop=True)           \n\n    # Add new Features\n    weather_df[\"datetime\"] = pd.to_datetime(weather_df[\"timestamp\"])\n    weather_df[\"day\"] = weather_df[\"datetime\"].dt.day\n    weather_df[\"week\"] = weather_df[\"datetime\"].dt.week\n    weather_df[\"month\"] = weather_df[\"datetime\"].dt.month\n    \n    # Reset Index for Fast Update\n    weather_df = weather_df.set_index(['site_id','day','month'])\n\n    air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean(),columns=[\"air_temperature\"])\n    weather_df.update(air_temperature_filler,overwrite=False)\n\n    # Step 1\n    cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()\n    # Step 2\n    cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=[\"cloud_coverage\"])\n\n    weather_df.update(cloud_coverage_filler,overwrite=False)\n\n    due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean(),columns=[\"dew_temperature\"])\n    weather_df.update(due_temperature_filler,overwrite=False)\n\n    # Step 1\n    sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()\n    # Step 2\n    sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])\n\n    weather_df.update(sea_level_filler,overwrite=False)\n\n    wind_direction_filler =  pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean(),columns=['wind_direction'])\n    weather_df.update(wind_direction_filler,overwrite=False)\n\n    wind_speed_filler =  pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean(),columns=['wind_speed'])\n    weather_df.update(wind_speed_filler,overwrite=False)\n\n    # Step 1\n    precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()\n    # Step 2\n    precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])\n\n    weather_df.update(precip_depth_filler,overwrite=False)\n\n    weather_df = weather_df.reset_index()\n    weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)\n    \n    def get_meteorological_features(data):\n        def calculate_rh(df):\n            df['relative_humidity'] = 100 * (np.exp((17.625 * df['dew_temperature']) / (243.04 + df['dew_temperature'])) / np.exp((17.625 * df['air_temperature'])/(243.04 + df['air_temperature'])))\n        def calculate_fl(df):\n            flike_final = []\n            flike = []\n            # calculate Feels Like temperature\n            for i in range(len(df)):\n                at = df['air_temperature'][i]\n                rh = df['relative_humidity'][i]\n                ws = df['wind_speed'][i]\n                flike.append(feels_like(Temp(at, unit = 'C'), rh, ws))\n            for i in range(len(flike)):\n                flike_final.append(flike[i].f)\n            df['feels_like'] = flike_final\n            del flike_final, flike, at, rh, ws\n        calculate_rh(data)\n        calculate_fl(data)\n        return data\n\n    weather_df = get_meteorological_features(weather_df)\n    \n    beaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8), (6, 10.8, 13.9), \n          (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5), (11, 28.5, 33), (12, 33, 200)]\n    for item in beaufort:\n        weather_df.loc[(weather_df['wind_speed']>=item[1]) & (weather_df['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n    \n    return weather_df\n\n# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\n\ndef features_engineering(df):\n    \n    # Sort by timestamp\n    df.sort_values(\"timestamp\")\n    df.reset_index(drop=True)\n    \n    # Add more features\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"],format=\"%Y-%m-%d %H:%M:%S\")\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"dayofweek\"] = df[\"timestamp\"].dt.weekday\n    holidays = [\"2016-01-01\", \"2016-01-18\", \"2016-02-15\", \"2016-05-30\", \"2016-07-04\",\n                    \"2016-09-05\", \"2016-10-10\", \"2016-11-11\", \"2016-11-24\", \"2016-12-26\",\n                    \"2017-01-02\", \"2017-01-16\", \"2017-02-20\", \"2017-05-29\", \"2017-07-04\",\n                    \"2017-09-04\", \"2017-10-09\", \"2017-11-10\", \"2017-11-23\", \"2017-12-25\",\n                    \"2018-01-01\", \"2018-01-15\", \"2018-02-19\", \"2018-05-28\", \"2018-07-04\",\n                    \"2018-09-03\", \"2018-10-08\", \"2018-11-12\", \"2018-11-22\", \"2018-12-25\",\n                    \"2019-01-01\"]\n    df['group'] = df['timestamp'].dt.month\n    df['group'].replace((1, 2, 3, 4), 1, inplace = True)\n    df['group'].replace((5, 6, 7, 8), 2, inplace = True)\n    df['group'].replace((9, 10, 11, 12), 3, inplace = True)\n    df[\"is_holiday\"] = (df.timestamp.isin(holidays)).astype(int)\n    df['square_feet'] =  np.log1p(df['square_feet'])\n    \n    \n    # Remove Unused Columns\n    drop = [\"timestamp\",\"is_holiday\"]\n    df = df.drop(drop, axis=1)\n    gc.collect()\n    \n    # Encode Categorical Data\n    le = LabelEncoder()\n    df[\"primary_use\"] = le.fit_transform(df[\"primary_use\"])\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# weather manipulation\nweather_df = fill_weather_dataset(weather_df)\n\n# memory reduction\ntrain_df = reduce_mem_usage(train_df,use_float16=True)\nbuilding_df = reduce_mem_usage(building_df,use_float16=True)\nweather_df = reduce_mem_usage(weather_df,use_float16=True)\n\n# merge data\ntrain_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')\ntrain_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])\ndel weather_df\ngc.collect()\n\n# feature engineering\ntrain_df = features_engineering(train_df)\n\n# transform target variable\ntrain_df['meter_reading'] = np.log1p(train_df[\"meter_reading\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# eliminate bad rows\nbad_rows = pd.read_csv('../input/bad-data/rows_to_drop.csv')\ntrain_df.drop(bad_rows.loc[:, '0'], inplace = True)\ntrain_df.reset_index(drop = True, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train model \n### KFold Cross Validation with LGBM"},{"metadata":{"trusted":true},"cell_type":"code","source":"# declare target, categorical and numeric columns\ntarget = 'meter_reading'\ncategorical_features = ['building_id', 'site_id', 'primary_use', 'meter', 'dayofweek']\nnumeric_features = [col for col in train_df.columns if col not in categorical_features + [target, 'timestamp', 'group']]\nall_features = categorical_features + numeric_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cv = 4\nmodels = {}\nseed = 42\ncv_scores = {\"site_id\": [], \"cv_score\": []}\n\nfor site_id in tqdm(range(16), desc=\"site_id\"):\n    print(cv, \"fold CV for site_id:\", site_id)\n    kf = KFold(n_splits=cv, random_state=seed)\n    models[site_id] = []\n\n    X_train_site = train_df[train_df.site_id==site_id].reset_index(drop=True)\n    y_train_site = X_train_site.meter_reading\n    y_pred_train_site = np.zeros(X_train_site.shape[0])\n    \n    score = 0\n\n    for fold, (train_index, valid_index) in enumerate(kf.split(X_train_site, y_train_site)):\n        X_train, X_valid = X_train_site.loc[train_index, all_features], X_train_site.loc[valid_index, all_features]\n        y_train, y_valid = y_train_site.iloc[train_index], y_train_site.iloc[valid_index]\n\n        dtrain = lgb.Dataset(X_train, label=y_train, categorical_feature=categorical_features)\n        dvalid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=categorical_features)\n\n        watchlist = [dtrain, dvalid]\n\n        params = {\"objective\": \"regression\",\n                  \"num_leaves\": 41,\n                  \"learning_rate\": 0.049,\n                  \"bagging_freq\": 5,\n                  \"bagging_fraction\": 0.51,\n                  \"feature_fraction\": 0.81,\n                  \"metric\": \"rmse\"\n                  }\n\n        model_lgb = lgb.train(params, train_set=dtrain, num_boost_round=999, valid_sets=watchlist, verbose_eval=101, early_stopping_rounds=21)\n        models[site_id].append(model_lgb)\n\n        y_pred_valid = model_lgb.predict(X_valid, num_iteration=model_lgb.best_iteration)\n        y_pred_train_site[valid_index] = y_pred_valid\n\n        rmse = np.sqrt(mean_squared_error(y_valid, y_pred_valid))\n        print(\"Site Id:\", site_id, \", Fold:\", fold+1, \", RMSE:\", rmse)\n        score += rmse / cv\n        \n        gc.collect()\n        \n    cv_scores[\"site_id\"].append(site_id)\n    cv_scores[\"cv_score\"].append(score)\n        \n    print(\"\\nSite Id:\", site_id, \", CV RMSE:\", np.sqrt(mean_squared_error(y_train_site, y_pred_train_site)), \"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# read test\ntest_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv')\nrow_ids = test_df[\"row_id\"]\ntest_df.drop(\"row_id\", axis=1, inplace=True)\ntest_df = reduce_mem_usage(test_df)\n\n# merge with building info\ntest_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')\ndel building_df\ngc.collect()\n\n# fill test weather data\nweather_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv')\nweather_df = fill_weather_dataset(weather_df)\nweather_df = reduce_mem_usage(weather_df)\n\n# merge weather data\ntest_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])\ndel weather_df\ngc.collect()\n\n# feature engineering\ntest_df = features_engineering(test_df)\ntest_df[\"row_id\"] = row_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df_sites = []\n\nfor site_id in tqdm(range(16), desc=\"site_id\"):\n    print(\"Preparing test data for site_id\", site_id)\n\n    X_test_site = test_df[test_df.site_id==site_id]\n    #weather_test_site = weather_test[weather_test.site_id==site_id]\n    \n    #X_test_site = X_test_site.merge(weather_test_site, on=[\"site_id\", \"timestamp\"], how=\"left\")\n    \n    row_ids_site = X_test_site.row_id\n\n    X_test_site = X_test_site[all_features]\n    y_pred_test_site = np.zeros(X_test_site.shape[0])\n\n    print(\"Scoring for site_id\", site_id)    \n    for fold in range(cv):\n        model_lgb = models[site_id][fold]\n        y_pred_test_site += model_lgb.predict(X_test_site, num_iteration=model_lgb.best_iteration) / cv\n        gc.collect()\n        \n    test_df_site = pd.DataFrame({\"row_id\": row_ids_site, \"meter_reading\": y_pred_test_site})\n    test_df_sites.append(test_df_site)\n    \n    print(\"Scoring for site_id\", site_id, \"completed\\n\")\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.concat(test_df_sites)\nsubmit.meter_reading = np.clip(np.expm1(submit.meter_reading), 0, a_max=None)\nsubmit.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}