{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Divide and Conquer\nThis notebook is to explore features and optimize models for each site_id. The idea is to resolve some data discrepancies that are present by dividing the data rather than cleaning.   \n\nNote that this is just another approach, need not necessarily be better or worse, but probably can add some value to ensembles irrespective of its CV or public LB scores."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\n\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom tqdm.notebook import tqdm\n\npath_data = \"/kaggle/input/ashrae-energy-prediction/\"\npath_train = path_data + \"train.csv\"\npath_test = path_data + \"test.csv\"\npath_building = path_data + \"building_metadata.csv\"\npath_weather_train = path_data + \"weather_train.csv\"\npath_weather_test = path_data + \"weather_test.csv\"\n\nmyfavouritenumber = 13\nseed = myfavouritenumber","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preparing data\nThere are two files with features that need to be merged with the data. One is building metadata that has information on the buildings and the other is weather data that has information on the weather."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"df_train = pd.read_csv(path_train)\ndf_test = pd.read_csv(path_test)\n\nbuilding = pd.read_csv(path_building)\nle = LabelEncoder()\nbuilding.primary_use = le.fit_transform(building.primary_use)\n\nweather_train = pd.read_csv(path_weather_train)\nweather_test = pd.read_csv(path_weather_test)\n\nweather_train.drop([\"sea_level_pressure\", \"wind_direction\", \"wind_speed\"], axis=1, inplace=True)\nweather_test.drop([\"sea_level_pressure\", \"wind_direction\", \"wind_speed\"], axis=1, inplace=True)\n\nweather_train = weather_train.groupby(\"site_id\").apply(lambda group: group.interpolate(limit_direction=\"both\"))\nweather_test = weather_test.groupby(\"site_id\").apply(lambda group: group.interpolate(limit_direction=\"both\"))\n\ndf_train = df_train.merge(building, on=\"building_id\")\ndf_train = df_train.merge(weather_train, on=[\"site_id\", \"timestamp\"], how=\"left\")\ndf_train = df_train[~((df_train.site_id==0) & (df_train.meter==0) & (df_train.building_id <= 104) & (df_train.timestamp < \"2016-05-21\"))]\ndf_train.reset_index(drop=True, inplace=True)\ndf_train.timestamp = pd.to_datetime(df_train.timestamp, format='%Y-%m-%d %H:%M:%S')\ndf_train[\"log_meter_reading\"] = np.log1p(df_train.meter_reading)\n\ndf_test = df_test.merge(building, on=\"building_id\")\ndf_test = df_test.merge(weather_test, on=[\"site_id\", \"timestamp\"], how=\"left\")\ndf_test.reset_index(drop=True, inplace=True)\ndf_test.timestamp = pd.to_datetime(df_test.timestamp, format='%Y-%m-%d %H:%M:%S')\n\ndel building, le\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"## Memory Optimization\n\n# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n# Modified to support timestamp type, categorical type\n# Modified to add option to use float16\n\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = reduce_mem_usage(df_train, use_float16=True)\ndf_test = reduce_mem_usage(df_test, use_float16=True)\n\nweather_train.timestamp = pd.to_datetime(weather_train.timestamp, format='%Y-%m-%d %H:%M:%S')\nweather_test.timestamp = pd.to_datetime(weather_test.timestamp, format='%Y-%m-%d %H:%M:%S')\nweather_train = reduce_mem_usage(weather_train, use_float16=True)\nweather_test = reduce_mem_usage(weather_test, use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature Engineering: Time\nCreating time-based features."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"hour\"] = df_train.timestamp.dt.hour\ndf_train[\"weekday\"] = df_train.timestamp.dt.weekday\n\ndf_test[\"hour\"] = df_test.timestamp.dt.hour\ndf_test[\"weekday\"] = df_test.timestamp.dt.weekday","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature Engineering: Aggregation\nCreating aggregate features for buildings at various levels."},{"metadata":{"trusted":true},"cell_type":"code","source":"df_building_meter = df_train.groupby([\"building_id\", \"meter\"]).agg(mean_building_meter=(\"log_meter_reading\", \"mean\"),\n                                                             median_building_meter=(\"log_meter_reading\", \"median\")).reset_index()\n\ndf_train = df_train.merge(df_building_meter, on=[\"building_id\", \"meter\"])\ndf_test = df_test.merge(df_building_meter, on=[\"building_id\", \"meter\"])\n\ndf_building_meter_hour = df_train.groupby([\"building_id\", \"meter\", \"hour\"]).agg(mean_building_meter=(\"log_meter_reading\", \"mean\"),\n                                                                                median_building_meter=(\"log_meter_reading\", \"median\")).reset_index()\n\ndf_train = df_train.merge(df_building_meter_hour, on=[\"building_id\", \"meter\", \"hour\"])\ndf_test = df_test.merge(df_building_meter_hour, on=[\"building_id\", \"meter\", \"hour\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature Engineering: Lags\nCreating lag-based features. These are statistics of available features looking back in time by fixed intervals.   \nThese features are created in the weather data itself and then merged with the train and test data."},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_lag_features(df, window):\n    \"\"\"\n    Creating lag-based features looking back in time.\n    \"\"\"\n    \n    feature_cols = [\"air_temperature\", \"cloud_coverage\", \"dew_temperature\", \"precip_depth_1_hr\"]\n    df_site = df.groupby(\"site_id\")\n    \n    df_rolled = df_site[feature_cols].rolling(window=window, min_periods=0)\n    \n    df_mean = df_rolled.mean().reset_index().astype(np.float16)\n    df_median = df_rolled.median().reset_index().astype(np.float16)\n    df_min = df_rolled.min().reset_index().astype(np.float16)\n    df_max = df_rolled.max().reset_index().astype(np.float16)\n    df_std = df_rolled.std().reset_index().astype(np.float16)\n    df_skew = df_rolled.skew().reset_index().astype(np.float16)\n    \n    for feature in feature_cols:\n        df[f\"{feature}_mean_lag{window}\"] = df_mean[feature]\n        df[f\"{feature}_median_lag{window}\"] = df_median[feature]\n        df[f\"{feature}_min_lag{window}\"] = df_min[feature]\n        df[f\"{feature}_max_lag{window}\"] = df_max[feature]\n        df[f\"{feature}_std_lag{window}\"] = df_std[feature]\n        df[f\"{feature}_skew_lag{window}\"] = df_std[feature]\n        \n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Features\nCreating and selecting all the features."},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"weather_train = create_lag_features(weather_train, 18)\nweather_train.drop([\"air_temperature\", \"cloud_coverage\", \"dew_temperature\", \"precip_depth_1_hr\"], axis=1, inplace=True)\n\ndf_train = df_train.merge(weather_train, on=[\"site_id\", \"timestamp\"], how=\"left\")\n\ndel weather_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_features = [\n    \"building_id\",\n    \"primary_use\",\n    \"meter\",\n    \"weekday\",\n    \"hour\"\n]\n\nall_features = [col for col in df_train.columns if col not in [\"timestamp\", \"site_id\", \"meter_reading\", \"log_meter_reading\"]]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## KFold Cross Validation with LGBM\nSince the test data is out of time and longer than train data, creating a reliable validation strategy is going to be a major challenge. Just using a simple KFold CV here.\n\nThe folds are applied to each site individually, thus building 16 sites x 3 folds = 48 models in total."},{"metadata":{"trusted":true},"cell_type":"code","source":"cv = 2\nmodels = {}\ncv_scores = {\"site_id\": [], \"cv_score\": []}\n\nfor site_id in tqdm(range(16), desc=\"site_id\"):\n    print(cv, \"fold CV for site_id:\", site_id)\n    kf = KFold(n_splits=cv, random_state=seed)\n    models[site_id] = []\n\n    X_train_site = df_train[df_train.site_id==site_id].reset_index(drop=True)\n    y_train_site = X_train_site.log_meter_reading\n    y_pred_train_site = np.zeros(X_train_site.shape[0])\n    \n    score = 0\n\n    for fold, (train_index, valid_index) in enumerate(kf.split(X_train_site, y_train_site)):\n        X_train, X_valid = X_train_site.loc[train_index, all_features], X_train_site.loc[valid_index, all_features]\n        y_train, y_valid = y_train_site.iloc[train_index], y_train_site.iloc[valid_index]\n\n        dtrain = lgb.Dataset(X_train, label=y_train, categorical_feature=categorical_features)\n        dvalid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=categorical_features)\n\n        watchlist = [dtrain, dvalid]\n\n        params = {\"objective\": \"regression\",\n                  \"num_leaves\": 41,\n                  \"learning_rate\": 0.049,\n                  \"bagging_freq\": 5,\n                  \"bagging_fraction\": 0.51,\n                  \"feature_fraction\": 0.81,\n                  \"metric\": \"rmse\"\n                  }\n\n        model_lgb = lgb.train(params, train_set=dtrain, num_boost_round=999, valid_sets=watchlist, verbose_eval=101, early_stopping_rounds=21)\n        models[site_id].append(model_lgb)\n\n        y_pred_valid = model_lgb.predict(X_valid, num_iteration=model_lgb.best_iteration)\n        y_pred_train_site[valid_index] = y_pred_valid\n\n        rmse = np.sqrt(mean_squared_error(y_valid, y_pred_valid))\n        print(\"Site Id:\", site_id, \", Fold:\", fold+1, \", RMSE:\", rmse)\n        score += rmse / cv\n        \n        gc.collect()\n        \n    cv_scores[\"site_id\"].append(site_id)\n    cv_scores[\"cv_score\"].append(score)\n        \n    print(\"\\nSite Id:\", site_id, \", CV RMSE:\", np.sqrt(mean_squared_error(y_train_site, y_pred_train_site)), \"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame.from_dict(cv_scores)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"del df_train, X_train_site, y_train_site, X_train, y_train, dtrain, X_valid, y_valid, dvalid, y_pred_train_site, y_pred_valid, rmse, score, cv_scores\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Scoring on test data\nThe test data for each site is scored individually using the 3 models, one from each fold. The final prediction is the average of the 3 models."},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = create_lag_features(weather_test, 18)\nweather_test.drop([\"air_temperature\", \"cloud_coverage\", \"dew_temperature\", \"precip_depth_1_hr\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_sites = []\n\nfor site_id in tqdm(range(16), desc=\"site_id\"):\n    print(\"Preparing test data for site_id\", site_id)\n\n    X_test_site = df_test[df_test.site_id==site_id]\n    weather_test_site = weather_test[weather_test.site_id==site_id]\n    \n    X_test_site = X_test_site.merge(weather_test_site, on=[\"site_id\", \"timestamp\"], how=\"left\")\n    \n    row_ids_site = X_test_site.row_id\n\n    X_test_site = X_test_site[all_features]\n    y_pred_test_site = np.zeros(X_test_site.shape[0])\n\n    print(\"Scoring for site_id\", site_id)    \n    for fold in range(cv):\n        model_lgb = models[site_id][fold]\n        y_pred_test_site += model_lgb.predict(X_test_site, num_iteration=model_lgb.best_iteration) / cv\n        gc.collect()\n        \n    df_test_site = pd.DataFrame({\"row_id\": row_ids_site, \"meter_reading\": y_pred_test_site})\n    df_test_sites.append(df_test_site)\n    \n    print(\"Scoring for site_id\", site_id, \"completed\\n\")\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission\nPreparing final file for submission."},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.concat(df_test_sites)\nsubmit.meter_reading = np.clip(np.expm1(submit.meter_reading), 0, a_max=None)\nsubmit.to_csv(\"submission_noleak.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## adding leak\nleak0 = pd.read_csv(\"/kaggle/input/ashrae-leak-data/site0.csv\")\nleak1 = pd.read_csv(\"/kaggle/input/ashrae-leak-data/site1.csv\")\nleak2 = pd.read_csv(\"/kaggle/input/ashrae-leak-data/site2.csv\")\nleak4 = pd.read_csv(\"/kaggle/input/ashrae-leak-data/site4.csv\")\nleak15 = pd.read_csv(\"/kaggle/input/ashrae-leak-data/site15.csv\")\n\nleak = pd.concat([leak0, leak1, leak2, leak4, leak15])\n\ndel leak0, leak1, leak2, leak4, leak15, df_test_sites\ngc.collect()\n\ntest = pd.read_csv(path_test)\ntest = test[test.building_id.isin(leak.building_id.unique())]\n\nleak = leak.merge(test, on=[\"building_id\", \"meter\", \"timestamp\"])\n\ndel test\ngc.collect()\n\nsubmit = submit.merge(leak[[\"row_id\", \"meter_reading_scraped\"]], on=[\"row_id\"], how=\"left\")\nsubmit.loc[submit.meter_reading_scraped.notnull(), \"meter_reading\"] = submit.loc[submit.meter_reading_scraped.notnull(), \"meter_reading_scraped\"] \nsubmit.drop([\"meter_reading_scraped\"], axis=1, inplace=True)\n\nsubmit.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}