{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","colab":{"base_uri":"https://localhost:8080/","height":74},"colab_type":"code","id":"-VDEp3w3-ESB","outputId":"cb761176-e35b-4c52-a0c1-d55a361d87b9","trusted":true},"cell_type":"code","source":"## This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nimport gc\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import mean_squared_log_error\nimport hyperopt\nfrom hyperopt import fmin, tpe, hp, STATUS_OK, Trials\nfrom hyperopt import space_eval\nimport time\nimport math\nfrom hyperopt.pyll.base import scope\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import plot_importance\nfrom tqdm import tqdm_notebook as tqdm\nimport catboost\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, Pool\nfrom sklearn.preprocessing import LabelEncoder\nimport pprint\npp = pprint.PrettyPrinter(indent=4)\nfrom pandas.tseries.holiday import USFederalHolidayCalendar as calendar\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"yneb3v6q2V6U","trusted":true},"cell_type":"code","source":"# project_dir=\"/content/drive/My Drive/ashrae-energy-prediction\"\n# project_dir=\".\"\nproject_dir = '/kaggle/input/ashrae-energy-prediction'\ndata_dir = project_dir","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","colab":{},"colab_type":"code","id":"6uhvlDAO2SMS","trusted":true},"cell_type":"code","source":"train = pd.read_csv(data_dir + \"/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"O16MIVJM2SMV","trusted":true},"cell_type":"code","source":"build = pd.read_csv(data_dir + \"/building_metadata.csv\")","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"ODa9kpkR2SMZ","trusted":true},"cell_type":"code","source":"weather_train = pd.read_csv(data_dir + \"/weather_train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_df(train, build, weather, should_compress=False, should_create_dummies=True):    \n    train_build = train.merge(right=build,left_on=\"building_id\", right_on=\"building_id\", how=\"left\")\n    train_build = train_build.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')\n    train_build[\"month\"] = pd.to_datetime(train_build[\"timestamp\"]).dt.month.astype(np.int8)\n    train_build[\"year\"] = pd.to_datetime(train_build[\"timestamp\"]).dt.year\n    train_build[\"day_of_week\"] = pd.to_datetime(train_build[\"timestamp\"]).dt.dayofweek\n#     train_build[\"is_weekend\"] = train_build[\"day_of_week\"] >= 5\n    dates_range = pd.date_range(start='2015-12-31', end='2019-01-01')\n    us_holidays = calendar().holidays(start=dates_range.min(), end=dates_range.max())\n    train_build['is_holiday'] = (pd.to_datetime(train_build['timestamp']).dt.date.astype('datetime64').isin(us_holidays))\n    train_build.loc[(train_build['day_of_week'] == 5) | (train_build['day_of_week'] == 6) , 'is_holiday'] = True\n    train_build[\"weekofmonth\"] = np.ceil(pd.to_datetime(train_build[\"timestamp\"]).dt.day/7).astype(np.int8)\n    month_season_bins = pd.IntervalIndex.from_tuples([(1, 2), (3, 5), (6, 8), (9, 11), (12, 12)])\n    encoder = LabelEncoder()\n    train_build[\"season\"] =  encoder.fit_transform(pd.cut(train_build[\"month\"], month_season_bins).astype(str)).astype(np.uint8)\n    train_build[\"hour\"] = pd.to_datetime(train_build[\"timestamp\"]).dt.hour.astype(np.int8)\n    half = np.floor(pd.to_datetime(train_build[\"timestamp\"]).dt.minute/30).astype(np.int8)\n    train_build[\"hour_half\"] = (train_build[\"hour\"] * 2 + half).astype(np.int8)\n    train_build[\"is_working_hour\"] = (train_build[\"hour\"] >= 8) & (train_build[\"hour\"] <= 18)\n    train_build['square_feet'] = np.log(train_build['square_feet'])\n    train_build[\"square_feet_per_floor\"] = train_build[\"square_feet\"]/train_build[\"floor_count\"]\n    train_build[\"square_feet_multiplied_by_floor\"] = train_build[\"square_feet\"] * train_build[\"floor_count\"]\n    train_build[\"age\"] = train_build[\"year\"] - train_build[\"year_built\"]\n    train_build.loc[(train_build['primary_use'] == \"Education\") & (train_build['month'] >= 6) & (train_build['month'] <= 8), 'is_vacation_month'] = np.int8(1)\n    train_build.loc[train_build['is_vacation_month']!=1, 'is_vacation_month'] = np.int8(0)\n    encoder = LabelEncoder()\n    train_build[\"primary_use\"] = encoder.fit_transform(train_build[\"primary_use\"]).astype(np.uint8)\n    \n    train_build_weather = train_build.merge(right=weather, left_on=[\"timestamp\", \"site_id\"], right_on=[\"timestamp\", \"site_id\"], how=\"left\")\n    train_build_weather[\"air_temperature_2\"] = train_build_weather[\"air_temperature\"] ** 2\n    train_build_weather[\"dew_temperature_2\"] = train_build_weather[\"dew_temperature\"] ** 2  \n    train_build_weather[\"wind_direction_cat\"] = train_build_weather[\"wind_direction\"]\n    replacement = {}\n    for i in range(0, 351, 30):\n        key = tuple(list(range(i, i+30)))\n        replacement[key] = i\n    for (key,value) in replacement.items():\n        train_build_weather[\"wind_direction_cat\"].replace(key, value, inplace=True)\n        gc.collect()\n    \n    beaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8), (6, 10.8, 13.9), \n          (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5), (11, 28.5, 33), (12, 33, 200)]\n    \n    for item in beaufort:\n        train_build_weather.loc[(train_build_weather['wind_speed']>=item[1]) & (train_build_weather['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n        gc.collect()\n#     train_build_weather[\"is_weekend\"] = pd.to_datetime(train_build_weather[\"timestamp\"]).dt.dayofweek >= 5\n    \n    train_build_weather.drop(columns=[\"timestamp\"], inplace=True)    \n    if \"meter_reading\" in train_build_weather.columns:\n      train_build_weather[\"meter_reading\"] = np.log1p(train_build_weather['meter_reading']).astype(np.float32)\n    return train_build_weather\n\ncat_cols = [\n            \"hour_half\",\n            \"is_working_hour\",\n            \"is_holiday\",\n#             \"day_of_week\",\n            \"season\",\n            \"is_vacation_month\",\n            \"site_id\", \n            \"building_id\", \n            \"meter\",\n            \"primary_use\",\n           \"wind_direction_cat\"\n           ]\ncols_drop_x = [\n                \"hour\",\n               \"weekofmonth\", \n               \"year\",\n               \"year_built\", \n               \"meter_reading\",\n#                \"wind_direction\",\n               \"sea_level_pressure\",\n               \"wind_speed\",\n                \"wind_direction\",\n#     'precip_depth_1_hr'\n              ]","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"sLXwjZxO-ETj","trusted":true},"cell_type":"code","source":"number_of_evals = 200\nprint(\"number_of_evals\", number_of_evals)\ndef find_best_params_for_lgb(X_train, y_train, X_val, y_val):\n    evaluated_point_scores = {}\n    \n    def objective(params):\n        garbage=gc.collect()\n        if (str(params) in evaluated_point_scores):\n            return evaluated_point_scores[str(params)]\n        else:          \n            \n            train_data = lgb.Dataset(X_train.values, \n                            label=y_train.values.ravel(),\n                            feature_name=list(X_train.columns),\n                            categorical_feature=cat_cols\n                            )\n            \n            validation_data = lgb.Dataset(X_val.values, \n                            label=y_val.values.ravel(),\n                            feature_name=list(X_val.columns),\n                            categorical_feature=cat_cols\n                             )\n            evals_result = {}\n            bst = lgb.train(params, train_data, \n                            valid_sets=[train_data, validation_data], \n                            valid_names=['train', 'val'], \n                            evals_result=evals_result, \n                            num_boost_round=10000,\n                            early_stopping_rounds=100,\n                     verbose_eval=100,\n                   categorical_feature=cat_cols)\n            \n            y_val_preds = bst.predict(X_val)\n            y_val_preds[y_val_preds < 0] = 0\n            score = np.sqrt(mean_squared_log_error( np.expm1(y_val_preds), np.expm1(y_val) ))\n#             print(\"Evaluating params:\")\n            pp.pprint(params)\n#             print(\"rmsle: \" + str(score))\n            evaluated_point_scores[str(params)] = score\n            f = open(\"params_lgb_score_\"+ str(number_of_evals)+ \".json\", \"w+\")\n            f.write(str(evaluated_point_scores))\n            f.close()\n            return score\n\n    param_space = {\n        'objective': hp.choice(\"objective\", [\"regression\"]),        \n        \"max_depth\": scope.int(hp.quniform(\"max_depth\", 6, 30, 1)),\n        \"learning_rate\": hp.choice(\"learning_rate\", [0.2]),\n        \"num_leaves\": scope.int(hp.quniform(\"num_leaves\", 50, 1000, 50)),\n        \"max_bin\": scope.int(hp.quniform(\"max_bin\", 50, 500, 10)),\n        \"bagging_fraction\": hp.quniform('bagging_fraction', 0.50, 1.0, 0.05),\n        \"bagging_freq\": hp.choice(\"bagging_freq\", [1]),\n        \"loss_function\": hp.choice(\"loss_function\", [\"RMSE\"]), \n        \"eval_metric\": hp.choice(\"eval_metric\", [\"RMSE\"]),\n        \"feature_fraction\": hp.uniform(\"feature_fraction\", 0.80, 1.0),\n        \"metric\": hp.choice(\"metric\", [\"rmse\"]),        \n        \"lambda_l1\": hp.quniform('lambda_l1', 1.0, 10.0, 1),\n        \"lambda_l2\": hp.quniform('lambda_l2', 1.0, 100.0, 5.0),\n        \"random_state\": hp.choice(\"random_state\", [7]),\n        \"verbose\": hp.choice(\"verbose\", [0]),\n        \"verbose_eval\": hp.choice(\"verbose_eval\", [False])\n    }\n    start_time = time.time()\n    best_params = space_eval(\n        param_space, \n        fmin(objective, \n             param_space, \n             algo=hyperopt.tpe.suggest,\n             max_evals=number_of_evals))\n    \n    pp.pprint(best_params)\n    elapsed_time = (time.time() - start_time) / 60\n    print('Elapsed computation time: {:.3f} mins'.format(elapsed_time))\n    print(\"Finding best number of iterations with learning rate: \", best_params[\"learning_rate\"])\n\n    train_data = lgb.Dataset(X_train.values, \n                            label=y_train.values.ravel(),\n                            feature_name=list(X_train.columns),\n                            categorical_feature=cat_cols\n                            )\n            \n    validation_data = lgb.Dataset(X_val.values, \n                    label=y_val.values.ravel(),\n                    feature_name=list(X_val.columns),\n                    categorical_feature=cat_cols\n                     )\n    evals_result = {}\n    bst = lgb.train(best_params, train_data, \n                    valid_sets=[train_data, validation_data], \n                    valid_names=['train', 'val'], \n                    evals_result=evals_result, \n                    num_boost_round=10000,\n                    early_stopping_rounds=100,\n                    verbose_eval=False,\n                    categorical_feature=cat_cols)\n    best_params[\"num_iterations\"] = bst.best_iteration\n    print (\"Best params:\")\n    pp.pprint(best_params)\n    return best_params","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"lCAOChSQ-EVN","trusted":true},"cell_type":"code","source":"gc.collect()\ndf = prepare_df(\n#                 train,\n                train.sample(n=1000000, random_state=7), \n                build, weather_train, should_compress=False).dropna(subset=[\"meter_reading\"])\ngarbage=gc.collect()\ntrain_df = df[df[\"month\"] <= 6]\nval_df = df[df[\"month\"] > 6].sample(n=100000, random_state=42)\nprint(\"training rows: \", train_df.shape[0])\nprint(\"validation rows: \", val_df.shape[0])\ndel(df)\nX_train = train_df.drop(columns=cols_drop_x)\ny_train = train_df[[\"meter_reading\"]]\ndel(train_df)\nX_val = val_df.drop(columns=cols_drop_x)\ny_val = val_df[[\"meter_reading\"]]\ndel(val_df)\ngarbage=gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":366},"colab_type":"code","id":"197gNoXn-EVX","outputId":"6aae6606-1fee-4017-bbf6-1cfe1cb0dd6a","trusted":true},"cell_type":"code","source":"best_params = find_best_params_for_lgb(X_train, y_train, X_val, y_val)","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"2PVvDI0wJPHG","trusted":true},"cell_type":"code","source":"f = open(\"best_params_lgb_\"+str(number_of_evals)+\"_.json\", \"w+\")\nf.write(str(best_params))\nf.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The steps for generating predictions on test data are saved in another notebook called prediction."}],"metadata":{"accelerator":"GPU","colab":{"collapsed_sections":[],"machine_shape":"hm","name":"kernel_lgb_with_param_tuning_10.ipynb","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}